Phase 13: integrations (research folder, git, Prometheus, signed webhooks)

Adds the integration layer that turns external sources into provenance-tagged
observations without ever executing what it reads.

- research folder poller: os.OpenRoot, text allowlist, binary/archive/symlink
  rejection, content fingerprints, duplicate tracking
- git poller: bare shallow mirror with hooks disabled and ext protocol refused,
  commit observations, co-change pairs
- Prometheus poller: threshold crossings and relative-change detection
- webhooks: HMAC over timestamp.body, +/-300s window, replay cache, 256 KiB cap,
  per-IP rate limit; refused entirely without a secret
- capabilities read_metrics (allow) and repository.read (ask), scoped to the
  invoking mind's integrations
- migrations 0013, API, UI pages, noemactl integrations, docs/integrations.md

cmd/noemactl/integrations.go

@@ -0,0 +1,91 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"text/tabwriter"
+)
+
+func init() {
+	extraCommands = append(extraCommands,
+		command{"integrations", "list|add <mind> <kind> <name> <target> [secret]|run <id>|runs <id>|delete <id> – external sources", cmdIntegrations},
+	)
+}
+
+func cmdIntegrations(args []string) int {
+	if len(args) < 1 {
+		fmt.Fprintln(os.Stderr, "usage: noemactl integrations list | add <mind> research_folder|git|prometheus|webhook <name> <path|url> [secret] | run <id> | runs <id> | delete <id>")
+		return exitUsage
+	}
+	switch args[0] {
+	case "list":
+		var out struct{ Items []map[string]any }
+		if code, err := apiRequest("GET", "/api/v1/integrations", nil, &out); err != nil {
+			return exitFor(code, err)
+		}
+		tw := tabwriter.NewWriter(os.Stdout, 0, 4, 2, ' ', 0)
+		fmt.Fprintln(tw, "NAME\tKIND\tSTATUS\tINGESTED\tID")
+		for _, i := range out.Items {
+			fmt.Fprintf(tw, "%v\t%v\t%v\t%v\t%v\n", i["name"], i["kind"], i["status"], i["ingested_count"], i["id"])
+		}
+		tw.Flush()
+		return exitOK
+	case "add":
+		if len(args) < 5 {
+			return exitUsage
+		}
+		cfg := map[string]any{}
+		switch args[2] {
+		case "research_folder":
+			cfg["path"] = args[4]
+		case "git":
+			cfg["url"] = args[4]
+		case "prometheus":
+			cfg["endpoint"] = args[4]
+		}
+		body := map[string]any{"mind": args[1], "kind": args[2], "name": args[3], "config": cfg, "enabled": true}
+		if len(args) > 5 {
+			body["secret"] = strings.Join(args[5:], " ")
+		}
+		var res map[string]any
+		if code, err := apiRequest("POST", "/api/v1/integrations", body, &res); err != nil {
+			return exitFor(code, err)
+		}
+		fmt.Println("created", res["id"])
+		return exitOK
+	case "run":
+		if len(args) < 2 {
+			return exitUsage
+		}
+		var res map[string]any
+		if code, err := apiRequest("POST", "/api/v1/integrations/"+args[1]+"/run", nil, &res); err != nil {
+			return exitFor(code, err)
+		}
+		run, _ := res["run"].(map[string]any)
+		fmt.Printf("items=%v observations=%v errors=%v %v\n", run["items"], run["observations"], run["errors"], run["error"])
+		return exitOK
+	case "runs":
+		if len(args) < 2 {
+			return exitUsage
+		}
+		var out struct{ Items []map[string]any }
+		if code, err := apiRequest("GET", "/api/v1/integrations/"+args[1]+"/runs", nil, &out); err != nil {
+			return exitFor(code, err)
+		}
+		for _, r := range out.Items {
+			fmt.Printf("%v items=%v obs=%v errors=%v %v\n", r["started_at"], r["items"], r["observations"], r["errors"], r["error"])
+		}
+		return exitOK
+	case "delete":
+		if len(args) < 2 {
+			return exitUsage
+		}
+		if code, err := apiRequest("DELETE", "/api/v1/integrations/"+args[1], nil, nil); err != nil {
+			return exitFor(code, err)
+		}
+		fmt.Println("deleted")
+		return exitOK
+	}
+	return exitUsage
+}

cmd/noemad/main.go

@@ -27,6 +27,7 @@ import (
 	evstore "github.com/andydixon/noema/internal/events"
 	"github.com/andydixon/noema/internal/goals"
 	"github.com/andydixon/noema/internal/graph"
+	"github.com/andydixon/noema/internal/integrations"
 	"github.com/andydixon/noema/internal/llm/prompts"
 	"github.com/andydixon/noema/internal/llm/router"
 	"github.com/andydixon/noema/internal/memory"
@@ -137,6 +138,11 @@ func run() error {
 	for _, c := range capabilities.Builtins() {
 		capReg.Add(c)
 	}
+	deps.Integrations = &integrations.Manager{Store: integrations.NewStore(db), Events: deps.Events, Graph: deps.Graph, Secrets: sec, Log: log, Pool: scheduler.NewPool(2),
+		Pollers: map[integrations.Kind]integrations.Poller{integrations.ResearchFolder: integrations.ResearchPoller{}, integrations.Git: integrations.GitPoller{DataDir: "./data/git"}, integrations.Prometheus: integrations.PrometheusPoller{Secrets: sec}}}
+	for _, c := range integrations.Capabilities(deps.Integrations, integrations.GitPoller{DataDir: "./data/git"}, integrations.PrometheusPoller{Secrets: sec}) {
+		capReg.Add(c)
+	}
 	deps.Capabilities = &capabilities.Service{Registry: capReg, DB: db, Events: deps.Events, Log: log, Metrics: metrics, ApprovalTTL: 24 * time.Hour, SessionTTL: 8 * time.Hour}
 	selfSync := cognition.SelfModelSync{Minds: deps.Minds, Caps: deps.Capabilities, Events: deps.Events, Log: log}
 	deps.Capabilities.OnSelfModel = selfSync.Sync
@@ -152,6 +158,7 @@ func run() error {
 		Log: log, Metrics: metrics, Services: brain.Services{Events: deps.Events, Memory: deps.Memory, Beliefs: deps.Beliefs, LLM: deps.LLM, Prompts: deps.Prompts,
 			Goals: deps.Goals, Motivation: deps.Motivation, Predictions: deps.Predictions, Capabilities: deps.Capabilities}}, group)
 	selfSync.Sync(ctx, ids.Nil)
+	group.Every("integrations.tick", time.Minute, deps.Integrations.Tick)
 	group.Every("approvals.expire", 5*time.Minute, func(ctx context.Context) error { _, err := deps.Capabilities.ExpirePending(ctx); return err })
 	group.Every("predictions.resolve", time.Minute, deps.Resolver.Run)
 	group.Every("cognition.background", 5*time.Minute, func(ctx context.Context) error { return backgroundCognition(ctx, deps) })