Phase 13: integrations (research folder, git, Prometheus, signed webhooks)
Adds the integration layer that turns external sources into provenance-tagged observations without ever executing what it reads. - research folder poller: os.OpenRoot, text allowlist, binary/archive/symlink rejection, content fingerprints, duplicate tracking - git poller: bare shallow mirror with hooks disabled and ext protocol refused, commit observations, co-change pairs - Prometheus poller: threshold crossings and relative-change detection - webhooks: HMAC over timestamp.body, +/-300s window, replay cache, 256 KiB cap, per-IP rate limit; refused entirely without a secret - capabilities read_metrics (allow) and repository.read (ask), scoped to the invoking mind's integrations - migrations 0013, API, UI pages, noemactl integrations, docs/integrations.md
This commit is contained in:
Andy Dixon
@@ -22,6 +22,7 @@ import (
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/goals"
|
||||
"github.com/andydixon/noema/internal/graph"
|
||||
"github.com/andydixon/noema/internal/integrations"
|
||||
"github.com/andydixon/noema/internal/llm/prompts"
|
||||
"github.com/andydixon/noema/internal/llm/router"
|
||||
"github.com/andydixon/noema/internal/memory"
|
||||
@@ -57,6 +58,7 @@ func testServer(t *testing.T) (*httptest.Server, string) {
|
||||
capReg.Add(c)
|
||||
}
|
||||
deps.Capabilities = &capabilities.Service{Registry: capReg, DB: pool, Events: deps.Events, Log: log}
|
||||
deps.Integrations = &integrations.Manager{Store: integrations.NewStore(pool), Events: deps.Events, Graph: deps.Graph, Log: log, Pollers: map[integrations.Kind]integrations.Poller{}}
|
||||
deps.Narrative = narrative.NewStore(pool)
|
||||
deps.Notifications = notifications.NewStore(pool)
|
||||
deps.Consolidation = &consolidation.Service{DB: pool, Memory: deps.Memory, Beliefs: deps.Beliefs, Events: deps.Events, Minds: deps.Minds, Log: log}
|
||||
|
||||
@@ -0,0 +1,263 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/auth"
|
||||
"github.com/andydixon/noema/internal/integrations"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
func (s *Server) integrationRoutes() {
|
||||
m := s.mux
|
||||
m.Handle("GET /api/v1/integrations", require(auth.RoleObserver, s.integrationsList))
|
||||
m.Handle("POST /api/v1/integrations", require(auth.RoleAdministrator, s.integrationsCreate))
|
||||
m.Handle("GET /api/v1/integrations/{id}", require(auth.RoleObserver, s.integrationGet))
|
||||
m.Handle("PUT /api/v1/integrations/{id}", require(auth.RoleAdministrator, s.integrationPut))
|
||||
m.Handle("DELETE /api/v1/integrations/{id}", require(auth.RoleAdministrator, s.integrationDelete))
|
||||
m.Handle("POST /api/v1/integrations/{id}/run", require(auth.RoleOperator, s.integrationRun))
|
||||
m.Handle("GET /api/v1/integrations/{id}/runs", require(auth.RoleObserver, s.integrationRuns))
|
||||
m.Handle("GET /api/v1/integrations/{id}/documents", require(auth.RoleObserver, s.integrationDocuments))
|
||||
// Webhook delivery: authenticated by signature, not by session.
|
||||
m.HandleFunc("POST /api/v1/webhooks/{id}", s.webhookReceive)
|
||||
}
|
||||
|
||||
func integErr(err error) error {
|
||||
if errors.Is(err, integrations.ErrInvalid) {
|
||||
return ValidationError(err.Error())
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
type integrationIn struct {
|
||||
MindID string `json:"mind"` // slug or id
|
||||
Kind integrations.Kind `json:"kind"`
|
||||
Name string `json:"name"`
|
||||
Config map[string]any `json:"config"`
|
||||
Secret string `json:"secret"` // plaintext (encrypted) or env:/file:/enc:; empty keeps existing
|
||||
Trust events.Trust `json:"trust"`
|
||||
Enabled *bool `json:"enabled"`
|
||||
}
|
||||
|
||||
func (s *Server) integrationInput(r *http.Request, in integrationIn) (integrations.Input, error) {
|
||||
var mind ids.ID
|
||||
if in.MindID != "" {
|
||||
m, err := s.deps.Minds.Resolve(r.Context(), in.MindID)
|
||||
if err != nil {
|
||||
return integrations.Input{}, err
|
||||
}
|
||||
mind = m.ID
|
||||
}
|
||||
ref, err := s.keyRef(in.Secret)
|
||||
if err != nil {
|
||||
return integrations.Input{}, ValidationError(err.Error())
|
||||
}
|
||||
enabled := true
|
||||
if in.Enabled != nil {
|
||||
enabled = *in.Enabled
|
||||
}
|
||||
return integrations.Input{MindID: mind, Kind: in.Kind, Name: in.Name, Config: in.Config, SecretRef: ref, Trust: in.Trust, Enabled: enabled}, nil
|
||||
}
|
||||
|
||||
func (s *Server) integrationsList(w http.ResponseWriter, r *http.Request) {
|
||||
var mind ids.ID
|
||||
if v := r.URL.Query().Get("mind"); v != "" {
|
||||
if m, err := s.deps.Minds.Resolve(r.Context(), v); err == nil {
|
||||
mind = m.ID
|
||||
}
|
||||
}
|
||||
list, err := s.deps.Integrations.Store.List(r.Context(), mind)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
if list == nil {
|
||||
list = []integrations.Integration{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, page[integrations.Integration]{Items: list})
|
||||
}
|
||||
|
||||
func (s *Server) integrationsCreate(w http.ResponseWriter, r *http.Request) {
|
||||
var in integrationIn
|
||||
if err := decode(r, &in); err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
input, err := s.integrationInput(r, in)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
i, err := s.deps.Integrations.Store.Create(r.Context(), input)
|
||||
if err != nil {
|
||||
fail(w, r, integErr(err))
|
||||
return
|
||||
}
|
||||
s.deps.Audit.Log(r.Context(), "integration.create", "integration", i.ID.String(), map[string]any{"kind": i.Kind, "name": i.Name, "mind": i.MindID})
|
||||
s.deps.Events.OperatorAction(r.Context(), i.MindID, actor(r), "integration.create", "Operator "+actor(r)+" connected "+string(i.Kind)+" integration "+i.Name, map[string]any{"integration_id": i.ID})
|
||||
writeJSON(w, http.StatusCreated, i)
|
||||
}
|
||||
|
||||
func (s *Server) integration(w http.ResponseWriter, r *http.Request) (integrations.Integration, bool) {
|
||||
id, err := ids.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, r, ValidationError("bad id"))
|
||||
return integrations.Integration{}, false
|
||||
}
|
||||
i, err := s.deps.Integrations.Store.Get(r.Context(), id)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return integrations.Integration{}, false
|
||||
}
|
||||
return i, true
|
||||
}
|
||||
|
||||
func (s *Server) integrationGet(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
runs, _ := s.deps.Integrations.Store.Runs(r.Context(), i.ID, 10)
|
||||
docs, _ := s.deps.Integrations.Store.Documents(r.Context(), i.ID, 20)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"integration": i, "runs": runs, "documents": docs})
|
||||
}
|
||||
|
||||
func (s *Server) integrationPut(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var in integrationIn
|
||||
if err := decode(r, &in); err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
input, err := s.integrationInput(r, in)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
out, err := s.deps.Integrations.Store.Update(r.Context(), i.ID, input)
|
||||
if err != nil {
|
||||
fail(w, r, integErr(err))
|
||||
return
|
||||
}
|
||||
s.deps.Audit.Log(r.Context(), "integration.update", "integration", i.ID.String(), map[string]any{"enabled": out.Enabled, "secret_changed": in.Secret != ""})
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (s *Server) integrationDelete(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := s.deps.Integrations.Store.Delete(r.Context(), i.ID); err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
s.deps.Audit.Log(r.Context(), "integration.delete", "integration", i.ID.String(), map[string]any{"name": i.Name})
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}
|
||||
|
||||
func (s *Server) integrationRun(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
run, err := s.deps.Integrations.RunOne(r.Context(), i.ID)
|
||||
s.deps.Audit.Log(r.Context(), "integration.run", "integration", i.ID.String(), map[string]any{"observations": run.Observations, "error": run.Error})
|
||||
if err != nil {
|
||||
writeJSON(w, http.StatusOK, map[string]any{"run": run, "error": err.Error()})
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"run": run})
|
||||
}
|
||||
|
||||
func (s *Server) integrationRuns(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
runs, err := s.deps.Integrations.Store.Runs(r.Context(), i.ID, limit)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
if runs == nil {
|
||||
runs = []integrations.Run{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, page[integrations.Run]{Items: runs})
|
||||
}
|
||||
|
||||
func (s *Server) integrationDocuments(w http.ResponseWriter, r *http.Request) {
|
||||
i, ok := s.integration(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
limit, _ := strconv.Atoi(r.URL.Query().Get("limit"))
|
||||
docs, err := s.deps.Integrations.Store.Documents(r.Context(), i.ID, limit)
|
||||
if err != nil {
|
||||
fail(w, r, err)
|
||||
return
|
||||
}
|
||||
if docs == nil {
|
||||
docs = []integrations.Document{}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, page[integrations.Document]{Items: docs})
|
||||
}
|
||||
|
||||
// webhookReceive verifies the HMAC signature and timestamp, rejects replays,
|
||||
// and ingests the body as an observation. Unsigned integrations (no secret)
|
||||
// are refused outright.
|
||||
func (s *Server) webhookReceive(w http.ResponseWriter, r *http.Request) {
|
||||
id, err := ids.Parse(r.PathValue("id"))
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "no such webhook")
|
||||
return
|
||||
}
|
||||
i, err := s.deps.Integrations.Store.Get(r.Context(), id)
|
||||
if err != nil || i.Kind != integrations.Webhook || !i.Enabled {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "no such webhook")
|
||||
return
|
||||
}
|
||||
if !s.webhookLimiter.Allow(id.String()) {
|
||||
writeError(w, r, http.StatusTooManyRequests, "rate_limited", "slow down")
|
||||
return
|
||||
}
|
||||
if i.SecretRef == "" {
|
||||
writeError(w, r, http.StatusForbidden, "unsigned", "this webhook has no secret configured; unsigned deliveries are refused")
|
||||
return
|
||||
}
|
||||
secret, err := s.deps.Secrets.Resolve(i.SecretRef)
|
||||
if err != nil {
|
||||
writeError(w, r, http.StatusInternalServerError, "secret", "webhook secret unavailable")
|
||||
return
|
||||
}
|
||||
body, err := io.ReadAll(io.LimitReader(r.Body, 256<<10+1))
|
||||
if err != nil || len(body) > 256<<10 {
|
||||
writeError(w, r, http.StatusRequestEntityTooLarge, "too_large", "body over 256 KiB")
|
||||
return
|
||||
}
|
||||
sig := r.Header.Get("X-Noema-Signature")
|
||||
if err := integrations.Verify(secret, r.Header.Get("X-Noema-Timestamp"), sig, body, time.Now()); err != nil {
|
||||
s.deps.Audit.Log(r.Context(), "webhook.rejected", "integration", id.String(), map[string]any{"reason": err.Error()})
|
||||
writeError(w, r, http.StatusUnauthorized, "signature", err.Error())
|
||||
return
|
||||
}
|
||||
if s.replays.Check(sig, time.Now()) {
|
||||
writeError(w, r, http.StatusConflict, "replay", integrations.ErrReplay.Error())
|
||||
return
|
||||
}
|
||||
ev, err := s.deps.Integrations.Ingest(r.Context(), i, body)
|
||||
if err != nil {
|
||||
fail(w, r, integErr(err))
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusAccepted, map[string]any{"event_id": ev.ID})
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ipBucket is a small token bucket keyed by string, used to bound webhook
|
||||
// deliveries per integration (T18). Auth has its own limiter.
|
||||
type ipBucket struct {
|
||||
mu sync.Mutex
|
||||
buckets map[string]*tokens
|
||||
rate float64
|
||||
burst float64
|
||||
}
|
||||
|
||||
type tokens struct {
|
||||
n float64
|
||||
last time.Time
|
||||
}
|
||||
|
||||
func newIPBucket(perMinute, burst int) *ipBucket {
|
||||
return &ipBucket{buckets: map[string]*tokens{}, rate: float64(perMinute) / 60, burst: float64(burst)}
|
||||
}
|
||||
|
||||
// Allow consumes a token for key.
|
||||
func (b *ipBucket) Allow(key string) bool {
|
||||
b.mu.Lock()
|
||||
defer b.mu.Unlock()
|
||||
now := time.Now()
|
||||
t := b.buckets[key]
|
||||
if t == nil {
|
||||
t = &tokens{n: b.burst, last: now}
|
||||
b.buckets[key] = t
|
||||
}
|
||||
t.n += now.Sub(t.last).Seconds() * b.rate
|
||||
if t.n > b.burst {
|
||||
t.n = b.burst
|
||||
}
|
||||
t.last = now
|
||||
if len(b.buckets) > 10000 {
|
||||
for k, v := range b.buckets {
|
||||
if now.Sub(v.last) > time.Hour {
|
||||
delete(b.buckets, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
if t.n < 1 {
|
||||
return false
|
||||
}
|
||||
t.n--
|
||||
return true
|
||||
}
|
||||
@@ -7,18 +7,21 @@ import (
|
||||
|
||||
"github.com/andydixon/noema/internal/app"
|
||||
"github.com/andydixon/noema/internal/auth"
|
||||
"github.com/andydixon/noema/internal/integrations"
|
||||
)
|
||||
|
||||
// Server composes the API routes, the web UI and the middleware chain.
|
||||
type Server struct {
|
||||
deps app.Deps
|
||||
mux *http.ServeMux
|
||||
deps app.Deps
|
||||
mux *http.ServeMux
|
||||
replays *integrations.ReplayCache
|
||||
webhookLimiter *ipBucket
|
||||
}
|
||||
|
||||
// New builds the server. ui registers the HTML routes on the same mux so that
|
||||
// a single middleware chain (auth, CSRF, headers, limits) covers both.
|
||||
func New(deps app.Deps, ui interface{ Register(*http.ServeMux) }) *Server {
|
||||
s := &Server{deps: deps, mux: http.NewServeMux()}
|
||||
s := &Server{deps: deps, mux: http.NewServeMux(), replays: integrations.NewReplayCache(), webhookLimiter: newIPBucket(120, 60)}
|
||||
s.routes()
|
||||
if ui != nil {
|
||||
ui.Register(s.mux)
|
||||
@@ -64,6 +67,7 @@ func (s *Server) routes() {
|
||||
s.cognitionRoutes()
|
||||
s.graphRoutes()
|
||||
s.capabilityRoutes()
|
||||
s.integrationRoutes()
|
||||
|
||||
m.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, r, http.StatusNotFound, "not_found", "no such endpoint")
|
||||
|
||||
Reference in New Issue
Block a user