Phase 13: integrations (research folder, git, Prometheus, signed webhooks)

Adds the integration layer that turns external sources into provenance-tagged
observations without ever executing what it reads.

- research folder poller: os.OpenRoot, text allowlist, binary/archive/symlink
  rejection, content fingerprints, duplicate tracking
- git poller: bare shallow mirror with hooks disabled and ext protocol refused,
  commit observations, co-change pairs
- Prometheus poller: threshold crossings and relative-change detection
- webhooks: HMAC over timestamp.body, +/-300s window, replay cache, 256 KiB cap,
  per-IP rate limit; refused entirely without a secret
- capabilities read_metrics (allow) and repository.read (ask), scoped to the
  invoking mind's integrations
- migrations 0013, API, UI pages, noemactl integrations, docs/integrations.md
This commit is contained in:
Andy Dixon
2026-06-29 21:04:50 +01:00
parent b35dbe0299
commit 89edbb7c06
26 changed files with 2391 additions and 7 deletions
+53
View File
@@ -0,0 +1,53 @@
package api
import (
"sync"
"time"
)
// ipBucket is a small token bucket keyed by string, used to bound webhook
// deliveries per integration (T18). Auth has its own limiter.
type ipBucket struct {
mu sync.Mutex
buckets map[string]*tokens
rate float64
burst float64
}
type tokens struct {
n float64
last time.Time
}
func newIPBucket(perMinute, burst int) *ipBucket {
return &ipBucket{buckets: map[string]*tokens{}, rate: float64(perMinute) / 60, burst: float64(burst)}
}
// Allow consumes a token for key.
func (b *ipBucket) Allow(key string) bool {
b.mu.Lock()
defer b.mu.Unlock()
now := time.Now()
t := b.buckets[key]
if t == nil {
t = &tokens{n: b.burst, last: now}
b.buckets[key] = t
}
t.n += now.Sub(t.last).Seconds() * b.rate
if t.n > b.burst {
t.n = b.burst
}
t.last = now
if len(b.buckets) > 10000 {
for k, v := range b.buckets {
if now.Sub(v.last) > time.Hour {
delete(b.buckets, k)
}
}
}
if t.n < 1 {
return false
}
t.n--
return true
}