feat: add immutable cognitive event store, live streams and event explorer

Phase 3. Events are the append-only record of everything a mind does
(ADR-0002). pkg/events defines the closed type vocabulary, categories,
evidence references and the observation payload with explicit trust
levels. The store validates and appends inside the caller's transaction,
publishes to a bounded fan-out bus only after commit, queries by seq
cursor with type/category/source/salience/confidence/time/search
filters, and walks provenance: cause, parents, children, cited evidence,
derived events and correlation groups. Durable per-consumer cursors never
move backwards.

Server-Sent Events streams (per mind and global) subscribe before
replaying history after Last-Event-ID so nothing is missed, apply the
same filter vocabulary live, and heartbeat. Operators can inject
observations tagged with operator trust for demos and tests.

UI: live thought stream with filters and paging, forensic event page
with causal chain and evidence navigation, dashboard important events.
Personality and status changes now leave operator_action events in the
mind's own history. CLI: event inspect/list/tail, observe.

internal/api/api_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/andydixon/noema/internal/audit"
 	"github.com/andydixon/noema/internal/auth"
 	"github.com/andydixon/noema/internal/config"
+	evstore "github.com/andydixon/noema/internal/events"
 	"github.com/andydixon/noema/internal/minds"
 	"github.com/andydixon/noema/internal/observability"
 	"github.com/andydixon/noema/internal/storage"
@@ -27,7 +28,7 @@ func testServer(t *testing.T) (*httptest.Server, string) {
 	cfg.DatabaseURL = "postgres://test"
 	log := slog.New(slog.DiscardHandler)
 	deps := app.Deps{Config: cfg, Log: log, DB: pool, Auth: auth.NewService(pool, time.Hour, 2*time.Hour),
-		Audit: audit.New(pool, log), Metrics: observability.NewMetrics(), Minds: minds.NewStore(pool), Migrations: migrations.FS, Started: time.Now()}
+		Audit: audit.New(pool, log), Metrics: observability.NewMetrics(), Minds: minds.NewStore(pool), Events: evstore.NewStore(pool, evstore.NewBus(50), nil), Migrations: migrations.FS, Started: time.Now()}
 	ctx := context.Background()
 	u, err := deps.Auth.CreateUser(ctx, "admin", "administrator-pass", auth.RoleAdministrator)
 	if err != nil {

internal/api/events.go

@@ -0,0 +1,335 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/andydixon/noema/internal/auth"
+	evstore "github.com/andydixon/noema/internal/events"
+	"github.com/andydixon/noema/pkg/events"
+	"github.com/andydixon/noema/pkg/ids"
+)
+
+func (s *Server) eventRoutes() {
+	m := s.mux
+	m.Handle("GET /api/v1/minds/{mind}/events", require(auth.RoleObserver, s.eventsList))
+	m.Handle("GET /api/v1/minds/{mind}/events/stream", require(auth.RoleObserver, s.eventsStream))
+	m.Handle("GET /api/v1/minds/{mind}/events/stats", require(auth.RoleObserver, s.eventsStats))
+	m.Handle("POST /api/v1/minds/{mind}/observations", require(auth.RoleOperator, s.observationCreate))
+	m.Handle("GET /api/v1/events/stream", require(auth.RoleObserver, s.eventsStream))
+	m.Handle("GET /api/v1/events/{id}", require(auth.RoleObserver, s.eventGet))
+	m.Handle("GET /api/v1/events/{id}/context", require(auth.RoleObserver, s.eventContext))
+	m.Handle("GET /api/v1/events/types", require(auth.RoleObserver, func(w http.ResponseWriter, r *http.Request) {
+		writeJSON(w, http.StatusOK, map[string]any{"types": events.AllTypes()})
+	}))
+}
+
+// ParseEventFilter reads the shared query-string filter vocabulary used by
+// the API, SSE stream and UI pages.
+func ParseEventFilter(q url.Values) evstore.Filter {
+	f := evstore.Filter{}
+	for _, t := range q["type"] {
+		for _, part := range strings.Split(t, ",") {
+			if part = strings.TrimSpace(part); part != "" {
+				f.Types = append(f.Types, events.Type(part))
+			}
+		}
+	}
+	for _, c := range q["category"] {
+		for _, part := range strings.Split(c, ",") {
+			if part = strings.TrimSpace(part); part != "" {
+				f.Categories = append(f.Categories, events.Category(part))
+			}
+		}
+	}
+	for _, src := range q["source"] {
+		if src = strings.TrimSpace(src); src != "" {
+			f.Sources = append(f.Sources, src)
+		}
+	}
+	f.MinSalience, _ = strconv.ParseFloat(q.Get("min_salience"), 64)
+	f.MinConfidence, _ = strconv.ParseFloat(q.Get("min_confidence"), 64)
+	f.MaxConfidence, _ = strconv.ParseFloat(q.Get("max_confidence"), 64)
+	if v := q.Get("since"); v != "" {
+		if t, err := time.Parse(time.RFC3339, v); err == nil {
+			f.Since = t
+		} else if d, err := time.ParseDuration(v); err == nil {
+			f.Since = time.Now().Add(-d)
+		}
+	}
+	if v := q.Get("until"); v != "" {
+		if t, err := time.Parse(time.RFC3339, v); err == nil {
+			f.Until = t
+		}
+	}
+	f.AfterSeq, _ = strconv.ParseInt(q.Get("after"), 10, 64)
+	f.BeforeSeq, _ = strconv.ParseInt(q.Get("before"), 10, 64)
+	if v := q.Get("correlation"); v != "" {
+		if id, err := ids.Parse(v); err == nil {
+			f.CorrelationID = id
+		}
+	}
+	f.Search = q.Get("q")
+	f.Limit, _ = strconv.Atoi(q.Get("limit"))
+	f.Ascending = q.Get("order") == "asc"
+	return f
+}
+
+func (s *Server) eventsList(w http.ResponseWriter, r *http.Request) {
+	m, ok := s.mind(w, r)
+	if !ok {
+		return
+	}
+	f := ParseEventFilter(r.URL.Query())
+	f.MindID = m.ID
+	list, err := s.deps.Events.Query(r.Context(), f)
+	if err != nil {
+		fail(w, r, err)
+		return
+	}
+	if list == nil {
+		list = []events.Event{}
+	}
+	out := page[events.Event]{Items: list}
+	if len(list) > 0 && (f.Limit == 0 && len(list) == 100 || f.Limit > 0 && len(list) == f.Limit) {
+		out.NextCursor = strconv.FormatInt(list[len(list)-1].Seq, 10)
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+func (s *Server) eventsStats(w http.ResponseWriter, r *http.Request) {
+	m, ok := s.mind(w, r)
+	if !ok {
+		return
+	}
+	var since time.Time
+	if d, err := time.ParseDuration(r.URL.Query().Get("window")); err == nil {
+		since = time.Now().Add(-d)
+	}
+	st, err := s.deps.Events.Stats(r.Context(), m.ID, since)
+	if err != nil {
+		fail(w, r, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, st)
+}
+
+func (s *Server) eventGet(w http.ResponseWriter, r *http.Request) {
+	id, err := ids.Parse(r.PathValue("id"))
+	if err != nil {
+		fail(w, r, ValidationError("bad id"))
+		return
+	}
+	ev, err := s.deps.Events.Get(r.Context(), id)
+	if err != nil {
+		fail(w, r, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, ev)
+}
+
+// EventContext is the forensic view: the event, what caused it, what it
+// caused, what it cites and what cites it.
+type EventContext struct {
+	Event       events.Event         `json:"event"`
+	Cause       *events.Event        `json:"cause,omitempty"`
+	Parents     []events.Event       `json:"parents"`
+	Children    []events.Event       `json:"children"`
+	Evidence    []events.Event       `json:"evidence_events"`
+	Derived     []events.Event       `json:"derived"`
+	Correlated  []events.Event       `json:"correlated"`
+	EvidenceAll []events.EvidenceRef `json:"evidence"`
+}
+
+// Context assembles the forensic context for an event; shared with the UI.
+func (s *Server) Context(r *http.Request, id ids.ID) (EventContext, error) {
+	ctx := r.Context()
+	ev, err := s.deps.Events.Get(ctx, id)
+	if err != nil {
+		return EventContext{}, err
+	}
+	out := EventContext{Event: ev, Parents: []events.Event{}, Children: []events.Event{}, Evidence: []events.Event{}, Derived: []events.Event{}, Correlated: []events.Event{}, EvidenceAll: ev.Evidence}
+	if !ev.CausationID.IsZero() {
+		if c, err := s.deps.Events.Get(ctx, ev.CausationID); err == nil {
+			out.Cause = &c
+		}
+	}
+	if len(ev.ParentIDs) > 0 {
+		out.Parents, _ = s.deps.Events.GetMany(ctx, ev.ParentIDs)
+	}
+	out.Children, _ = s.deps.Events.Children(ctx, ev.ID, 50)
+	var evIDs []ids.ID
+	for _, ref := range ev.Evidence {
+		if ref.Kind == "event" {
+			if eid, err := ids.Parse(ref.ID); err == nil {
+				evIDs = append(evIDs, eid)
+			}
+		}
+	}
+	if len(evIDs) > 0 {
+		out.Evidence, _ = s.deps.Events.GetMany(ctx, evIDs)
+	}
+	out.Derived, _ = s.deps.Events.DerivedFrom(ctx, "event", string(ev.ID), 50)
+	if !ev.CorrelationID.IsZero() {
+		out.Correlated, _ = s.deps.Events.Query(ctx, evstore.Filter{MindID: ev.MindID, CorrelationID: ev.CorrelationID, Ascending: true, Limit: 200})
+	}
+	for _, l := range []*[]events.Event{&out.Parents, &out.Children, &out.Evidence, &out.Derived, &out.Correlated} {
+		if *l == nil {
+			*l = []events.Event{}
+		}
+	}
+	return out, nil
+}
+
+func (s *Server) eventContext(w http.ResponseWriter, r *http.Request) {
+	id, err := ids.Parse(r.PathValue("id"))
+	if err != nil {
+		fail(w, r, ValidationError("bad id"))
+		return
+	}
+	out, err := s.Context(r, id)
+	if err != nil {
+		fail(w, r, err)
+		return
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+// observationCreate lets an operator inject an observation directly. It is
+// tagged with operator trust and used by demos, tests and the CLI.
+func (s *Server) observationCreate(w http.ResponseWriter, r *http.Request) {
+	m, ok := s.mind(w, r)
+	if !ok {
+		return
+	}
+	var in struct {
+		Text      string  `json:"text"`
+		Source    string  `json:"source"`
+		Reference string  `json:"reference"`
+		Salience  float64 `json:"salience"`
+	}
+	if err := decode(r, &in); err != nil {
+		fail(w, r, err)
+		return
+	}
+	in.Text = strings.TrimSpace(in.Text)
+	if in.Text == "" || len(in.Text) > 20000 {
+		fail(w, r, ValidationError("text required (max 20000 chars)"))
+		return
+	}
+	if in.Source == "" {
+		in.Source = "operator"
+	}
+	if in.Salience == 0 {
+		in.Salience = 0.5
+	}
+	summary := in.Text
+	if len(summary) > 160 {
+		summary = summary[:157] + "..."
+	}
+	evs, err := s.deps.Events.Append(r.Context(), events.New{MindID: m.ID, Type: events.Observation, Source: in.Source, Summary: summary,
+		Salience: in.Salience, Confidence: 1, Metadata: map[string]any{"trust": events.TrustOperator, "actor": actor(r)},
+		Payload: events.ObservationPayload{Text: in.Text, ContentType: "text/plain", Source: in.Source, Trust: events.TrustOperator, Method: "api", Reference: in.Reference, Actor: actor(r)}})
+	if err != nil {
+		fail(w, r, err)
+		return
+	}
+	s.deps.Audit.Log(r.Context(), "observation.inject", "mind", m.ID.String(), map[string]any{"event_id": evs[0].ID, "source": in.Source})
+	writeJSON(w, http.StatusCreated, evs[0])
+}
+
+// eventsStream is the Server-Sent Events feed. It subscribes before replaying
+// history after Last-Event-ID so no event is missed, then relays live events
+// that match the filter. Heartbeats keep proxies from closing idle streams.
+func (s *Server) eventsStream(w http.ResponseWriter, r *http.Request) {
+	f := ParseEventFilter(r.URL.Query())
+	if r.PathValue("mind") != "" {
+		m, ok := s.mind(w, r)
+		if !ok {
+			return
+		}
+		f.MindID = m.ID
+	}
+	flusher, ok := w.(http.Flusher)
+	if !ok {
+		writeError(w, r, http.StatusInternalServerError, "internal", "streaming unsupported")
+		return
+	}
+	ctx := r.Context()
+	ch, err := s.deps.Events.Bus().Subscribe(ctx, f.MindID, 256)
+	if err != nil {
+		var tooMany evstore.ErrTooManySubscribers
+		if errors.As(err, &tooMany) {
+			writeError(w, r, http.StatusServiceUnavailable, "busy", "too many live streams")
+			return
+		}
+		fail(w, r, err)
+		return
+	}
+	h := w.Header()
+	h.Set("Content-Type", "text/event-stream")
+	h.Set("Cache-Control", "no-cache")
+	h.Set("X-Accel-Buffering", "no")
+	w.WriteHeader(http.StatusOK)
+	fmt.Fprint(w, "retry: 3000\n\n")
+	flusher.Flush()
+
+	var lastSeq int64
+	if v := r.Header.Get("Last-Event-ID"); v != "" {
+		lastSeq, _ = strconv.ParseInt(v, 10, 64)
+	} else if f.AfterSeq > 0 {
+		lastSeq = f.AfterSeq
+	}
+	send := func(ev events.Event) bool {
+		if ev.Seq <= lastSeq {
+			return true
+		}
+		lastSeq = ev.Seq
+		data, _ := json.Marshal(ev)
+		if _, err := fmt.Fprintf(w, "id: %d\nevent: cognitive_event\ndata: %s\n\n", ev.Seq, data); err != nil {
+			return false
+		}
+		flusher.Flush()
+		return true
+	}
+	if lastSeq > 0 {
+		replay := f
+		replay.AfterSeq, replay.Ascending, replay.Limit = lastSeq, true, 500
+		if hist, err := s.deps.Events.Query(ctx, replay); err == nil {
+			for _, ev := range hist {
+				if !send(ev) {
+					return
+				}
+			}
+		}
+	}
+	heartbeat := time.NewTicker(15 * time.Second)
+	defer heartbeat.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-heartbeat.C:
+			if _, err := fmt.Fprint(w, ": ping\n\n"); err != nil {
+				return
+			}
+			flusher.Flush()
+		case ev, ok := <-ch:
+			if !ok {
+				return
+			}
+			if !f.Matches(ev) {
+				continue
+			}
+			if !send(ev) {
+				return
+			}
+		}
+	}
+}

internal/api/events_test.go

@@ -0,0 +1,69 @@
+package api
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/andydixon/noema/pkg/events"
+)
+
+func TestObservationAndSSEStream(t *testing.T) {
+	srv, tok := testServer(t)
+	do := func(method, path, body string) (*http.Response, error) {
+		req, _ := http.NewRequest(method, srv.URL+path, strings.NewReader(body))
+		req.Header.Set("Authorization", "Bearer "+tok)
+		req.Header.Set("Content-Type", "application/json")
+		return http.DefaultClient.Do(req)
+	}
+	if res, _ := do("POST", "/api/v1/minds", `{"slug":"sse"}`); res.StatusCode != 201 {
+		t.Fatalf("create mind: %d", res.StatusCode)
+	}
+	// Open the stream first.
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+	defer cancel()
+	req, _ := http.NewRequestWithContext(ctx, "GET", srv.URL+"/api/v1/minds/sse/events/stream?type=observation", nil)
+	req.Header.Set("Authorization", "Bearer "+tok)
+	res, err := http.DefaultClient.Do(req)
+	if err != nil || res.StatusCode != 200 || !strings.HasPrefix(res.Header.Get("Content-Type"), "text/event-stream") {
+		t.Fatalf("stream open failed: %v %v", err, res)
+	}
+	defer res.Body.Close()
+
+	if r, _ := do("POST", "/api/v1/minds/sse/observations", `{"text":"disk latency rising"}`); r.StatusCode != 201 {
+		t.Fatalf("observation: %d", r.StatusCode)
+	}
+	if r, _ := do("POST", "/api/v1/minds/sse/observations", `{"text":""}`); r.StatusCode != 400 {
+		t.Fatalf("empty observation should be 400, got %d", r.StatusCode)
+	}
+	sc := bufio.NewScanner(res.Body)
+	var got events.Event
+	for sc.Scan() {
+		line := sc.Text()
+		if strings.HasPrefix(line, "data: ") {
+			if err := json.Unmarshal([]byte(line[6:]), &got); err != nil {
+				t.Fatal(err)
+			}
+			break
+		}
+	}
+	if got.Type != events.Observation || got.Summary != "disk latency rising" || got.Seq == 0 {
+		t.Fatalf("unexpected stream event: %+v", got)
+	}
+	var payload events.ObservationPayload
+	json.Unmarshal(got.Payload, &payload) //nolint:errcheck
+	if payload.Trust != events.TrustOperator || payload.Method != "api" {
+		t.Fatalf("observation provenance missing: %+v", payload)
+	}
+	// Forensic context endpoint.
+	r, _ := do("GET", "/api/v1/events/"+string(got.ID)+"/context", "")
+	var c EventContext
+	json.NewDecoder(r.Body).Decode(&c) //nolint:errcheck
+	if c.Event.ID != got.ID || c.Children == nil {
+		t.Fatalf("context wrong: %+v", c)
+	}
+}

internal/api/minds.go

@@ -2,6 +2,7 @@ package api
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"strconv"
 
@@ -119,6 +120,9 @@ func (s *Server) mindPatch(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	s.deps.Audit.Log(r.Context(), "mind.update", "mind", m.ID.String(), in)
+	if in.Status != nil {
+		s.deps.Events.OperatorAction(r.Context(), m.ID, actor(r), "mind.status", "Operator "+actor(r)+" set status to "+string(*in.Status), map[string]any{"status": *in.Status})
+	}
 	writeJSON(w, http.StatusOK, out)
 }
 
@@ -195,6 +199,7 @@ func (s *Server) personalityPut(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	s.deps.Audit.Log(r.Context(), "personality.update", "mind", m.ID.String(), map[string]any{"version": v.Version, "note": in.ChangeNote, "changes": minds.Diff(m.Personality, v.Personality)})
+	s.deps.Events.OperatorAction(r.Context(), m.ID, actor(r), "personality.update", fmt.Sprintf("Operator %s updated personality to version %d: %s", actor(r), v.Version, in.ChangeNote), map[string]any{"version": v.Version, "changes": minds.Diff(m.Personality, v.Personality)})
 	writeJSON(w, http.StatusOK, v)
 }
 
@@ -245,6 +250,7 @@ func (s *Server) personalityRestore(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	s.deps.Audit.Log(r.Context(), "personality.restore", "mind", m.ID.String(), map[string]any{"restored": n, "version": v.Version})
+	s.deps.Events.OperatorAction(r.Context(), m.ID, actor(r), "personality.restore", fmt.Sprintf("Operator %s restored personality version %d as version %d", actor(r), n, v.Version), map[string]any{"restored": n, "version": v.Version})
 	writeJSON(w, http.StatusOK, v)
 }
 
@@ -290,6 +296,7 @@ func (s *Server) personalityTemplate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	s.deps.Audit.Log(r.Context(), "personality.template", "mind", m.ID.String(), map[string]any{"template_id": in.TemplateID, "version": v.Version})
+	s.deps.Events.OperatorAction(r.Context(), m.ID, actor(r), "personality.template", "Operator "+actor(r)+": "+v.ChangeNote, map[string]any{"version": v.Version})
 	writeJSON(w, http.StatusOK, v)
 }
 

internal/api/server.go

@@ -52,6 +52,7 @@ func (s *Server) routes() {
 	m.Handle("DELETE /api/v1/me/tokens/{id}", require(auth.RoleObserver, s.myTokenRevoke))
 
 	s.mindRoutes()
+	s.eventRoutes()
 
 	m.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
 		writeError(w, r, http.StatusNotFound, "not_found", "no such endpoint")