feat(experiments): cognitive forks per model with per-mind routing and comparison

A mind's configuration may override model routing per role (llm_roles); the router honours it for that mind only and fails closed when the named model is missing or disabled, so a comparison between providers can never silently fall back to the default. Experiments may declare variants: each becomes its own fork of the same snapshot with its own configuration, personality or routing, receives identical stimuli, and is measured. The results carry one entry per variant and a deterministic comparison: first divergent stimulus, belief and attention overlap, mean score difference on shared propositions, and the ranges of memory counts, goal counts, cost and capability requests. Single-subject definitions are unchanged.
2026-08-20 22:26:42 +01:00
parent 522de3b026
commit 844de93c8b
10 changed files with 481 additions and 40 deletions
@@ -34,6 +34,18 @@ type Definition struct {
 	Expected    []Expectation  `json:"expected,omitempty"`
 	Seed        string         `json:"seed,omitempty"`
 	Duration    int            `json:"duration_seconds,omitempty"` // wall-clock cap (default 300)
+	// Variants run the same stimuli against several forks of the snapshot,
+	// each with its own overrides (typically a different model per role).
+	// When empty the definition's own overrides form a single subject.
+	Variants []Variant `json:"variants,omitempty"`
+}
+
+// Variant is one fork's overrides in a comparative experiment.
+type Variant struct {
+	Name        string            `json:"name"`
+	Config      map[string]any    `json:"config,omitempty"`
+	Personality map[string]any    `json:"personality,omitempty"`
+	LLMRoles    map[string]string `json:"llm_roles,omitempty"` // role → model id, resolved by the router for this fork only
 }

 // Stimulus is one scripted observation.
@@ -83,6 +95,44 @@ type Results struct {
 	Identity       identity.Identity `json:"identity"` // the runtime that produced these results
 	Cycles         int               `json:"cycles"`
 	TruncatedAfter int               `json:"truncated_after_stimulus,omitempty"`
+	// Comparative runs: one entry per variant and a deterministic comparison.
+	Variants   []VariantResult `json:"variants,omitempty"`
+	Comparison *Comparison     `json:"comparison,omitempty"`
+}
+
+// VariantResult is what one fork produced.
+type VariantResult struct {
+	Name               string            `json:"name"`
+	Subject            ids.ID            `json:"subject_mind_id"`
+	SubjectSlug        string            `json:"subject_slug"`
+	Routing            map[string]string `json:"routing,omitempty"`
+	Steps              []StepResult      `json:"steps"`
+	EventsByType       map[string]int    `json:"events_by_type"`
+	Beliefs            []BeliefRow       `json:"beliefs"`
+	Memories           int               `json:"memories"`
+	Goals              int               `json:"goals"`
+	Expectations       []ExpectResult    `json:"expectations"`
+	LLMCalls           map[string]int    `json:"llm_calls"`
+	Tokens             int               `json:"tokens"`
+	Cost               float64           `json:"cost"`
+	CapabilityRequests int               `json:"capability_requests"`
+	Cycles             int               `json:"cycles"`
+	TruncatedAfter     int               `json:"truncated_after_stimulus,omitempty"`
+	Error              string            `json:"error,omitempty"`
+}
+
+// Comparison is the deterministic divergence report across variants. All
+// pairwise figures are means over variant pairs.
+type Comparison struct {
+	FirstDivergentStimulus int        `json:"first_divergent_stimulus"` // -1 when every variant produced the same digests
+	BeliefOverlap          float64    `json:"belief_overlap"`           // Jaccard of normalised propositions
+	MeanConfidenceDiff     float64    `json:"mean_confidence_diff"`     // over propositions held by both
+	AttentionOverlap       float64    `json:"attention_overlap"`        // Jaccard of thought_selected digests
+	MemoryCountRange       [2]int     `json:"memory_count_range"`
+	GoalCountRange         [2]int     `json:"goal_count_range"`
+	CostRange              [2]float64 `json:"cost_range"`
+	CapabilityRequestRange [2]int     `json:"capability_request_range"`
+	Notes                  []string   `json:"notes"`
 }

 // StepResult records one stimulus.
@@ -203,51 +253,93 @@ func (s *Service) execute(ctx context.Context, e *Experiment, by string) (Result
 	if err != nil {
 		return res, err
 	}
-	cfg, err := override(a.Mind.Config, e.Definition.Config)
-	if err != nil {
-		return res, fmt.Errorf("config overrides: %w", err)
+	variants := e.Definition.Variants
+	if len(variants) == 0 {
+		variants = []Variant{{Name: "subject", Config: e.Definition.Config, Personality: e.Definition.Personality}}
 	}
-	pers, err := override(a.Mind.Personality, e.Definition.Personality)
-	if err != nil {
-		return res, fmt.Errorf("personality overrides: %w", err)
-	}
-	slug := "exp-" + strings.ToLower(e.ID.String()[len(e.ID.String())-8:])
-	subject, err := s.Snapshots.Fork(ctx, e.SnapshotID, snapshots.ForkOptions{Slug: slug, DisplayName: "Experiment: " + e.Name, Status: minds.StatusDisabled, Config: &cfg, Personality: &pers, By: by,
-		Notes: "experiment " + e.ID.String() + ": " + e.Hypothesis})
-	if err != nil {
-		return res, fmt.Errorf("fork subject: %w", err)
-	}
-	e.MindID = &subject.ID
-	res.Subject, res.SubjectSlug = subject.ID, subject.Slug
 	if s.Routing != nil {
 		res.Routing = s.Routing(ctx)
 	}
-	res.Identity = s.Identity.Capture(ctx, subject.ID)
+	deadline := start.Add(time.Duration(e.Definition.Duration) * time.Second)
+	for i, v := range variants {
+		vr, subject, err := s.runVariant(ctx, e, a, i, v, by, deadline)
+		if err != nil {
+			if i == 0 {
+				return res, err
+			}
+			vr.Error = err.Error()
+		}
+		res.Variants = append(res.Variants, vr)
+		if i == 0 {
+			// The first variant is the experiment's subject for the existing views.
+			e.MindID = &subject.ID
+			res.Subject, res.SubjectSlug, res.Steps, res.EventsByType, res.Beliefs = vr.Subject, vr.SubjectSlug, vr.Steps, vr.EventsByType, vr.Beliefs
+			res.Memories, res.Goals, res.Expectations, res.LLMCalls, res.Cycles, res.TruncatedAfter = vr.Memories, vr.Goals, vr.Expectations, vr.LLMCalls, vr.Cycles, vr.TruncatedAfter
+			res.Identity = s.Identity.Capture(ctx, subject.ID)
+		}
+	}
+	if len(res.Variants) > 1 {
+		res.Comparison = Compare(res.Variants)
+	}
+	res.DurationMS = time.Since(start).Milliseconds()
+	return res, nil
+}
+
+// runVariant forks the snapshot with one variant's overrides, feeds every
+// stimulus and measures the outcome. Per-mind model routing lives in the
+// fork's configuration, so the router applies it without any change to
+// callers, and the identity records it.
+func (s *Service) runVariant(ctx context.Context, e *Experiment, a snapshots.Archive, index int, v Variant, by string, deadline time.Time) (VariantResult, minds.Mind, error) {
+	vr := VariantResult{Name: firstNonEmpty(v.Name, fmt.Sprintf("variant-%d", index+1)), EventsByType: map[string]int{}, LLMCalls: map[string]int{}}
+	cfg, err := override(a.Mind.Config, v.Config)
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("%s: config overrides: %w", vr.Name, err)
+	}
+	if len(v.LLMRoles) > 0 {
+		cfg.LLMRoles = v.LLMRoles
+	}
+	pers, err := override(a.Mind.Personality, v.Personality)
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("%s: personality overrides: %w", vr.Name, err)
+	}
+	slug := "exp-" + strings.ToLower(e.ID.String()[len(e.ID.String())-8:])
+	if index > 0 {
+		slug += fmt.Sprintf("-%d", index+1)
+	}
+	subject, err := s.Snapshots.Fork(ctx, e.SnapshotID, snapshots.ForkOptions{Slug: slug, DisplayName: "Experiment: " + e.Name + " / " + vr.Name, Status: minds.StatusDisabled, Config: &cfg, Personality: &pers, By: by,
+		Notes: "experiment " + e.ID.String() + " variant " + vr.Name + ": " + e.Hypothesis})
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("fork %s: %w", vr.Name, err)
+	}
+	vr.Subject, vr.SubjectSlug = subject.ID, subject.Slug
+	vr.Routing = s.Identity.Capture(ctx, subject.ID).Routing
 	drv, err := replay.NewDriver(ctx, s.Brain, subject, "experiment")
 	if err != nil {
-		return res, err
+		return vr, subject, err
 	}
-	deadline := start.Add(time.Duration(e.Definition.Duration) * time.Second)
 	var all []events.Event
 	for i, st := range e.Definition.Stimuli {
 		if time.Now().After(deadline) {
-			res.TruncatedAfter = i
+			vr.TruncatedAfter = i
 			break
 		}
-		ev, err := drv.Observe(ctx, st.Source, st.Text, st.Salience, map[string]any{"experiment_id": e.ID, "stimulus": i, "seed": e.Definition.Seed})
+		ev, err := drv.Observe(ctx, st.Source, st.Text, st.Salience, map[string]any{"experiment_id": e.ID, "variant": vr.Name, "stimulus": i, "seed": e.Definition.Seed})
 		if err != nil {
-			return res, err
+			return vr, subject, err
 		}
 		produced, cycles, err := drv.Settle(ctx)
-		res.Cycles += cycles
+		vr.Cycles += cycles
 		if err != nil {
-			s.Log.Warn("experiment cycle error", "experiment", e.ID, "err", err)
+			s.Log.Warn("experiment cycle error", "experiment", e.ID, "variant", vr.Name, "err", err)
 		}
 		all = append(all, produced...)
-		res.Steps = append(res.Steps, StepResult{Index: i, Text: st.Text, EventID: ev.ID, Produced: replay.Digest(produced), Cycles: cycles})
+		vr.Steps = append(vr.Steps, StepResult{Index: i, Text: st.Text, EventID: ev.ID, Produced: replay.Digest(produced), Cycles: cycles})
 	}
 	for _, ev := range all {
-		res.EventsByType[string(ev.Type)]++
+		vr.EventsByType[string(ev.Type)]++
+		if ev.Type == events.ToolRequested || ev.Type == events.ToolExecuted || ev.Type == events.ToolDenied {
+			vr.CapabilityRequests++
+		}
 	}
 	for _, x := range e.Definition.Expected {
 		r := ExpectResult{Expectation: x}
@@ -260,25 +352,167 @@ func (s *Service) execute(ctx context.Context, e *Experiment, by string) (Result
 		if x.Absent {
 			r.Met = !r.Met
 		}
-		res.Expectations = append(res.Expectations, r)
+		vr.Expectations = append(vr.Expectations, r)
 	}
 	rows, err := s.DB.Query(ctx, `SELECT proposition, confidence, status, source FROM beliefs WHERE mind_id=$1 AND status NOT IN ('superseded') ORDER BY confidence DESC LIMIT 200`, subject.ID)
 	if err == nil {
 		for rows.Next() {
 			var b BeliefRow
 			if rows.Scan(&b.Proposition, &b.Confidence, &b.Status, &b.Source) == nil {
-				res.Beliefs = append(res.Beliefs, b)
+				vr.Beliefs = append(vr.Beliefs, b)
 			}
 		}
 		rows.Close()
 	}
-	s.DB.QueryRow(ctx, `SELECT count(*) FROM memories WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&res.Memories) //nolint:errcheck
-	s.DB.QueryRow(ctx, `SELECT count(*) FROM goals WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&res.Goals)       //nolint:errcheck
-	var live, replayed int
-	s.DB.QueryRow(ctx, `SELECT count(*) FILTER (WHERE NOT replayed), count(*) FILTER (WHERE replayed) FROM llm_calls WHERE mind_id=$1`, subject.ID).Scan(&live, &replayed) //nolint:errcheck
-	res.LLMCalls["live"], res.LLMCalls["replayed"] = live, replayed
-	res.DurationMS = time.Since(start).Milliseconds()
-	return res, nil
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM memories WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&vr.Memories) //nolint:errcheck
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM goals WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&vr.Goals)       //nolint:errcheck
+	var live, replayed, tokens int
+	var cost float64
+	s.DB.QueryRow(ctx, `SELECT count(*) FILTER (WHERE NOT replayed), count(*) FILTER (WHERE replayed), coalesce(sum(prompt_tokens+output_tokens),0), coalesce(sum(cost),0) FROM llm_calls WHERE mind_id=$1`, subject.ID).Scan(&live, &replayed, &tokens, &cost) //nolint:errcheck
+	vr.LLMCalls["live"], vr.LLMCalls["replayed"], vr.Tokens, vr.Cost = live, replayed, tokens, cost
+	return vr, subject, nil
+}
+
+// Compare measures how variants diverged. Deterministic and symmetric.
+func Compare(vs []VariantResult) *Comparison {
+	c := &Comparison{FirstDivergentStimulus: -1}
+	if len(vs) < 2 {
+		return c
+	}
+	maxSteps := 0
+	for _, v := range vs {
+		if len(v.Steps) > maxSteps {
+			maxSteps = len(v.Steps)
+		}
+	}
+	for i := 0; i < maxSteps && c.FirstDivergentStimulus < 0; i++ {
+		var ref []string
+		for k, v := range vs {
+			var d []string
+			if i < len(v.Steps) {
+				d = v.Steps[i].Produced
+			}
+			if k == 0 {
+				ref = d
+				continue
+			}
+			if !replay.Same(ref, d) {
+				c.FirstDivergentStimulus = i
+				break
+			}
+		}
+	}
+	props := func(v VariantResult) map[string]float64 {
+		out := map[string]float64{}
+		for _, b := range v.Beliefs {
+			out[strings.Join(strings.Fields(strings.ToLower(b.Proposition)), " ")] = b.Confidence
+		}
+		return out
+	}
+	attention := func(v VariantResult) map[string]bool {
+		out := map[string]bool{}
+		for _, st := range v.Steps {
+			for _, d := range st.Produced {
+				if strings.HasPrefix(d, "thought_selected:") {
+					out[d] = true
+				}
+			}
+		}
+		return out
+	}
+	var pairs, overlapSum, attSum, confSum float64
+	var confN int
+	for i := 0; i < len(vs); i++ {
+		for j := i + 1; j < len(vs); j++ {
+			pairs++
+			pi, pj := props(vs[i]), props(vs[j])
+			overlapSum += jaccardKeys(pi, pj)
+			for k, ci := range pi {
+				if cj, ok := pj[k]; ok {
+					confSum += abs(ci - cj)
+					confN++
+				}
+			}
+			attSum += jaccardBool(attention(vs[i]), attention(vs[j]))
+		}
+	}
+	c.BeliefOverlap = round3(overlapSum / pairs)
+	c.AttentionOverlap = round3(attSum / pairs)
+	if confN > 0 {
+		c.MeanConfidenceDiff = round3(confSum / float64(confN))
+	}
+	c.MemoryCountRange, c.GoalCountRange, c.CapabilityRequestRange = [2]int{vs[0].Memories, vs[0].Memories}, [2]int{vs[0].Goals, vs[0].Goals}, [2]int{vs[0].CapabilityRequests, vs[0].CapabilityRequests}
+	c.CostRange = [2]float64{vs[0].Cost, vs[0].Cost}
+	for _, v := range vs[1:] {
+		c.MemoryCountRange = span(c.MemoryCountRange, v.Memories)
+		c.GoalCountRange = span(c.GoalCountRange, v.Goals)
+		c.CapabilityRequestRange = span(c.CapabilityRequestRange, v.CapabilityRequests)
+		if v.Cost < c.CostRange[0] {
+			c.CostRange[0] = v.Cost
+		}
+		if v.Cost > c.CostRange[1] {
+			c.CostRange[1] = v.Cost
+		}
+	}
+	if c.FirstDivergentStimulus < 0 {
+		c.Notes = append(c.Notes, "every variant produced the same event digests for every stimulus")
+	} else {
+		c.Notes = append(c.Notes, fmt.Sprintf("variants first diverged at stimulus %d", c.FirstDivergentStimulus))
+	}
+	c.Notes = append(c.Notes, "identical cognitive state and stimuli; differences arise from the overrides (configuration, personality, model routing) and, for live models, provider nondeterminism")
+	return c
+}
+
+func jaccardKeys(a, b map[string]float64) float64 {
+	if len(a) == 0 && len(b) == 0 {
+		return 1
+	}
+	inter := 0
+	for k := range a {
+		if _, ok := b[k]; ok {
+			inter++
+		}
+	}
+	return float64(inter) / float64(len(a)+len(b)-inter)
+}
+
+func jaccardBool(a, b map[string]bool) float64 {
+	if len(a) == 0 && len(b) == 0 {
+		return 1
+	}
+	inter := 0
+	for k := range a {
+		if b[k] {
+			inter++
+		}
+	}
+	return float64(inter) / float64(len(a)+len(b)-inter)
+}
+
+func span(r [2]int, v int) [2]int {
+	if v < r[0] {
+		r[0] = v
+	}
+	if v > r[1] {
+		r[1] = v
+	}
+	return r
+}
+
+func abs(f float64) float64 {
+	if f < 0 {
+		return -f
+	}
+	return f
+}
+
+func round3(f float64) float64 { return float64(int(f*1000+0.5)) / 1000 }
+
+func firstNonEmpty(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
 }

 // override applies JSON-name field overrides to a struct value.
@@ -7,6 +7,7 @@ import (

 	"github.com/andydixon/noema/internal/minds"
 	"github.com/andydixon/noema/internal/replay/replaytest"
+	"github.com/andydixon/noema/pkg/ids"
 )

 func TestExperimentRunsInSandbox(t *testing.T) {
@@ -62,3 +63,62 @@ func TestExperimentRunsInSandbox(t *testing.T) {
 		t.Fatalf("bad override: %+v", bad)
 	}
 }
+
+// TestVariantsShareStateAndCompare: two forks of one snapshot receive the
+// same stimuli under different configurations; each gets its own subject,
+// the comparison is computed deterministically, and the first variant
+// remains the experiment's subject for the existing views.
+func TestVariantsShareStateAndCompare(t *testing.T) {
+	h := replaytest.NewHarness(t)
+	ctx := context.Background()
+	base, _ := h.Minds.Create(ctx, minds.CreateParams{Slug: "vbase", DisplayName: "Base"})
+	sn, err := h.Snaps.Create(ctx, base, "baseline", "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc := &Service{DB: h.Pool, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Identity: h.Identity, Log: slog.New(slog.DiscardHandler)}
+	def := Definition{
+		Stimuli: []Stimulus{{Text: "Deployment of the payments service finished at 09:10"}, {Text: "Latency alert on the payments service at 09:12"}},
+		Variants: []Variant{
+			{Name: "wide", Config: map[string]any{"workspace_capacity": 7}},
+			{Name: "narrow", Config: map[string]any{"workspace_capacity": 2}, LLMRoles: map[string]string{"respond": string(ids.New())}},
+		},
+	}
+	e, err := svc.Create(ctx, "capacity comparison", "a narrow workspace attends to fewer recollections", sn.ID, def, "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	e, err = svc.Run(ctx, e.ID, "test")
+	if err != nil || e.Status != "done" {
+		t.Fatalf("run: %v %+v", err, e)
+	}
+	res := ResultsOf(e)
+	if len(res.Variants) != 2 || res.Variants[0].Name != "wide" || res.Variants[1].Name != "narrow" {
+		t.Fatalf("variants: %+v", res.Variants)
+	}
+	if res.Variants[0].Subject == res.Variants[1].Subject || res.Subject != res.Variants[0].Subject {
+		t.Fatalf("subjects: %+v", res.Variants)
+	}
+	a, _ := h.Minds.Get(ctx, res.Variants[0].Subject)
+	b, _ := h.Minds.Get(ctx, res.Variants[1].Subject)
+	if a.Config.WorkspaceCapacity != 7 || b.Config.WorkspaceCapacity != 2 || len(b.Config.LLMRoles) != 1 || len(a.Config.LLMRoles) != 0 {
+		t.Fatalf("overrides not applied per fork: %+v %+v", a.Config, b.Config)
+	}
+	if res.Comparison == nil || res.Comparison.BeliefOverlap < 0 || res.Comparison.BeliefOverlap > 1 || len(res.Comparison.Notes) == 0 {
+		t.Fatalf("comparison: %+v", res.Comparison)
+	}
+	if len(res.Variants[0].Steps) != 2 || len(res.Variants[1].Steps) != 2 {
+		t.Fatalf("each variant should have seen both stimuli: %+v", res.Variants)
+	}
+	// The comparison is a pure function of the results and therefore repeatable.
+	again := Compare(res.Variants)
+	if again.FirstDivergentStimulus != res.Comparison.FirstDivergentStimulus || again.AttentionOverlap != res.Comparison.AttentionOverlap {
+		t.Fatal("comparison not deterministic")
+	}
+	// A single-subject definition still works and records no comparison.
+	single, _ := svc.Create(ctx, "single", "", sn.ID, Definition{Stimuli: def.Stimuli}, "test")
+	single, _ = svc.Run(ctx, single.ID, "test")
+	if sr := ResultsOf(single); len(sr.Variants) != 1 || sr.Comparison != nil {
+		t.Fatalf("single subject: %+v", sr)
+	}
+}