feat(experiments): cognitive forks per model with per-mind routing and comparison

A mind's configuration may override model routing per role (llm_roles);
the router honours it for that mind only and fails closed when the named
model is missing or disabled, so a comparison between providers can never
silently fall back to the default. Experiments may declare variants: each
becomes its own fork of the same snapshot with its own configuration,
personality or routing, receives identical stimuli, and is measured. The
results carry one entry per variant and a deterministic comparison: first
divergent stimulus, belief and attention overlap, mean score difference on
shared propositions, and the ranges of memory counts, goal counts, cost and
capability requests. Single-subject definitions are unchanged.

internal/experiments/experiments.go

@@ -34,6 +34,18 @@ type Definition struct {
 	Expected    []Expectation  `json:"expected,omitempty"`
 	Seed        string         `json:"seed,omitempty"`
 	Duration    int            `json:"duration_seconds,omitempty"` // wall-clock cap (default 300)
+	// Variants run the same stimuli against several forks of the snapshot,
+	// each with its own overrides (typically a different model per role).
+	// When empty the definition's own overrides form a single subject.
+	Variants []Variant `json:"variants,omitempty"`
+}
+
+// Variant is one fork's overrides in a comparative experiment.
+type Variant struct {
+	Name        string            `json:"name"`
+	Config      map[string]any    `json:"config,omitempty"`
+	Personality map[string]any    `json:"personality,omitempty"`
+	LLMRoles    map[string]string `json:"llm_roles,omitempty"` // role → model id, resolved by the router for this fork only
 }
 
 // Stimulus is one scripted observation.
@@ -83,6 +95,44 @@ type Results struct {
 	Identity       identity.Identity `json:"identity"` // the runtime that produced these results
 	Cycles         int               `json:"cycles"`
 	TruncatedAfter int               `json:"truncated_after_stimulus,omitempty"`
+	// Comparative runs: one entry per variant and a deterministic comparison.
+	Variants   []VariantResult `json:"variants,omitempty"`
+	Comparison *Comparison     `json:"comparison,omitempty"`
+}
+
+// VariantResult is what one fork produced.
+type VariantResult struct {
+	Name               string            `json:"name"`
+	Subject            ids.ID            `json:"subject_mind_id"`
+	SubjectSlug        string            `json:"subject_slug"`
+	Routing            map[string]string `json:"routing,omitempty"`
+	Steps              []StepResult      `json:"steps"`
+	EventsByType       map[string]int    `json:"events_by_type"`
+	Beliefs            []BeliefRow       `json:"beliefs"`
+	Memories           int               `json:"memories"`
+	Goals              int               `json:"goals"`
+	Expectations       []ExpectResult    `json:"expectations"`
+	LLMCalls           map[string]int    `json:"llm_calls"`
+	Tokens             int               `json:"tokens"`
+	Cost               float64           `json:"cost"`
+	CapabilityRequests int               `json:"capability_requests"`
+	Cycles             int               `json:"cycles"`
+	TruncatedAfter     int               `json:"truncated_after_stimulus,omitempty"`
+	Error              string            `json:"error,omitempty"`
+}
+
+// Comparison is the deterministic divergence report across variants. All
+// pairwise figures are means over variant pairs.
+type Comparison struct {
+	FirstDivergentStimulus int        `json:"first_divergent_stimulus"` // -1 when every variant produced the same digests
+	BeliefOverlap          float64    `json:"belief_overlap"`           // Jaccard of normalised propositions
+	MeanConfidenceDiff     float64    `json:"mean_confidence_diff"`     // over propositions held by both
+	AttentionOverlap       float64    `json:"attention_overlap"`        // Jaccard of thought_selected digests
+	MemoryCountRange       [2]int     `json:"memory_count_range"`
+	GoalCountRange         [2]int     `json:"goal_count_range"`
+	CostRange              [2]float64 `json:"cost_range"`
+	CapabilityRequestRange [2]int     `json:"capability_request_range"`
+	Notes                  []string   `json:"notes"`
 }
 
 // StepResult records one stimulus.
@@ -203,51 +253,93 @@ func (s *Service) execute(ctx context.Context, e *Experiment, by string) (Result
 	if err != nil {
 		return res, err
 	}
-	cfg, err := override(a.Mind.Config, e.Definition.Config)
-	if err != nil {
-		return res, fmt.Errorf("config overrides: %w", err)
+	variants := e.Definition.Variants
+	if len(variants) == 0 {
+		variants = []Variant{{Name: "subject", Config: e.Definition.Config, Personality: e.Definition.Personality}}
 	}
-	pers, err := override(a.Mind.Personality, e.Definition.Personality)
-	if err != nil {
-		return res, fmt.Errorf("personality overrides: %w", err)
-	}
-	slug := "exp-" + strings.ToLower(e.ID.String()[len(e.ID.String())-8:])
-	subject, err := s.Snapshots.Fork(ctx, e.SnapshotID, snapshots.ForkOptions{Slug: slug, DisplayName: "Experiment: " + e.Name, Status: minds.StatusDisabled, Config: &cfg, Personality: &pers, By: by,
-		Notes: "experiment " + e.ID.String() + ": " + e.Hypothesis})
-	if err != nil {
-		return res, fmt.Errorf("fork subject: %w", err)
-	}
-	e.MindID = &subject.ID
-	res.Subject, res.SubjectSlug = subject.ID, subject.Slug
 	if s.Routing != nil {
 		res.Routing = s.Routing(ctx)
 	}
-	res.Identity = s.Identity.Capture(ctx, subject.ID)
+	deadline := start.Add(time.Duration(e.Definition.Duration) * time.Second)
+	for i, v := range variants {
+		vr, subject, err := s.runVariant(ctx, e, a, i, v, by, deadline)
+		if err != nil {
+			if i == 0 {
+				return res, err
+			}
+			vr.Error = err.Error()
+		}
+		res.Variants = append(res.Variants, vr)
+		if i == 0 {
+			// The first variant is the experiment's subject for the existing views.
+			e.MindID = &subject.ID
+			res.Subject, res.SubjectSlug, res.Steps, res.EventsByType, res.Beliefs = vr.Subject, vr.SubjectSlug, vr.Steps, vr.EventsByType, vr.Beliefs
+			res.Memories, res.Goals, res.Expectations, res.LLMCalls, res.Cycles, res.TruncatedAfter = vr.Memories, vr.Goals, vr.Expectations, vr.LLMCalls, vr.Cycles, vr.TruncatedAfter
+			res.Identity = s.Identity.Capture(ctx, subject.ID)
+		}
+	}
+	if len(res.Variants) > 1 {
+		res.Comparison = Compare(res.Variants)
+	}
+	res.DurationMS = time.Since(start).Milliseconds()
+	return res, nil
+}
+
+// runVariant forks the snapshot with one variant's overrides, feeds every
+// stimulus and measures the outcome. Per-mind model routing lives in the
+// fork's configuration, so the router applies it without any change to
+// callers, and the identity records it.
+func (s *Service) runVariant(ctx context.Context, e *Experiment, a snapshots.Archive, index int, v Variant, by string, deadline time.Time) (VariantResult, minds.Mind, error) {
+	vr := VariantResult{Name: firstNonEmpty(v.Name, fmt.Sprintf("variant-%d", index+1)), EventsByType: map[string]int{}, LLMCalls: map[string]int{}}
+	cfg, err := override(a.Mind.Config, v.Config)
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("%s: config overrides: %w", vr.Name, err)
+	}
+	if len(v.LLMRoles) > 0 {
+		cfg.LLMRoles = v.LLMRoles
+	}
+	pers, err := override(a.Mind.Personality, v.Personality)
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("%s: personality overrides: %w", vr.Name, err)
+	}
+	slug := "exp-" + strings.ToLower(e.ID.String()[len(e.ID.String())-8:])
+	if index > 0 {
+		slug += fmt.Sprintf("-%d", index+1)
+	}
+	subject, err := s.Snapshots.Fork(ctx, e.SnapshotID, snapshots.ForkOptions{Slug: slug, DisplayName: "Experiment: " + e.Name + " / " + vr.Name, Status: minds.StatusDisabled, Config: &cfg, Personality: &pers, By: by,
+		Notes: "experiment " + e.ID.String() + " variant " + vr.Name + ": " + e.Hypothesis})
+	if err != nil {
+		return vr, minds.Mind{}, fmt.Errorf("fork %s: %w", vr.Name, err)
+	}
+	vr.Subject, vr.SubjectSlug = subject.ID, subject.Slug
+	vr.Routing = s.Identity.Capture(ctx, subject.ID).Routing
 	drv, err := replay.NewDriver(ctx, s.Brain, subject, "experiment")
 	if err != nil {
-		return res, err
+		return vr, subject, err
 	}
-	deadline := start.Add(time.Duration(e.Definition.Duration) * time.Second)
 	var all []events.Event
 	for i, st := range e.Definition.Stimuli {
 		if time.Now().After(deadline) {
-			res.TruncatedAfter = i
+			vr.TruncatedAfter = i
 			break
 		}
-		ev, err := drv.Observe(ctx, st.Source, st.Text, st.Salience, map[string]any{"experiment_id": e.ID, "stimulus": i, "seed": e.Definition.Seed})
+		ev, err := drv.Observe(ctx, st.Source, st.Text, st.Salience, map[string]any{"experiment_id": e.ID, "variant": vr.Name, "stimulus": i, "seed": e.Definition.Seed})
 		if err != nil {
-			return res, err
+			return vr, subject, err
 		}
 		produced, cycles, err := drv.Settle(ctx)
-		res.Cycles += cycles
+		vr.Cycles += cycles
 		if err != nil {
-			s.Log.Warn("experiment cycle error", "experiment", e.ID, "err", err)
+			s.Log.Warn("experiment cycle error", "experiment", e.ID, "variant", vr.Name, "err", err)
 		}
 		all = append(all, produced...)
-		res.Steps = append(res.Steps, StepResult{Index: i, Text: st.Text, EventID: ev.ID, Produced: replay.Digest(produced), Cycles: cycles})
+		vr.Steps = append(vr.Steps, StepResult{Index: i, Text: st.Text, EventID: ev.ID, Produced: replay.Digest(produced), Cycles: cycles})
 	}
 	for _, ev := range all {
-		res.EventsByType[string(ev.Type)]++
+		vr.EventsByType[string(ev.Type)]++
+		if ev.Type == events.ToolRequested || ev.Type == events.ToolExecuted || ev.Type == events.ToolDenied {
+			vr.CapabilityRequests++
+		}
 	}
 	for _, x := range e.Definition.Expected {
 		r := ExpectResult{Expectation: x}
@@ -260,25 +352,167 @@ func (s *Service) execute(ctx context.Context, e *Experiment, by string) (Result
 		if x.Absent {
 			r.Met = !r.Met
 		}
-		res.Expectations = append(res.Expectations, r)
+		vr.Expectations = append(vr.Expectations, r)
 	}
 	rows, err := s.DB.Query(ctx, `SELECT proposition, confidence, status, source FROM beliefs WHERE mind_id=$1 AND status NOT IN ('superseded') ORDER BY confidence DESC LIMIT 200`, subject.ID)
 	if err == nil {
 		for rows.Next() {
 			var b BeliefRow
 			if rows.Scan(&b.Proposition, &b.Confidence, &b.Status, &b.Source) == nil {
-				res.Beliefs = append(res.Beliefs, b)
+				vr.Beliefs = append(vr.Beliefs, b)
 			}
 		}
 		rows.Close()
 	}
-	s.DB.QueryRow(ctx, `SELECT count(*) FROM memories WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&res.Memories) //nolint:errcheck
-	s.DB.QueryRow(ctx, `SELECT count(*) FROM goals WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&res.Goals)       //nolint:errcheck
-	var live, replayed int
-	s.DB.QueryRow(ctx, `SELECT count(*) FILTER (WHERE NOT replayed), count(*) FILTER (WHERE replayed) FROM llm_calls WHERE mind_id=$1`, subject.ID).Scan(&live, &replayed) //nolint:errcheck
-	res.LLMCalls["live"], res.LLMCalls["replayed"] = live, replayed
-	res.DurationMS = time.Since(start).Milliseconds()
-	return res, nil
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM memories WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&vr.Memories) //nolint:errcheck
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM goals WHERE mind_id=$1 AND status='active'`, subject.ID).Scan(&vr.Goals)       //nolint:errcheck
+	var live, replayed, tokens int
+	var cost float64
+	s.DB.QueryRow(ctx, `SELECT count(*) FILTER (WHERE NOT replayed), count(*) FILTER (WHERE replayed), coalesce(sum(prompt_tokens+output_tokens),0), coalesce(sum(cost),0) FROM llm_calls WHERE mind_id=$1`, subject.ID).Scan(&live, &replayed, &tokens, &cost) //nolint:errcheck
+	vr.LLMCalls["live"], vr.LLMCalls["replayed"], vr.Tokens, vr.Cost = live, replayed, tokens, cost
+	return vr, subject, nil
+}
+
+// Compare measures how variants diverged. Deterministic and symmetric.
+func Compare(vs []VariantResult) *Comparison {
+	c := &Comparison{FirstDivergentStimulus: -1}
+	if len(vs) < 2 {
+		return c
+	}
+	maxSteps := 0
+	for _, v := range vs {
+		if len(v.Steps) > maxSteps {
+			maxSteps = len(v.Steps)
+		}
+	}
+	for i := 0; i < maxSteps && c.FirstDivergentStimulus < 0; i++ {
+		var ref []string
+		for k, v := range vs {
+			var d []string
+			if i < len(v.Steps) {
+				d = v.Steps[i].Produced
+			}
+			if k == 0 {
+				ref = d
+				continue
+			}
+			if !replay.Same(ref, d) {
+				c.FirstDivergentStimulus = i
+				break
+			}
+		}
+	}
+	props := func(v VariantResult) map[string]float64 {
+		out := map[string]float64{}
+		for _, b := range v.Beliefs {
+			out[strings.Join(strings.Fields(strings.ToLower(b.Proposition)), " ")] = b.Confidence
+		}
+		return out
+	}
+	attention := func(v VariantResult) map[string]bool {
+		out := map[string]bool{}
+		for _, st := range v.Steps {
+			for _, d := range st.Produced {
+				if strings.HasPrefix(d, "thought_selected:") {
+					out[d] = true
+				}
+			}
+		}
+		return out
+	}
+	var pairs, overlapSum, attSum, confSum float64
+	var confN int
+	for i := 0; i < len(vs); i++ {
+		for j := i + 1; j < len(vs); j++ {
+			pairs++
+			pi, pj := props(vs[i]), props(vs[j])
+			overlapSum += jaccardKeys(pi, pj)
+			for k, ci := range pi {
+				if cj, ok := pj[k]; ok {
+					confSum += abs(ci - cj)
+					confN++
+				}
+			}
+			attSum += jaccardBool(attention(vs[i]), attention(vs[j]))
+		}
+	}
+	c.BeliefOverlap = round3(overlapSum / pairs)
+	c.AttentionOverlap = round3(attSum / pairs)
+	if confN > 0 {
+		c.MeanConfidenceDiff = round3(confSum / float64(confN))
+	}
+	c.MemoryCountRange, c.GoalCountRange, c.CapabilityRequestRange = [2]int{vs[0].Memories, vs[0].Memories}, [2]int{vs[0].Goals, vs[0].Goals}, [2]int{vs[0].CapabilityRequests, vs[0].CapabilityRequests}
+	c.CostRange = [2]float64{vs[0].Cost, vs[0].Cost}
+	for _, v := range vs[1:] {
+		c.MemoryCountRange = span(c.MemoryCountRange, v.Memories)
+		c.GoalCountRange = span(c.GoalCountRange, v.Goals)
+		c.CapabilityRequestRange = span(c.CapabilityRequestRange, v.CapabilityRequests)
+		if v.Cost < c.CostRange[0] {
+			c.CostRange[0] = v.Cost
+		}
+		if v.Cost > c.CostRange[1] {
+			c.CostRange[1] = v.Cost
+		}
+	}
+	if c.FirstDivergentStimulus < 0 {
+		c.Notes = append(c.Notes, "every variant produced the same event digests for every stimulus")
+	} else {
+		c.Notes = append(c.Notes, fmt.Sprintf("variants first diverged at stimulus %d", c.FirstDivergentStimulus))
+	}
+	c.Notes = append(c.Notes, "identical cognitive state and stimuli; differences arise from the overrides (configuration, personality, model routing) and, for live models, provider nondeterminism")
+	return c
+}
+
+func jaccardKeys(a, b map[string]float64) float64 {
+	if len(a) == 0 && len(b) == 0 {
+		return 1
+	}
+	inter := 0
+	for k := range a {
+		if _, ok := b[k]; ok {
+			inter++
+		}
+	}
+	return float64(inter) / float64(len(a)+len(b)-inter)
+}
+
+func jaccardBool(a, b map[string]bool) float64 {
+	if len(a) == 0 && len(b) == 0 {
+		return 1
+	}
+	inter := 0
+	for k := range a {
+		if b[k] {
+			inter++
+		}
+	}
+	return float64(inter) / float64(len(a)+len(b)-inter)
+}
+
+func span(r [2]int, v int) [2]int {
+	if v < r[0] {
+		r[0] = v
+	}
+	if v > r[1] {
+		r[1] = v
+	}
+	return r
+}
+
+func abs(f float64) float64 {
+	if f < 0 {
+		return -f
+	}
+	return f
+}
+
+func round3(f float64) float64 { return float64(int(f*1000+0.5)) / 1000 }
+
+func firstNonEmpty(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
 }
 
 // override applies JSON-name field overrides to a struct value.

internal/experiments/experiments_test.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/andydixon/noema/internal/minds"
 	"github.com/andydixon/noema/internal/replay/replaytest"
+	"github.com/andydixon/noema/pkg/ids"
 )
 
 func TestExperimentRunsInSandbox(t *testing.T) {
@@ -62,3 +63,62 @@ func TestExperimentRunsInSandbox(t *testing.T) {
 		t.Fatalf("bad override: %+v", bad)
 	}
 }
+
+// TestVariantsShareStateAndCompare: two forks of one snapshot receive the
+// same stimuli under different configurations; each gets its own subject,
+// the comparison is computed deterministically, and the first variant
+// remains the experiment's subject for the existing views.
+func TestVariantsShareStateAndCompare(t *testing.T) {
+	h := replaytest.NewHarness(t)
+	ctx := context.Background()
+	base, _ := h.Minds.Create(ctx, minds.CreateParams{Slug: "vbase", DisplayName: "Base"})
+	sn, err := h.Snaps.Create(ctx, base, "baseline", "", "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	svc := &Service{DB: h.Pool, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Identity: h.Identity, Log: slog.New(slog.DiscardHandler)}
+	def := Definition{
+		Stimuli: []Stimulus{{Text: "Deployment of the payments service finished at 09:10"}, {Text: "Latency alert on the payments service at 09:12"}},
+		Variants: []Variant{
+			{Name: "wide", Config: map[string]any{"workspace_capacity": 7}},
+			{Name: "narrow", Config: map[string]any{"workspace_capacity": 2}, LLMRoles: map[string]string{"respond": string(ids.New())}},
+		},
+	}
+	e, err := svc.Create(ctx, "capacity comparison", "a narrow workspace attends to fewer recollections", sn.ID, def, "test")
+	if err != nil {
+		t.Fatal(err)
+	}
+	e, err = svc.Run(ctx, e.ID, "test")
+	if err != nil || e.Status != "done" {
+		t.Fatalf("run: %v %+v", err, e)
+	}
+	res := ResultsOf(e)
+	if len(res.Variants) != 2 || res.Variants[0].Name != "wide" || res.Variants[1].Name != "narrow" {
+		t.Fatalf("variants: %+v", res.Variants)
+	}
+	if res.Variants[0].Subject == res.Variants[1].Subject || res.Subject != res.Variants[0].Subject {
+		t.Fatalf("subjects: %+v", res.Variants)
+	}
+	a, _ := h.Minds.Get(ctx, res.Variants[0].Subject)
+	b, _ := h.Minds.Get(ctx, res.Variants[1].Subject)
+	if a.Config.WorkspaceCapacity != 7 || b.Config.WorkspaceCapacity != 2 || len(b.Config.LLMRoles) != 1 || len(a.Config.LLMRoles) != 0 {
+		t.Fatalf("overrides not applied per fork: %+v %+v", a.Config, b.Config)
+	}
+	if res.Comparison == nil || res.Comparison.BeliefOverlap < 0 || res.Comparison.BeliefOverlap > 1 || len(res.Comparison.Notes) == 0 {
+		t.Fatalf("comparison: %+v", res.Comparison)
+	}
+	if len(res.Variants[0].Steps) != 2 || len(res.Variants[1].Steps) != 2 {
+		t.Fatalf("each variant should have seen both stimuli: %+v", res.Variants)
+	}
+	// The comparison is a pure function of the results and therefore repeatable.
+	again := Compare(res.Variants)
+	if again.FirstDivergentStimulus != res.Comparison.FirstDivergentStimulus || again.AttentionOverlap != res.Comparison.AttentionOverlap {
+		t.Fatal("comparison not deterministic")
+	}
+	// A single-subject definition still works and records no comparison.
+	single, _ := svc.Create(ctx, "single", "", sn.ID, Definition{Stimuli: def.Stimuli}, "test")
+	single, _ = svc.Run(ctx, single.ID, "test")
+	if sr := ResultsOf(single); len(sr.Variants) != 1 || sr.Comparison != nil {
+		t.Fatalf("single subject: %+v", sr)
+	}
+}

internal/llm/router/router.go

@@ -37,11 +37,16 @@ type Router struct {
 	cache    routing
 	cachedAt time.Time
 
-	DailyTokenBudget int64                          // global hard stop (0 = unlimited)
-	MindBudget       func(mind ids.ID) int64        // per-mind daily budget lookup (nil = unlimited)
-	Replay           bool                           // serve recorded responses when a matching call exists
-	Override         map[llm.Role]llm.LanguageModel // tests inject models per role
-	now              func() time.Time
+	DailyTokenBudget int64                   // global hard stop (0 = unlimited)
+	MindBudget       func(mind ids.ID) int64 // per-mind daily budget lookup (nil = unlimited)
+	Replay           bool                    // serve recorded responses when a matching call exists
+	// MindRoles returns a mind's own role → model overrides (from its cognitive
+	// configuration). An override names exactly one model and no fallback:
+	// an experiment comparing providers must fail closed rather than
+	// silently fall back to the daemon default.
+	MindRoles func(ctx context.Context, mind ids.ID) map[llm.Role]ids.ID
+	Override  map[llm.Role]llm.LanguageModel // tests inject models per role
+	now       func() time.Time
 }
 
 type routing struct {
@@ -216,7 +221,7 @@ func (r *Router) Complete(ctx context.Context, req llm.Request) (llm.Response, e
 	if err != nil {
 		return llm.Response{}, err
 	}
-	cands, err := r.Candidates(ctx, req.Role)
+	cands, err := r.candidatesFor(ctx, c, req.Role, req.MindID)
 	if err != nil {
 		return llm.Response{}, err
 	}
@@ -387,3 +392,23 @@ func (r *Router) TestProvider(ctx context.Context, providerID ids.ID, modelName
 	r.Invalidate()
 	return resp, nil
 }
+
+// candidatesFor resolves the models to try for a role, honouring a mind's
+// own override when it has one.
+func (r *Router) candidatesFor(ctx context.Context, c routing, role llm.Role, mind ids.ID) ([]Model, error) {
+	if r.MindRoles != nil && !mind.IsZero() {
+		if overrides := r.MindRoles(ctx, mind); overrides != nil {
+			if id, ok := overrides[role]; ok {
+				m, found := c.models[id]
+				if !found || !m.Enabled {
+					return nil, fmt.Errorf("%w: mind override for %s names model %s, which is missing or disabled", llm.ErrNoModel, role, id)
+				}
+				if p, ok := c.providers[m.ProviderID]; !ok || !p.Enabled {
+					return nil, fmt.Errorf("%w: mind override for %s names a disabled provider", llm.ErrNoModel, role)
+				}
+				return []Model{m}, nil
+			}
+		}
+	}
+	return r.Candidates(ctx, role)
+}

internal/llm/router/router_test.go

@@ -81,3 +81,58 @@ func TestRouterOverrideBudgetRecordingAndReplay(t *testing.T) {
 	}
 	_ = ids.Nil
 }
+
+// TestMindRoleOverridesFailClosed: a mind's own routing wins over the daemon
+// default for that mind only, and an override naming a missing or disabled
+// model errors rather than falling back, so a comparison between providers
+// cannot silently become a comparison of one provider with itself.
+func TestMindRoleOverridesFailClosed(t *testing.T) {
+	pool := storage.TestPool(t, migrations.FS)
+	ctx := context.Background()
+	st := NewStore(pool)
+	sec, _ := secrets.New("")
+	r := NewRouter(st, sec, slog.New(slog.DiscardHandler), nil, 2)
+	ms := minds.NewStore(pool)
+	a, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-a"})
+	b, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-b"})
+	p, err := st.CreateProvider(ctx, ProviderInput{Name: "local", Kind: llm.KindOllama, Endpoint: "http://127.0.0.1:1", AllowPrivate: true, Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	def, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "default-model", Kind: "chat", Enabled: true})
+	alt, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "alt-model", Kind: "chat", Enabled: true})
+	off, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "disabled-model", Kind: "chat", Enabled: false})
+	st.SetRole(ctx, RoleAssignment{Role: llm.RoleJudge, ModelID: &def.ID}) //nolint:errcheck
+	r.Invalidate()
+	r.MindRoles = func(_ context.Context, mind ids.ID) map[llm.Role]ids.ID {
+		switch mind {
+		case b.ID:
+			return map[llm.Role]ids.ID{llm.RoleJudge: alt.ID}
+		}
+		return nil
+	}
+	c, _ := r.load(ctx)
+	if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, a.ID); len(cands) != 1 || cands[0].Name != "default-model" {
+		t.Fatalf("mind without override should use the default: %+v", cands)
+	}
+	if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); len(cands) != 1 || cands[0].Name != "alt-model" {
+		t.Fatalf("mind override not honoured: %+v", cands)
+	}
+	r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID { return map[llm.Role]ids.ID{llm.RoleJudge: off.ID} }
+	if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
+		t.Fatalf("disabled override must fail closed, got %v", err)
+	}
+	r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID {
+		return map[llm.Role]ids.ID{llm.RoleJudge: ids.New()}
+	}
+	if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
+		t.Fatalf("unknown override must fail closed, got %v", err)
+	}
+	// The configuration layer drops malformed overrides rather than storing them.
+	cfg := minds.DefaultConfig()
+	cfg.LLMRoles = map[string]string{"judge": string(alt.ID), "bogus_role": string(alt.ID), "respond": "not-an-id"}
+	cfg.Normalise()
+	if len(cfg.LLMRoles) != 1 || cfg.LLMRoles["judge"] != string(alt.ID) {
+		t.Fatalf("normalise: %+v", cfg.LLMRoles)
+	}
+}

internal/minds/config.go

@@ -134,6 +134,15 @@ func (c *CognitiveConfig) Normalise() {
 	if c.DreamMaxHypotheses < 1 || c.DreamMaxHypotheses > 20 {
 		c.DreamMaxHypotheses = d.DreamMaxHypotheses
 	}
+	if len(c.LLMRoles) > 0 {
+		clean := map[string]string{}
+		for role, model := range c.LLMRoles {
+			if validRole(role) && looksLikeID(model) {
+				clean[role] = model
+			}
+		}
+		c.LLMRoles = clean
+	}
 	if c.ReflectionInterval < Duration(10*time.Minute) {
 		c.ReflectionInterval = d.ReflectionInterval
 	}
@@ -188,3 +197,31 @@ func (d *Duration) UnmarshalJSON(b []byte) error {
 
 // D returns the time.Duration.
 func (d Duration) D() time.Duration { return time.Duration(d) }
+
+// validRole accepts the model roles a mind may override.
+func validRole(r string) bool {
+	switch r {
+	case "perceive", "respond", "hypothesise", "summarise", "judge", "narrate", "embed":
+		return true
+	}
+	return false
+}
+
+// looksLikeID accepts a UUID-shaped model id; anything else is dropped.
+func looksLikeID(s string) bool {
+	if len(s) != 36 {
+		return false
+	}
+	for i, ch := range s {
+		switch {
+		case i == 8 || i == 13 || i == 18 || i == 23:
+			if ch != '-' {
+				return false
+			}
+		case (ch >= '0' && ch <= '9') || (ch >= 'a' && ch <= 'f'):
+		default:
+			return false
+		}
+	}
+	return true
+}