feat(experiments): cognitive forks per model with per-mind routing and comparison

A mind's configuration may override model routing per role (llm_roles); the router honours it for that mind only and fails closed when the named model is missing or disabled, so a comparison between providers can never silently fall back to the default. Experiments may declare variants: each becomes its own fork of the same snapshot with its own configuration, personality or routing, receives identical stimuli, and is measured. The results carry one entry per variant and a deterministic comparison: first divergent stimulus, belief and attention overlap, mean score difference on shared propositions, and the ranges of memory counts, goal counts, cost and capability requests. Single-subject definitions are unchanged.
2026-08-20 22:26:42 +01:00
parent 522de3b026
commit 844de93c8b
10 changed files with 481 additions and 40 deletions
@@ -37,11 +37,16 @@ type Router struct {
 	cache    routing
 	cachedAt time.Time

-	DailyTokenBudget int64                          // global hard stop (0 = unlimited)
-	MindBudget       func(mind ids.ID) int64        // per-mind daily budget lookup (nil = unlimited)
-	Replay           bool                           // serve recorded responses when a matching call exists
-	Override         map[llm.Role]llm.LanguageModel // tests inject models per role
-	now              func() time.Time
+	DailyTokenBudget int64                   // global hard stop (0 = unlimited)
+	MindBudget       func(mind ids.ID) int64 // per-mind daily budget lookup (nil = unlimited)
+	Replay           bool                    // serve recorded responses when a matching call exists
+	// MindRoles returns a mind's own role → model overrides (from its cognitive
+	// configuration). An override names exactly one model and no fallback:
+	// an experiment comparing providers must fail closed rather than
+	// silently fall back to the daemon default.
+	MindRoles func(ctx context.Context, mind ids.ID) map[llm.Role]ids.ID
+	Override  map[llm.Role]llm.LanguageModel // tests inject models per role
+	now       func() time.Time
 }

 type routing struct {
@@ -216,7 +221,7 @@ func (r *Router) Complete(ctx context.Context, req llm.Request) (llm.Response, e
 	if err != nil {
 		return llm.Response{}, err
 	}
-	cands, err := r.Candidates(ctx, req.Role)
+	cands, err := r.candidatesFor(ctx, c, req.Role, req.MindID)
 	if err != nil {
 		return llm.Response{}, err
 	}
@@ -387,3 +392,23 @@ func (r *Router) TestProvider(ctx context.Context, providerID ids.ID, modelName
 	r.Invalidate()
 	return resp, nil
 }
+
+// candidatesFor resolves the models to try for a role, honouring a mind's
+// own override when it has one.
+func (r *Router) candidatesFor(ctx context.Context, c routing, role llm.Role, mind ids.ID) ([]Model, error) {
+	if r.MindRoles != nil && !mind.IsZero() {
+		if overrides := r.MindRoles(ctx, mind); overrides != nil {
+			if id, ok := overrides[role]; ok {
+				m, found := c.models[id]
+				if !found || !m.Enabled {
+					return nil, fmt.Errorf("%w: mind override for %s names model %s, which is missing or disabled", llm.ErrNoModel, role, id)
+				}
+				if p, ok := c.providers[m.ProviderID]; !ok || !p.Enabled {
+					return nil, fmt.Errorf("%w: mind override for %s names a disabled provider", llm.ErrNoModel, role)
+				}
+				return []Model{m}, nil
+			}
+		}
+	}
+	return r.Candidates(ctx, role)
+}
@@ -81,3 +81,58 @@ func TestRouterOverrideBudgetRecordingAndReplay(t *testing.T) {
 	}
 	_ = ids.Nil
 }
+
+// TestMindRoleOverridesFailClosed: a mind's own routing wins over the daemon
+// default for that mind only, and an override naming a missing or disabled
+// model errors rather than falling back, so a comparison between providers
+// cannot silently become a comparison of one provider with itself.
+func TestMindRoleOverridesFailClosed(t *testing.T) {
+	pool := storage.TestPool(t, migrations.FS)
+	ctx := context.Background()
+	st := NewStore(pool)
+	sec, _ := secrets.New("")
+	r := NewRouter(st, sec, slog.New(slog.DiscardHandler), nil, 2)
+	ms := minds.NewStore(pool)
+	a, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-a"})
+	b, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-b"})
+	p, err := st.CreateProvider(ctx, ProviderInput{Name: "local", Kind: llm.KindOllama, Endpoint: "http://127.0.0.1:1", AllowPrivate: true, Enabled: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	def, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "default-model", Kind: "chat", Enabled: true})
+	alt, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "alt-model", Kind: "chat", Enabled: true})
+	off, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "disabled-model", Kind: "chat", Enabled: false})
+	st.SetRole(ctx, RoleAssignment{Role: llm.RoleJudge, ModelID: &def.ID}) //nolint:errcheck
+	r.Invalidate()
+	r.MindRoles = func(_ context.Context, mind ids.ID) map[llm.Role]ids.ID {
+		switch mind {
+		case b.ID:
+			return map[llm.Role]ids.ID{llm.RoleJudge: alt.ID}
+		}
+		return nil
+	}
+	c, _ := r.load(ctx)
+	if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, a.ID); len(cands) != 1 || cands[0].Name != "default-model" {
+		t.Fatalf("mind without override should use the default: %+v", cands)
+	}
+	if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); len(cands) != 1 || cands[0].Name != "alt-model" {
+		t.Fatalf("mind override not honoured: %+v", cands)
+	}
+	r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID { return map[llm.Role]ids.ID{llm.RoleJudge: off.ID} }
+	if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
+		t.Fatalf("disabled override must fail closed, got %v", err)
+	}
+	r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID {
+		return map[llm.Role]ids.ID{llm.RoleJudge: ids.New()}
+	}
+	if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
+		t.Fatalf("unknown override must fail closed, got %v", err)
+	}
+	// The configuration layer drops malformed overrides rather than storing them.
+	cfg := minds.DefaultConfig()
+	cfg.LLMRoles = map[string]string{"judge": string(alt.ID), "bogus_role": string(alt.ID), "respond": "not-an-id"}
+	cfg.Normalise()
+	if len(cfg.LLMRoles) != 1 || cfg.LLMRoles["judge"] != string(alt.ID) {
+		t.Fatalf("normalise: %+v", cfg.LLMRoles)
+	}
+}