feat(experiments): cognitive forks per model with per-mind routing and comparison

A mind's configuration may override model routing per role (llm_roles);
the router honours it for that mind only and fails closed when the named
model is missing or disabled, so a comparison between providers can never
silently fall back to the default. Experiments may declare variants: each
becomes its own fork of the same snapshot with its own configuration,
personality or routing, receives identical stimuli, and is measured. The
results carry one entry per variant and a deterministic comparison: first
divergent stimulus, belief and attention overlap, mean score difference on
shared propositions, and the ranges of memory counts, goal counts, cost and
capability requests. Single-subject definitions are unchanged.
This commit is contained in:
Andy Dixon
2026-08-20 22:26:42 +01:00
parent 522de3b026
commit 844de93c8b
10 changed files with 481 additions and 40 deletions
+55
View File
@@ -81,3 +81,58 @@ func TestRouterOverrideBudgetRecordingAndReplay(t *testing.T) {
}
_ = ids.Nil
}
// TestMindRoleOverridesFailClosed: a mind's own routing wins over the daemon
// default for that mind only, and an override naming a missing or disabled
// model errors rather than falling back, so a comparison between providers
// cannot silently become a comparison of one provider with itself.
func TestMindRoleOverridesFailClosed(t *testing.T) {
pool := storage.TestPool(t, migrations.FS)
ctx := context.Background()
st := NewStore(pool)
sec, _ := secrets.New("")
r := NewRouter(st, sec, slog.New(slog.DiscardHandler), nil, 2)
ms := minds.NewStore(pool)
a, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-a"})
b, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-b"})
p, err := st.CreateProvider(ctx, ProviderInput{Name: "local", Kind: llm.KindOllama, Endpoint: "http://127.0.0.1:1", AllowPrivate: true, Enabled: true})
if err != nil {
t.Fatal(err)
}
def, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "default-model", Kind: "chat", Enabled: true})
alt, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "alt-model", Kind: "chat", Enabled: true})
off, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "disabled-model", Kind: "chat", Enabled: false})
st.SetRole(ctx, RoleAssignment{Role: llm.RoleJudge, ModelID: &def.ID}) //nolint:errcheck
r.Invalidate()
r.MindRoles = func(_ context.Context, mind ids.ID) map[llm.Role]ids.ID {
switch mind {
case b.ID:
return map[llm.Role]ids.ID{llm.RoleJudge: alt.ID}
}
return nil
}
c, _ := r.load(ctx)
if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, a.ID); len(cands) != 1 || cands[0].Name != "default-model" {
t.Fatalf("mind without override should use the default: %+v", cands)
}
if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); len(cands) != 1 || cands[0].Name != "alt-model" {
t.Fatalf("mind override not honoured: %+v", cands)
}
r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID { return map[llm.Role]ids.ID{llm.RoleJudge: off.ID} }
if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
t.Fatalf("disabled override must fail closed, got %v", err)
}
r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID {
return map[llm.Role]ids.ID{llm.RoleJudge: ids.New()}
}
if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
t.Fatalf("unknown override must fail closed, got %v", err)
}
// The configuration layer drops malformed overrides rather than storing them.
cfg := minds.DefaultConfig()
cfg.LLMRoles = map[string]string{"judge": string(alt.ID), "bogus_role": string(alt.ID), "respond": "not-an-id"}
cfg.Normalise()
if len(cfg.LLMRoles) != 1 || cfg.LLMRoles["judge"] != string(alt.ID) {
t.Fatalf("normalise: %+v", cfg.LLMRoles)
}
}