feat(experiments): cognitive forks per model with per-mind routing and comparison
A mind's configuration may override model routing per role (llm_roles); the router honours it for that mind only and fails closed when the named model is missing or disabled, so a comparison between providers can never silently fall back to the default. Experiments may declare variants: each becomes its own fork of the same snapshot with its own configuration, personality or routing, receives identical stimuli, and is measured. The results carry one entry per variant and a deterministic comparison: first divergent stimulus, belief and attention overlap, mean score difference on shared propositions, and the ranges of memory counts, goal counts, cost and capability requests. Single-subject definitions are unchanged.
This commit is contained in:
@@ -81,3 +81,58 @@ func TestRouterOverrideBudgetRecordingAndReplay(t *testing.T) {
|
||||
}
|
||||
_ = ids.Nil
|
||||
}
|
||||
|
||||
// TestMindRoleOverridesFailClosed: a mind's own routing wins over the daemon
|
||||
// default for that mind only, and an override naming a missing or disabled
|
||||
// model errors rather than falling back, so a comparison between providers
|
||||
// cannot silently become a comparison of one provider with itself.
|
||||
func TestMindRoleOverridesFailClosed(t *testing.T) {
|
||||
pool := storage.TestPool(t, migrations.FS)
|
||||
ctx := context.Background()
|
||||
st := NewStore(pool)
|
||||
sec, _ := secrets.New("")
|
||||
r := NewRouter(st, sec, slog.New(slog.DiscardHandler), nil, 2)
|
||||
ms := minds.NewStore(pool)
|
||||
a, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-a"})
|
||||
b, _ := ms.Create(ctx, minds.CreateParams{Slug: "fork-b"})
|
||||
p, err := st.CreateProvider(ctx, ProviderInput{Name: "local", Kind: llm.KindOllama, Endpoint: "http://127.0.0.1:1", AllowPrivate: true, Enabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
def, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "default-model", Kind: "chat", Enabled: true})
|
||||
alt, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "alt-model", Kind: "chat", Enabled: true})
|
||||
off, _ := st.CreateModel(ctx, ModelInput{ProviderID: p.ID, Name: "disabled-model", Kind: "chat", Enabled: false})
|
||||
st.SetRole(ctx, RoleAssignment{Role: llm.RoleJudge, ModelID: &def.ID}) //nolint:errcheck
|
||||
r.Invalidate()
|
||||
r.MindRoles = func(_ context.Context, mind ids.ID) map[llm.Role]ids.ID {
|
||||
switch mind {
|
||||
case b.ID:
|
||||
return map[llm.Role]ids.ID{llm.RoleJudge: alt.ID}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
c, _ := r.load(ctx)
|
||||
if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, a.ID); len(cands) != 1 || cands[0].Name != "default-model" {
|
||||
t.Fatalf("mind without override should use the default: %+v", cands)
|
||||
}
|
||||
if cands, _ := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); len(cands) != 1 || cands[0].Name != "alt-model" {
|
||||
t.Fatalf("mind override not honoured: %+v", cands)
|
||||
}
|
||||
r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID { return map[llm.Role]ids.ID{llm.RoleJudge: off.ID} }
|
||||
if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
|
||||
t.Fatalf("disabled override must fail closed, got %v", err)
|
||||
}
|
||||
r.MindRoles = func(context.Context, ids.ID) map[llm.Role]ids.ID {
|
||||
return map[llm.Role]ids.ID{llm.RoleJudge: ids.New()}
|
||||
}
|
||||
if _, err := r.candidatesFor(ctx, c, llm.RoleJudge, b.ID); !errors.Is(err, llm.ErrNoModel) {
|
||||
t.Fatalf("unknown override must fail closed, got %v", err)
|
||||
}
|
||||
// The configuration layer drops malformed overrides rather than storing them.
|
||||
cfg := minds.DefaultConfig()
|
||||
cfg.LLMRoles = map[string]string{"judge": string(alt.ID), "bogus_role": string(alt.ID), "respond": "not-an-id"}
|
||||
cfg.Normalise()
|
||||
if len(cfg.LLMRoles) != 1 || cfg.LLMRoles["judge"] != string(alt.ID) {
|
||||
t.Fatalf("normalise: %+v", cfg.LLMRoles)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user