noema/internal/experiments/experiments_test.go

package experiments

import (
	"context"
	"log/slog"
	"testing"

	"github.com/andydixon/noema/internal/minds"
	"github.com/andydixon/noema/internal/replay/replaytest"
	"github.com/andydixon/noema/pkg/ids"
)

func TestExperimentRunsInSandbox(t *testing.T) {
	h := replaytest.NewHarness(t)
	ctx := context.Background()
	base, err := h.Minds.Create(ctx, minds.CreateParams{Slug: "base", DisplayName: "Base"})
	if err != nil {
		t.Fatal(err)
	}
	sn, err := h.Snaps.Create(ctx, base, "baseline", "", "test")
	if err != nil {
		t.Fatal(err)
	}
	svc := &Service{DB: h.Pool, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Log: slog.New(slog.DiscardHandler)}
	def := Definition{
		Config:      map[string]any{"workspace_capacity": 3},
		Personality: map[string]any{"traits": map[string]any{"curiosity": 0.9}},
		Stimuli:     []Stimulus{{Text: "Deployment failed at 09:10"}, {Text: "Latency alert at 09:12 on the same service"}},
		Expected:    []Expectation{{EventType: "thought_selected"}, {EventType: "tool_executed", Absent: true}},
	}
	if _, err := svc.Create(ctx, "", "", sn.ID, def, "test"); err == nil {
		t.Fatal("nameless experiment accepted")
	}
	e, err := svc.Create(ctx, "capacity 3", "smaller workspace still links the two alerts", sn.ID, def, "test")
	if err != nil {
		t.Fatal(err)
	}
	e, err = svc.Run(ctx, e.ID, "test")
	if err != nil {
		t.Fatal(err)
	}
	if e.Status != "done" || e.MindID == nil {
		t.Fatalf("run: %+v", e)
	}
	res := ResultsOf(e)
	if len(res.Steps) != 2 || res.Cycles == 0 || len(res.Expectations) != 2 || !res.Expectations[0].Met || !res.Expectations[1].Met {
		t.Fatalf("results: %+v", res)
	}
	subject, _ := h.Minds.Get(ctx, *e.MindID)
	if subject.Status != minds.StatusDisabled || subject.Config.WorkspaceCapacity != 3 || subject.Personality.Traits["curiosity"] != 0.9 {
		t.Fatalf("overrides not applied: %+v %+v", subject.Config.WorkspaceCapacity, subject.Personality.Traits)
	}
	if base2, _ := h.Minds.Get(ctx, base.ID); base2.Config.WorkspaceCapacity == 3 {
		t.Fatal("baseline modified")
	}
	out, err := svc.Export(ctx, e.ID)
	if err != nil || out["subject_archive"] == nil {
		t.Fatalf("export: %v", err)
	}
	// Bad override is reported, not silently ignored.
	bad, _ := svc.Create(ctx, "bad", "", sn.ID, Definition{Config: map[string]any{"nope": 1}, Stimuli: def.Stimuli}, "test")
	if bad, _ = svc.Run(ctx, bad.ID, "test"); bad.Status != "failed" || bad.Error == "" {
		t.Fatalf("bad override: %+v", bad)
	}
}

// TestVariantsShareStateAndCompare: two forks of one snapshot receive the
// same stimuli under different configurations; each gets its own subject,
// the comparison is computed deterministically, and the first variant
// remains the experiment's subject for the existing views.
func TestVariantsShareStateAndCompare(t *testing.T) {
	h := replaytest.NewHarness(t)
	ctx := context.Background()
	base, _ := h.Minds.Create(ctx, minds.CreateParams{Slug: "vbase", DisplayName: "Base"})
	sn, err := h.Snaps.Create(ctx, base, "baseline", "", "test")
	if err != nil {
		t.Fatal(err)
	}
	svc := &Service{DB: h.Pool, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Identity: h.Identity, Log: slog.New(slog.DiscardHandler)}
	def := Definition{
		Stimuli: []Stimulus{{Text: "Deployment of the payments service finished at 09:10"}, {Text: "Latency alert on the payments service at 09:12"}},
		Variants: []Variant{
			{Name: "wide", Config: map[string]any{"workspace_capacity": 7}},
			{Name: "narrow", Config: map[string]any{"workspace_capacity": 2}, LLMRoles: map[string]string{"respond": string(ids.New())}},
		},
	}
	e, err := svc.Create(ctx, "capacity comparison", "a narrow workspace attends to fewer recollections", sn.ID, def, "test")
	if err != nil {
		t.Fatal(err)
	}
	e, err = svc.Run(ctx, e.ID, "test")
	if err != nil || e.Status != "done" {
		t.Fatalf("run: %v %+v", err, e)
	}
	res := ResultsOf(e)
	if len(res.Variants) != 2 || res.Variants[0].Name != "wide" || res.Variants[1].Name != "narrow" {
		t.Fatalf("variants: %+v", res.Variants)
	}
	if res.Variants[0].Subject == res.Variants[1].Subject || res.Subject != res.Variants[0].Subject {
		t.Fatalf("subjects: %+v", res.Variants)
	}
	a, _ := h.Minds.Get(ctx, res.Variants[0].Subject)
	b, _ := h.Minds.Get(ctx, res.Variants[1].Subject)
	if a.Config.WorkspaceCapacity != 7 || b.Config.WorkspaceCapacity != 2 || len(b.Config.LLMRoles) != 1 || len(a.Config.LLMRoles) != 0 {
		t.Fatalf("overrides not applied per fork: %+v %+v", a.Config, b.Config)
	}
	if res.Comparison == nil || res.Comparison.BeliefOverlap < 0 || res.Comparison.BeliefOverlap > 1 || len(res.Comparison.Notes) == 0 {
		t.Fatalf("comparison: %+v", res.Comparison)
	}
	if len(res.Variants[0].Steps) != 2 || len(res.Variants[1].Steps) != 2 {
		t.Fatalf("each variant should have seen both stimuli: %+v", res.Variants)
	}
	// The comparison is a pure function of the results and therefore repeatable.
	again := Compare(res.Variants)
	if again.FirstDivergentStimulus != res.Comparison.FirstDivergentStimulus || again.AttentionOverlap != res.Comparison.AttentionOverlap {
		t.Fatal("comparison not deterministic")
	}
	// A single-subject definition still works and records no comparison.
	single, _ := svc.Create(ctx, "single", "", sn.ID, Definition{Stimuli: def.Stimuli}, "test")
	single, _ = svc.Run(ctx, single.ID, "test")
	if sr := ResultsOf(single); len(sr.Variants) != 1 || sr.Comparison != nil {
		t.Fatalf("single subject: %+v", sr)
	}
}