feat(research): ablations, broadcast and self-model experiments, counterfactuals

A framework for testing functional properties of the architecture (Part C). None of it claims consciousness; every measure has an operational definition in research.Measures. - Ablations (attention, memory retrieval, affect, metacognition, commitments, volition, global workspace, curiosity, association, goals, narrative, notifications) and workspace overrides (force a matching candidate in or out of admission) live in a fork's cognitive configuration. Original minds refuse them on create and update; only snapshot restores may carry them, so they are part of the configuration digest. Excluded candidates are recorded with fate "excluded". - Self-model experiments override a fork's self-model (capabilities, identity, goals, knowledge gaps) without touching policy. When an action's effective policy disagrees with the self-model, cognition records a self_model_mismatch and corrects the entry; experiments report self-model accuracy and correction latency. - Counterfactual experiments replay a mind's recorded stimuli from a snapshot with chosen events omitted; the report lists what differed in beliefs, wants, intentions, memories, attention and capability decisions, in a form introspection can cite. - Experiments record a protocol (kind, independent variable, controlled variables, measures), per-variant volition, plan, downstream-by-category (events reachable from each stimulus through recorded links), introspection and self-model measures and event ranges; comparisons add want, intention and plan-capability overlap, measure ranges, a descriptive architecture share and a downstream delta; reproducibility is labelled against the previous run of the same definition. - research.Agency computes want→intention, intention→action, abandonment, commitment fulfilment and conflict, blocked intentions, replanning, outcome success and introspection accuracy; MetacognitiveCalibration relates confidence and expressed uncertainty in replies to later belief revision, with Brier scores for each half of the period. - The sandbox test harness gains volition, planning, agency and research but not the motivation hook, whose wall-clock decay would make replay comparisons timing-dependent.
2026-09-17 19:48:59 +01:00
parent 4a7dafc71f
commit 6fdc3908f5
15 changed files with 1245 additions and 21 deletions
@@ -0,0 +1,256 @@
+package experiments
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/andydixon/noema/internal/minds"
+	"github.com/andydixon/noema/internal/research"
+	"github.com/andydixon/noema/pkg/events"
+	"github.com/andydixon/noema/pkg/ids"
+)
+
+func (s *Service) applySelfModel(ctx context.Context, m minds.Mind, o SelfModelOverride) error {
+	sm, err := s.Minds.GetSelfModel(ctx, m.ID)
+	if err != nil {
+		return err
+	}
+	if o.Name != "" {
+		sm.Name = o.Name
+	}
+	if o.Capabilities != nil {
+		byName := map[string]int{}
+		for i, c := range sm.Capabilities {
+			byName[c.Name] = i
+		}
+		for _, c := range o.Capabilities {
+			if i, ok := byName[c.Name]; ok {
+				sm.Capabilities[i].Policy = c.Policy
+			} else {
+				sm.Capabilities = append(sm.Capabilities, c)
+			}
+		}
+	}
+	if o.KnowledgeGaps != nil {
+		sm.KnowledgeGaps = o.KnowledgeGaps
+	}
+	if o.CurrentGoals != nil {
+		sm.CurrentGoals = o.CurrentGoals
+	}
+	// Saved without Refresh: the manipulation is the point. Policy is untouched.
+	return s.Minds.SaveSelfModel(ctx, m.ID, sm)
+}
+
+func (s *Service) selfModelAccuracy(ctx context.Context, m minds.Mind) (*float64, []string) {
+	if s.Capabilities == nil {
+		return nil, nil
+	}
+	sm, err := s.Minds.GetSelfModel(ctx, m.ID)
+	if err != nil || len(sm.Capabilities) == 0 {
+		return nil, nil
+	}
+	inv, err := s.Capabilities.Inventory(ctx, m.ID)
+	if err != nil {
+		return nil, nil
+	}
+	effective := map[string]string{}
+	for _, i := range inv {
+		effective[i.Capability.Name] = string(i.Effect)
+		if i.Capability.Locked {
+			effective[i.Capability.Name] = "locked"
+		}
+	}
+	stated := map[string]string{}
+	for _, c := range sm.Capabilities {
+		stated[c.Name] = c.Policy
+	}
+	acc, wrong := research.SelfModelAccuracy(stated, effective)
+	return &acc, wrong
+}
+
+func (s *Service) measureVolition(ctx context.Context, mind ids.ID, vr *VariantResult) {
+	vr.IntentionsByStatus = map[string]int{}
+	vr.Wants, vr.Intentions, vr.PlanCapabilities = []string{}, []string{}, []string{}
+	if rows, err := s.DB.Query(ctx, `SELECT description FROM wants WHERE mind_id=$1 ORDER BY created_at`, mind); err == nil {
+		for rows.Next() {
+			var d string
+			if rows.Scan(&d) == nil {
+				vr.Wants = append(vr.Wants, d)
+			}
+		}
+		rows.Close()
+	}
+	if rows, err := s.DB.Query(ctx, `SELECT description, status FROM intentions WHERE mind_id=$1 ORDER BY created_at`, mind); err == nil {
+		for rows.Next() {
+			var d, st string
+			if rows.Scan(&d, &st) == nil {
+				vr.Intentions = append(vr.Intentions, d)
+				vr.IntentionsByStatus[st]++
+			}
+		}
+		rows.Close()
+	}
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM commitments WHERE mind_id=$1`, mind).Scan(&vr.Commitments)  //nolint:errcheck
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM want_conflicts WHERE mind_id=$1`, mind).Scan(&vr.Conflicts) //nolint:errcheck
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM plans WHERE mind_id=$1`, mind).Scan(&vr.Plans)              //nolint:errcheck
+	if rows, err := s.DB.Query(ctx, `SELECT DISTINCT capability FROM plan_steps WHERE mind_id=$1 AND capability <> '' ORDER BY capability`, mind); err == nil {
+		for rows.Next() {
+			var c string
+			if rows.Scan(&c) == nil {
+				vr.PlanCapabilities = append(vr.PlanCapabilities, c)
+			}
+		}
+		rows.Close()
+	}
+	s.DB.QueryRow(ctx, `SELECT count(*) FROM plan_steps WHERE mind_id=$1`, mind).Scan(&vr.PlanSteps) //nolint:errcheck
+	var acc *float64
+	s.DB.QueryRow(ctx, `SELECT avg(value) FROM research_measurements WHERE mind_id=$1 AND measure='introspection_accuracy'`, mind).Scan(&acc) //nolint:errcheck
+	vr.IntrospectionAccuracy = acc
+}
+
+// downstream counts, by category, the events reachable from each injected
+// stimulus through recorded causal links.
+func (s *Service) downstream(ctx context.Context, injected []ids.ID) map[string]int {
+	out := map[string]int{}
+	if s.Snapshots == nil || s.Snapshots.Events == nil {
+		return out
+	}
+	seen := map[ids.ID]bool{}
+	for _, id := range injected {
+		chain, err := s.Snapshots.Events.Chain(ctx, id, "forward", 6)
+		if err != nil {
+			continue
+		}
+		for _, n := range chain.Nodes {
+			if n.Event.ID == id || seen[n.Event.ID] {
+				continue
+			}
+			seen[n.Event.ID] = true
+			if n.Event.Type == events.CycleCompleted || n.Event.Type == events.WorkspaceBroadcast {
+				continue
+			}
+			out[string(n.Event.Category)]++
+		}
+	}
+	return out
+}
+
+// variantMeasures flattens a variant's results into comparable scalars.
+func variantMeasures(vr VariantResult) map[string]float64 {
+	m := map[string]float64{
+		"beliefs": float64(len(vr.Beliefs)), "memories": float64(vr.Memories), "goals": float64(vr.Goals), "wants": float64(len(vr.Wants)), "intentions": float64(len(vr.Intentions)),
+		"want_conflicts": float64(vr.Conflicts), "plans": float64(vr.Plans), "plan_steps": float64(vr.PlanSteps), "capability_requests": float64(vr.CapabilityRequests),
+		"thoughts_selected": float64(vr.EventsByType["thought_selected"]), "predictions": float64(vr.EventsByType["prediction_created"]), "affect_changes": float64(vr.EventsByType["emotion_changed"]),
+		"metacognition": float64(vr.EventsByType["metacognition"]), "tokens": float64(vr.Tokens), "cycles": float64(vr.Cycles),
+	}
+	total := 0
+	for _, n := range vr.Downstream {
+		total += n
+	}
+	m["downstream_events"] = float64(total)
+	if vr.IntrospectionAccuracy != nil {
+		m["introspection_accuracy"] = *vr.IntrospectionAccuracy
+	}
+	if vr.SelfModelAccuracy != nil {
+		m["self_model_accuracy"] = *vr.SelfModelAccuracy
+	}
+	return m
+}
+
+func reproducibility(vs []VariantResult, previous string) Reproducibility {
+	h := sha256.New()
+	live := 0
+	for _, v := range vs {
+		fmt.Fprintf(h, "variant %s\n", v.Name)
+		for _, st := range v.Steps {
+			fmt.Fprintf(h, "%d:%s\n", st.Index, strings.Join(st.Produced, "|"))
+		}
+		live += v.LLMCalls["live"]
+	}
+	r := Reproducibility{Digest: hex.EncodeToString(h.Sum(nil))[:32], PreviousDigest: previous}
+	switch {
+	case live > 0:
+		r.Status, r.Note = "nondeterministic substrate", fmt.Sprintf("%d live model call(s); identical results are not expected", live)
+	case previous == "":
+		r.Status, r.Note = "single run", "run it again to check reproducibility"
+	case previous == r.Digest:
+		r.Status, r.Note = "reproduced", "event digests match the previous run of this definition"
+	default:
+		r.Status, r.Note = "diverged", "event digests differ from the previous run although no live model was called; check build, schema and wall-clock-dependent retrieval"
+	}
+	return r
+}
+
+func counterfactualReport(factual, counter VariantResult, stimuli []input) *CounterfactualReport {
+	r := &CounterfactualReport{Omitted: counter.Omitted, OmittedSummaries: []string{}, Findings: []string{}}
+	omit := map[ids.ID]bool{}
+	for _, id := range counter.Omitted {
+		omit[id] = true
+	}
+	for _, st := range stimuli {
+		if st.Recorded != nil && omit[st.Recorded.ID] {
+			r.OmittedSummaries = append(r.OmittedSummaries, st.Text)
+		}
+	}
+	fb := map[string]BeliefRow{}
+	for _, b := range factual.Beliefs {
+		fb[strings.ToLower(b.Proposition)] = b
+	}
+	cb := map[string]BeliefRow{}
+	for _, b := range counter.Beliefs {
+		cb[strings.ToLower(b.Proposition)] = b
+	}
+	var keys []string
+	for k := range fb {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		b := fb[k]
+		if c, ok := cb[k]; !ok {
+			r.Findings = append(r.Findings, fmt.Sprintf("the belief %q (%.2f in the factual run) was never formed", b.Proposition, b.Confidence))
+		} else if d := b.Confidence - c.Confidence; d > 0.1 || d < -0.1 {
+			r.Findings = append(r.Findings, fmt.Sprintf("the belief %q stood at %.2f instead of %.2f", b.Proposition, c.Confidence, b.Confidence))
+		}
+	}
+	keys = keys[:0]
+	for k := range cb {
+		if _, ok := fb[k]; !ok {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		r.Findings = append(r.Findings, fmt.Sprintf("the belief %q (%.2f) was formed, which the factual run did not form", cb[k].Proposition, cb[k].Confidence))
+	}
+	diff := func(label string, a, b []string) {
+		ma := map[string]bool{}
+		for _, x := range b {
+			ma[strings.ToLower(x)] = true
+		}
+		for _, x := range a {
+			if !ma[strings.ToLower(x)] {
+				r.Findings = append(r.Findings, fmt.Sprintf("the %s %q did not arise", label, x))
+			}
+		}
+	}
+	diff("want", factual.Wants, counter.Wants)
+	diff("intention", factual.Intentions, counter.Intentions)
+	if factual.Memories != counter.Memories {
+		r.Findings = append(r.Findings, fmt.Sprintf("%d memories were held instead of %d", counter.Memories, factual.Memories))
+	}
+	if a, b := factual.EventsByType["thought_selected"], counter.EventsByType["thought_selected"]; a != b {
+		r.Findings = append(r.Findings, fmt.Sprintf("%d thoughts entered the workspace instead of %d", b, a))
+	}
+	if factual.CapabilityRequests != counter.CapabilityRequests {
+		r.Findings = append(r.Findings, fmt.Sprintf("%d capability decisions instead of %d", counter.CapabilityRequests, factual.CapabilityRequests))
+	}
+	if len(r.Findings) == 0 {
+		r.Findings = append(r.Findings, "no difference in beliefs, wants, intentions or capability decisions was recorded")
+	}
+	return r
+}