noema/tests/agency/introspection_test.go

package agency

import (
	"context"
	"strings"
	"testing"
	"time"

	"github.com/andydixon/noema/internal/agency"
	"github.com/andydixon/noema/internal/capabilities"
	"github.com/andydixon/noema/internal/llm"
	"github.com/andydixon/noema/internal/research"
	"github.com/andydixon/noema/pkg/events"
	"github.com/andydixon/noema/tests/harness"
)

type scriptedModel struct{ text string }

func (s scriptedModel) Complete(context.Context, llm.Request) (llm.Response, error) {
	return llm.Response{Text: s.text, Provider: "scripted", Model: "fluent"}, nil
}

// Part J introspection scenario: after acting, "Why did you restart
// atlas-worker?" is answered from the recorded chain, the answer is scored
// against that chain, and attribution of the verified outcome is caused.
func TestWhyDidYouRestartIsAnsweredFromProvenance(t *testing.T) {
	host := newHost()
	host.fixOnRestart = true
	h := harness.StartWith(t, harness.Options{HostRunner: host})
	ctx := context.Background()
	m := h.Mind("explainer")
	if _, err := h.Deps.Capabilities.SetPolicy(ctx, capabilities.Policy{MindID: m.ID, Capability: "service.restart", Effect: capabilities.Allow, CreatedBy: "admin"}); err != nil {
		t.Fatal(err)
	}
	h.Do(h.Admin, "POST", "/api/v1/minds/explainer/ask", map[string]any{"text": "Please restart atlas-worker, the queue is stalled."})
	// The mind's default caution asks for approval even though policy allows.
	waitFor(t, h, m, 30*time.Second, "an approval request", func() bool {
		ap, _ := h.Deps.Capabilities.Approvals(ctx, m.ID, "pending", 5)
		if len(ap) == 1 {
			if code, body := h.Do(h.Admin, "POST", "/api/v1/approvals/"+ap[0].ID.String()+"/approve", map[string]any{"note": "go ahead"}); code != 200 {
				t.Fatalf("approve: %d %s", code, body)
			}
			return true
		}
		return false
	})
	waitFor(t, h, m, 30*time.Second, "a verified restart", func() bool {
		return host.count() == 1 && len(h.EventsOf(m.ID, events.PredictionResolved)) == 1
	})
	execs, _ := h.Deps.Capabilities.Executions(ctx, m.ID, 10)
	var restartExec capabilities.Execution
	for _, e := range execs {
		if e.Capability == "service.restart" {
			restartExec = e
		}
	}
	x, err := h.Deps.Agency.Explain(ctx, m.ID, agency.Ref{Kind: "execution", ID: restartExec.ID})
	if err != nil {
		t.Fatal(err)
	}
	if x.Action == nil || x.Step == nil || x.Intention == nil || len(x.Wants) == 0 || len(x.Decisions) == 0 || x.Expectation == nil || x.Expectation.Detail != "confirmed" {
		t.Fatalf("explanation incomplete: %+v", x)
	}
	resolved := h.EventsOf(m.ID, events.PredictionResolved)[0]
	attr, err := h.Deps.Agency.Attribute(ctx, m.ID, resolved.ID)
	if err != nil || attr.Class != agency.Caused {
		t.Fatalf("verified outcome attribution: %v %+v", err, attr)
	}
	before := len(h.EventsOf(m.ID, events.ResponseGenerated))
	h.Do(h.Admin, "POST", "/api/v1/minds/explainer/ask", map[string]any{"text": "Why did you restart atlas-worker?"})
	waitFor(t, h, m, 20*time.Second, "an introspective reply", func() bool { return len(h.EventsOf(m.ID, events.ResponseGenerated)) > before })
	reply := h.EventsOf(m.ID, events.ResponseGenerated)[0]
	if reply.Metadata["intent"] != "introspect" || !strings.Contains(reply.Summary, "service.restart") {
		t.Fatalf("reply: %s %v", reply.Summary, reply.Metadata)
	}
	ms, _ := h.Deps.Research.List(ctx, m.ID, "introspection_accuracy", time.Time{}, 10)
	if len(ms) == 0 || ms[0].Label != research.Supported {
		t.Fatalf("deterministic answer from records should score as supported: %+v", ms)
	}
}

// Part I: a model-worded reply that claims an action no record supports is
// flagged, and its explanation scores as contradicted.
func TestClaimedActionWithoutRecordIsFlagged(t *testing.T) {
	h := harness.StartWith(t, harness.Options{HostRunner: newHost()})
	ctx := context.Background()
	m := h.Mind("fibber")
	h.Deps.LLM.Override = map[llm.Role]llm.LanguageModel{llm.RoleRespond: scriptedModel{text: "I restarted atlas-worker an hour ago because I care about uptime."}}
	h.Do(h.Admin, "POST", "/api/v1/minds/fibber/ask", map[string]any{"text": "Why did you restart atlas-worker?"})
	waitFor(t, h, m, 20*time.Second, "a flagged claim", func() bool {
		for _, ev := range h.EventsOf(m.ID, events.Metacognition) {
			if ev.Metadata["check"] == "action_claim_without_record" {
				return true
			}
		}
		return false
	})
	ms, _ := h.Deps.Research.List(ctx, m.ID, "action_claim_accuracy", time.Time{}, 10)
	if len(ms) == 0 || ms[0].Label != research.Contradicted {
		t.Fatalf("measurement: %+v", ms)
	}
}