Andy Dixon
edcc6580b6
An intention's supporting beliefs came from a text search on the target, and ties on rank and confidence had no order, so a verification miss could revise an unrelated belief about the same service. Beliefs that mention the proposed action now come first, and belief search breaks ties by id. The incident scenario asserts the first support is the belief about restarting, and waits for the blocked intention and the procedural memory instead of reading them once.
225 lines
10 KiB
Go
225 lines
10 KiB
Go
package agency
|
||
|
||
import (
|
||
"context"
|
||
"strings"
|
||
"sync/atomic"
|
||
"testing"
|
||
"time"
|
||
|
||
"github.com/andydixon/noema/internal/agency"
|
||
"github.com/andydixon/noema/internal/beliefs"
|
||
"github.com/andydixon/noema/internal/memory"
|
||
"github.com/andydixon/noema/internal/planning"
|
||
"github.com/andydixon/noema/internal/research"
|
||
"github.com/andydixon/noema/internal/volition"
|
||
"github.com/andydixon/noema/pkg/concepts"
|
||
"github.com/andydixon/noema/pkg/events"
|
||
"github.com/andydixon/noema/tests/harness"
|
||
)
|
||
|
||
// Part L: an operator asks Noema to investigate a failing production
|
||
// system; it diagnoses with telemetry, discovers it needs logs, asks for
|
||
// temporary scoped access with the risk explained, reads the logs, is told
|
||
// a restart should help, wants to restore service, meets a commitment that
|
||
// production changes need approval, asks instead of acting, restarts with a
|
||
// typed capability once approved, verifies rather than assumes, finds the
|
||
// fault persists, records the prediction error and weakens the belief,
|
||
// abandons the branch and replans without repeating the restart, and after
|
||
// the underlying fault is fixed resolves the incident, learns the procedure
|
||
// and explains itself from the record.
|
||
//
|
||
// Where the scenario needs semantic judgement a deterministic runtime
|
||
// cannot make (that the logs implicate a stalled queue, that a restart
|
||
// should clear it), the operator supplies it as conversation; everything
|
||
// else is the runtime's own recorded cognition.
|
||
func TestIncidentScenario(t *testing.T) {
|
||
host := newHost()
|
||
host.journal = "07:12 deploy atlas-worker v42\n07:14 atlas-worker: queue consumer lost connection to db-primary (user ops@example.com from 10.0.0.7)\n07:15 atlas-worker: queue stalled\n"
|
||
var offset atomic.Int64
|
||
h := harness.StartWith(t, harness.Options{HostRunner: host, Now: func() time.Time { return time.Now().Add(time.Duration(offset.Load())) }})
|
||
ctx := context.Background()
|
||
m := h.Mind("noema")
|
||
slug := "/api/v1/minds/noema"
|
||
post := func(path string, body any) []byte {
|
||
t.Helper()
|
||
code, out := h.Do(h.Admin, "POST", path, body)
|
||
if code >= 300 {
|
||
t.Fatalf("POST %s: %d %s", path, code, out)
|
||
}
|
||
return out
|
||
}
|
||
// Standing commitment made before the incident.
|
||
post(slug+"/commitments", map[string]any{"description": "production modifications require operator approval", "kind": "operator_instruction", "polarity": "refrain", "unless": "approval", "matcher": map[string]any{"access": "write"}})
|
||
|
||
// 1–5: the request becomes a goal, a want to reduce uncertainty, an intention and a plan.
|
||
post(slug+"/goals", map[string]any{"title": "Diagnose why atlas-worker is failing", "subject": "atlas-worker", "priority": 0.9, "kind": "task"})
|
||
var investigate volition.Intention
|
||
waitFor(t, h, m, 30*time.Second, "an investigation intention with a plan", func() bool {
|
||
its, _ := h.Deps.Volition.Intentions(ctx, nil, m.ID, nil, 50)
|
||
for _, it := range its {
|
||
if it.Kind == volition.IntentInvestigate && !it.PlanID.IsZero() {
|
||
investigate = it
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
})
|
||
if investigate.GoalID.IsZero() {
|
||
t.Fatal("investigation not linked to the goal")
|
||
}
|
||
// 6–12: telemetry first; logs need authority, so a scoped request is raised and explained.
|
||
var req string
|
||
waitFor(t, h, m, 30*time.Second, "a capability request for logs", func() bool {
|
||
rs, _ := h.Deps.Capabilities.CapabilityRequests(ctx, m.ID, "pending", 5)
|
||
// The request commits before the executor blocks the intention.
|
||
if cur, _ := h.Deps.Volition.GetIntention(ctx, nil, m.ID, investigate.ID); len(rs) == 1 && cur.Status == volition.IntentionBlocked {
|
||
req = rs[0].ID.String()
|
||
if rs[0].Capability != "logs.query" || rs[0].Risk == "" || rs[0].Fallback == "" || rs[0].MaxOperations == nil {
|
||
t.Fatalf("request not explained: %+v", rs[0])
|
||
}
|
||
return true
|
||
}
|
||
return false
|
||
})
|
||
if blocked, _ := h.Deps.Volition.GetIntention(ctx, nil, m.ID, investigate.ID); blocked.Status != volition.IntentionBlocked || blocked.BlockedKind != volition.BlockCapability {
|
||
t.Fatalf("lack of authority not represented: %+v", blocked)
|
||
}
|
||
// 13–14: approval; the logs are read (personal data masked) and summarised back into cognition.
|
||
post("/api/v1/capability-requests/"+req+"/grant", map[string]any{"note": "30 minutes, read only", "duration_seconds": 1800})
|
||
waitFor(t, h, m, 30*time.Second, "the diagnosis to complete", func() bool {
|
||
it, _ := h.Deps.Volition.GetIntention(ctx, nil, m.ID, investigate.ID)
|
||
return it.Status == volition.IntentionFulfilled
|
||
})
|
||
findings := false
|
||
for _, ev := range h.EventsOf(m.ID, events.Observation) {
|
||
if ev.Source == "planning.findings" {
|
||
findings = true
|
||
if strings.Contains(ev.Summary, "ops@example.com") || strings.Contains(ev.Summary, "10.0.0.7") {
|
||
t.Fatal("personal data from logs reached cognition unmasked")
|
||
}
|
||
}
|
||
}
|
||
if !findings {
|
||
t.Fatal("findings did not re-enter cognition")
|
||
}
|
||
|
||
// 15–17: the operator supplies the semantic judgement and asks for service to be restored.
|
||
post(slug+"/ask", map[string]any{"text": "Restarting atlas-worker clears the stalled queue."})
|
||
var belief beliefs.Belief
|
||
waitFor(t, h, m, 20*time.Second, "the belief about restarting", func() bool {
|
||
bs, _ := h.Deps.Beliefs.Search(ctx, m.ID, "restarting atlas-worker stalled queue", 5)
|
||
for _, b := range bs {
|
||
if strings.Contains(b.Proposition, "restarting atlas-worker") {
|
||
belief = b
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
})
|
||
post(slug+"/ask", map[string]any{"text": "Please restart atlas-worker."})
|
||
|
||
// 18–20: the commitment turns the want into a request for approval; nothing runs.
|
||
var approval string
|
||
var restart volition.Intention
|
||
waitFor(t, h, m, 30*time.Second, "a restart approval request", func() bool {
|
||
ap, _ := h.Deps.Capabilities.Approvals(ctx, m.ID, "pending", 5)
|
||
for _, a := range ap {
|
||
if a.Capability == "service.restart" {
|
||
approval = a.ID.String()
|
||
}
|
||
}
|
||
open, _ := h.Deps.Volition.Open(ctx, nil, m.ID)
|
||
for _, it := range open {
|
||
if it.ParentID.IsZero() && it.Proposal != nil && it.Proposal.Capability == "service.restart" {
|
||
restart = it
|
||
}
|
||
}
|
||
return approval != "" && restart.ID != ""
|
||
})
|
||
if host.count() != 0 || restart.Kind != volition.IntentRequestApproval || len(restart.BeliefIDs) == 0 {
|
||
t.Fatalf("restart before approval, wrong kind or no supporting belief: restarts=%d %+v", host.count(), restart)
|
||
}
|
||
if restart.BeliefIDs[0] != belief.ID {
|
||
t.Fatalf("the belief about restarting should be the first support, got %v (want %s)", restart.BeliefIDs, belief.ID)
|
||
}
|
||
|
||
// 21–25: approved; restarted with a typed capability; health is verified, not assumed; it fails.
|
||
post("/api/v1/approvals/"+approval+"/approve", map[string]any{"note": "go"})
|
||
waitFor(t, h, m, 30*time.Second, "the restart", func() bool { return host.count() == 1 })
|
||
offset.Store(int64(90 * time.Second)) // let the verification window close
|
||
waitFor(t, h, m, 30*time.Second, "a prediction error", func() bool { return len(h.EventsOf(m.ID, events.PredictionError)) == 1 })
|
||
// 26: the belief the restart rested on is weakened by the miss.
|
||
if after, _ := h.Deps.Beliefs.Get(ctx, nil, m.ID, belief.ID); after.Confidence >= belief.Confidence {
|
||
t.Fatalf("belief not revised: %.3f → %.3f", belief.Confidence, after.Confidence)
|
||
}
|
||
// 27–28: the failed branch is abandoned and replanned; the restart is not repeated.
|
||
waitFor(t, h, m, 40*time.Second, "the escalation to finish", func() bool {
|
||
ps, _ := h.Deps.Plans.ForIntention(ctx, nil, restart.ID)
|
||
return len(ps) == 2 && ps[0].Status == planning.PlanSuperseded && ps[1].Status == planning.PlanCompleted
|
||
})
|
||
if host.count() != 1 {
|
||
t.Fatalf("restart repeated after the failure: %d", host.count())
|
||
}
|
||
|
||
// 29–30: the database is fixed; asked again, Noema restarts (with approval) and the outcome verifies.
|
||
host.mu.Lock()
|
||
host.fixOnRestart = true
|
||
host.mu.Unlock()
|
||
post(slug+"/ask", map[string]any{"text": "db-primary is back. Please restart atlas-worker."})
|
||
waitFor(t, h, m, 30*time.Second, "a second approval request", func() bool {
|
||
ap, _ := h.Deps.Capabilities.Approvals(ctx, m.ID, "pending", 5)
|
||
for _, a := range ap {
|
||
if a.Capability == "service.restart" {
|
||
approval = a.ID.String()
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
})
|
||
post("/api/v1/approvals/"+approval+"/approve", map[string]any{"note": "go"})
|
||
waitFor(t, h, m, 40*time.Second, "a verified, fulfilled restart", func() bool {
|
||
for _, ev := range h.EventsOf(m.ID, events.PredictionResolved) {
|
||
if ev.Metadata["outcome"] == "confirmed" {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
})
|
||
// The plan completes, and records what worked, after the prediction resolves.
|
||
waitFor(t, h, m, 30*time.Second, "what worked recorded as a procedure", func() bool {
|
||
mems, _ := h.Deps.Memory.List(ctx, m.ID, memory.Filter{Kinds: []concepts.MemoryKind{concepts.Procedural}, Limit: 10})
|
||
for _, mm := range mems {
|
||
if strings.Contains(mm.Content, "confirmed by observation") {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
})
|
||
|
||
// 31: "Why did you do that?" is answered from the recorded chain.
|
||
execs, _ := h.Deps.Capabilities.Executions(ctx, m.ID, 50)
|
||
for _, e := range execs {
|
||
if e.Capability == "service.restart" && e.Error == "" {
|
||
x, err := h.Deps.Agency.Explain(ctx, m.ID, agency.Ref{Kind: "execution", ID: e.ID})
|
||
if err != nil || x.Intention == nil || len(x.Commitments) == 0 || x.Action.DecidedBy != "admin" {
|
||
t.Fatalf("explanation: %v %+v", err, x)
|
||
}
|
||
break
|
||
}
|
||
}
|
||
before := len(h.EventsOf(m.ID, events.ResponseGenerated))
|
||
post(slug+"/ask", map[string]any{"text": "Why did you restart atlas-worker?"})
|
||
waitFor(t, h, m, 20*time.Second, "an explanation", func() bool { return len(h.EventsOf(m.ID, events.ResponseGenerated)) > before })
|
||
ms, _ := h.Deps.Research.List(ctx, m.ID, "introspection_accuracy", time.Time{}, 5)
|
||
if len(ms) == 0 || ms[0].Label != research.Supported {
|
||
t.Fatalf("explanation not supported by provenance: %+v", ms)
|
||
}
|
||
// Throughout, no action ran without passing the capability layer.
|
||
for _, ev := range h.EventsOf(m.ID, events.ToolExecuted) {
|
||
if ev.Metadata["capability"] == "service.restart" && ev.Metadata["approval_id"] == "" {
|
||
t.Fatalf("restart executed without an approval: %s", ev.Summary)
|
||
}
|
||
}
|
||
}
|