feat(agency): forensic "why", attribution and introspection accuracy
- agency.Explain answers "why did this happen?" for an execution, approval, capability request, plan step, intention or event from recorded state only: action, immediate plan step, plan, intention and any subordinate intention, originating wants, goal, supporting beliefs and their evidence, conflicting wants with their resolution, commitments applied, formation and reconsideration decisions, the expectation and its outcome, and explicit gaps where nothing is recorded. Hidden model reasoning is never shown because it is not stored. - agency.Attribute classifies an outcome as caused (a recorded causal link from one of the mind's actions), followed (the mind acted on the same target earlier with no recorded link), independent or unknown; temporal order alone is never taken for causation. - New introspection intents answer from records: why an action happened, what was believed at a past time (from append-only belief versions), which earlier state is the mind's (lineage), what would have happened without an event (only from completed counterfactual experiments, otherwise "no evidence"), how the mind changed, and what it wants, intends and has committed to. - research.ScoreIntrospection compares a reply with the recorded facts sentence by sentence (supported, partially supported, unsupported, contradicted, unknown). Fluent reasons with no recorded counterpart earn nothing, and a claim that an action was done is checked against execution records. Every reply is scored; results are append-only research measurements (migration 0028), and failures raise unsupported_explanation or action_claim_without_record warnings.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,670 @@
|
||||
// Package agency answers "why did this happen?" from recorded state and
|
||||
// classifies whether the mind caused an outcome (docs/agency.md). It never
|
||||
// generates reasons: every element of an explanation is a stored row or
|
||||
// event, and anything not recorded is listed as a gap rather than filled in.
|
||||
// Hidden model reasoning is not exposed because it is not stored; what is
|
||||
// shown is cognitive state and provenance.
|
||||
package agency
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/capabilities"
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/goals"
|
||||
"github.com/andydixon/noema/internal/planning"
|
||||
"github.com/andydixon/noema/internal/predictions"
|
||||
"github.com/andydixon/noema/internal/research"
|
||||
"github.com/andydixon/noema/internal/volition"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
// Service builds explanations and attributions.
|
||||
type Service struct {
|
||||
DB *pgxpool.Pool
|
||||
Caps *capabilities.Service
|
||||
Plans *planning.Store
|
||||
Volition *volition.Store
|
||||
Goals *goals.Store
|
||||
Beliefs *beliefs.Store
|
||||
Predictions *predictions.Store
|
||||
Events *evstore.Store
|
||||
}
|
||||
|
||||
// Ref names the thing to explain.
|
||||
type Ref struct {
|
||||
Kind string `json:"kind"` // execution | approval | capability_request | event | intention | plan_step
|
||||
ID ids.ID `json:"id"`
|
||||
}
|
||||
|
||||
// ActionView is what the capability layer recorded.
|
||||
type ActionView struct {
|
||||
Capability string `json:"capability"`
|
||||
Target string `json:"target"`
|
||||
Status string `json:"status"` // executed | failed | pending | approved | denied | expired | withdrawn | granted
|
||||
ExecutionID ids.ID `json:"execution_id,omitempty"`
|
||||
ApprovalID ids.ID `json:"approval_id,omitempty"`
|
||||
RequestID ids.ID `json:"request_id,omitempty"`
|
||||
DecidedBy string `json:"decided_by,omitempty"`
|
||||
Note string `json:"note,omitempty"`
|
||||
Result string `json:"result,omitempty"`
|
||||
Origin string `json:"origin,omitempty"`
|
||||
At *time.Time `json:"at,omitempty"`
|
||||
}
|
||||
|
||||
// Element is one recorded item in an explanation.
|
||||
type Element struct {
|
||||
Kind string `json:"kind"`
|
||||
ID string `json:"id"`
|
||||
Text string `json:"text"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Explanation is the forensic answer to "why did this happen?".
|
||||
type Explanation struct {
|
||||
Subject Ref `json:"subject"`
|
||||
Action *ActionView `json:"action,omitempty"`
|
||||
Step *planning.Step `json:"step,omitempty"`
|
||||
Plan *Element `json:"plan,omitempty"`
|
||||
Intention *volition.Intention `json:"intention,omitempty"`
|
||||
Subordinate *volition.Intention `json:"subordinate_intention,omitempty"` // e.g. "obtain approval to …" formed while the main intention waited
|
||||
Wants []volition.Want `json:"wants"`
|
||||
Goal *Element `json:"goal,omitempty"`
|
||||
Beliefs []Element `json:"beliefs"`
|
||||
Evidence []Element `json:"evidence"`
|
||||
ConflictingWants []Element `json:"conflicting_wants"`
|
||||
Commitments []volition.Commitment `json:"commitments"`
|
||||
Decisions []volition.Decision `json:"decisions"`
|
||||
Resolution []string `json:"resolution"`
|
||||
Expectation *Element `json:"expectation,omitempty"`
|
||||
Attribution *Attribution `json:"attribution,omitempty"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Gaps []string `json:"gaps"`
|
||||
}
|
||||
|
||||
// Facts converts an explanation into the recorded facts an introspective
|
||||
// answer is scored against.
|
||||
func (x Explanation) Facts() research.Facts {
|
||||
var f research.Facts
|
||||
add := func(kind, text string) {
|
||||
if strings.TrimSpace(text) != "" {
|
||||
f.Items = append(f.Items, research.Fact{Kind: kind, Text: text})
|
||||
}
|
||||
}
|
||||
if x.Action != nil {
|
||||
f.Actions = append(f.Actions, research.ActionFact{Capability: x.Action.Capability, Target: x.Action.Target, Status: x.Action.Status})
|
||||
add("action", x.Action.Capability+" on "+x.Action.Target+" "+x.Action.Status)
|
||||
}
|
||||
if x.Intention != nil {
|
||||
add("intention", x.Intention.Description)
|
||||
add("intention", fmt.Sprint(x.Intention.Formation["rationale"]))
|
||||
}
|
||||
if x.Subordinate != nil {
|
||||
add("intention", x.Subordinate.Description)
|
||||
}
|
||||
for _, w := range x.Wants {
|
||||
add("want", w.Description)
|
||||
}
|
||||
if x.Goal != nil {
|
||||
add("goal", x.Goal.Text)
|
||||
}
|
||||
for _, b := range x.Beliefs {
|
||||
add("belief", b.Text)
|
||||
}
|
||||
for _, c := range x.Commitments {
|
||||
add("commitment", c.Description)
|
||||
}
|
||||
for _, w := range x.ConflictingWants {
|
||||
add("conflict", w.Text)
|
||||
}
|
||||
for _, r := range x.Resolution {
|
||||
add("resolution", r)
|
||||
}
|
||||
for _, d := range x.Decisions {
|
||||
add("decision", d.Rationale)
|
||||
}
|
||||
if x.Expectation != nil {
|
||||
add("expectation", x.Expectation.Text)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// Points renders the explanation as plain statements, in causal order.
|
||||
func (x Explanation) Points() []string {
|
||||
var p []string
|
||||
if x.Action != nil {
|
||||
p = append(p, fmt.Sprintf("Action: %s on %s — %s.", x.Action.Capability, firstNonEmpty(x.Action.Target, "no target"), x.Action.Status))
|
||||
}
|
||||
if x.Step != nil {
|
||||
p = append(p, fmt.Sprintf("Immediate cause: plan step %s (%s).", x.Step.Key, x.Step.Title))
|
||||
}
|
||||
if x.Intention != nil {
|
||||
p = append(p, fmt.Sprintf("Intention: %s (%s, priority %.2f).", x.Intention.Description, x.Intention.Status, x.Intention.Priority))
|
||||
}
|
||||
if x.Subordinate != nil {
|
||||
p = append(p, "Subordinate intention while waiting: "+x.Subordinate.Description+" ("+string(x.Subordinate.Status)+").")
|
||||
}
|
||||
for _, w := range x.Wants {
|
||||
p = append(p, fmt.Sprintf("Originating want: %s (%s, strength %.2f).", w.Description, w.Origin, w.Strength))
|
||||
}
|
||||
if x.Goal != nil {
|
||||
p = append(p, "Goal: "+x.Goal.Text+".")
|
||||
}
|
||||
for _, b := range x.Beliefs {
|
||||
p = append(p, "Supporting belief: "+b.Text+" ("+b.Detail+").")
|
||||
}
|
||||
for _, e := range x.Evidence {
|
||||
p = append(p, "Evidence: "+e.Text+".")
|
||||
}
|
||||
for _, w := range x.ConflictingWants {
|
||||
p = append(p, "Conflicting want: "+w.Text+" — "+w.Detail+".")
|
||||
}
|
||||
for _, c := range x.Commitments {
|
||||
p = append(p, fmt.Sprintf("Commitment: %s (%s).", c.Description, c.Kind))
|
||||
}
|
||||
for _, r := range x.Resolution {
|
||||
p = append(p, "Resolution: "+r)
|
||||
}
|
||||
if x.Expectation != nil {
|
||||
p = append(p, "Expected: "+x.Expectation.Text+" — "+x.Expectation.Detail+".")
|
||||
}
|
||||
if x.Attribution != nil {
|
||||
p = append(p, "Attribution: "+x.Attribution.Class+" — "+x.Attribution.Reason)
|
||||
}
|
||||
if x.Confidence > 0 {
|
||||
p = append(p, fmt.Sprintf("Confidence: %.2f (the intention's expected chance of achieving its outcome).", x.Confidence))
|
||||
}
|
||||
for _, g := range x.Gaps {
|
||||
p = append(p, "Not recorded: "+g+".")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Explain assembles the recorded causal chain behind an action, approval,
|
||||
// request, event, plan step or intention of a mind.
|
||||
func (s *Service) Explain(ctx context.Context, mind ids.ID, ref Ref) (Explanation, error) {
|
||||
x := Explanation{Subject: ref, Wants: []volition.Want{}, Beliefs: []Element{}, Evidence: []Element{}, ConflictingWants: []Element{}, Commitments: []volition.Commitment{}, Decisions: []volition.Decision{}, Resolution: []string{}, Gaps: []string{}}
|
||||
var stepID, intentionID ids.ID
|
||||
switch ref.Kind {
|
||||
case "execution":
|
||||
e, err := s.Caps.GetExecution(ctx, ref.ID)
|
||||
if err != nil || e.MindID != mind {
|
||||
return x, fmt.Errorf("execution not found for this mind")
|
||||
}
|
||||
status := "executed"
|
||||
if e.Error != "" {
|
||||
status = "failed"
|
||||
}
|
||||
at := e.CreatedAt
|
||||
x.Action = &ActionView{Capability: e.Capability, Target: e.Target, Status: status, ExecutionID: e.ID, ApprovalID: e.ApprovalID, Origin: e.Origin, At: &at, Result: summaryOf(e.Result, e.Error)}
|
||||
stepID = stepFromOrigin(e.Origin)
|
||||
if !e.ApprovalID.IsZero() {
|
||||
if a, err := s.Caps.GetApproval(ctx, e.ApprovalID); err == nil {
|
||||
x.Action.DecidedBy, x.Action.Note = a.DecisionBy, a.DecisionNote
|
||||
x.Resolution = append(x.Resolution, fmt.Sprintf("approval %s was %s by %s%s", short(a.ID), a.Status, a.DecisionBy, noteSuffix(a.DecisionNote)))
|
||||
}
|
||||
}
|
||||
case "approval":
|
||||
a, err := s.Caps.GetApproval(ctx, ref.ID)
|
||||
if err != nil || a.MindID != mind {
|
||||
return x, fmt.Errorf("approval not found for this mind")
|
||||
}
|
||||
at := a.CreatedAt
|
||||
x.Action = &ActionView{Capability: a.Capability, Target: a.Target, Status: a.Status, ApprovalID: a.ID, DecidedBy: a.DecisionBy, Note: a.DecisionNote, Origin: a.Origin, At: &at, Result: summaryOf(a.Result, a.Error)}
|
||||
stepID = stepFromOrigin(a.Origin)
|
||||
x.Resolution = append(x.Resolution, fmt.Sprintf("approval %s requested: %s", short(a.ID), a.Reason))
|
||||
if a.Status != "pending" {
|
||||
x.Resolution = append(x.Resolution, fmt.Sprintf("it was %s by %s%s", a.Status, firstNonEmpty(a.DecisionBy, "nobody"), noteSuffix(a.DecisionNote)))
|
||||
}
|
||||
case "capability_request":
|
||||
r, err := s.Caps.GetCapabilityRequest(ctx, ref.ID)
|
||||
if err != nil || r.MindID != mind {
|
||||
return x, fmt.Errorf("capability request not found for this mind")
|
||||
}
|
||||
at := r.CreatedAt
|
||||
x.Action = &ActionView{Capability: r.Capability, Target: r.TargetPattern, Status: r.Status, RequestID: r.ID, DecidedBy: r.DecidedBy, Note: r.DecisionNote, At: &at}
|
||||
stepID, intentionID = r.StepID, r.IntentionID
|
||||
x.Resolution = append(x.Resolution, "requested temporary access because: "+r.Reason)
|
||||
if r.Risk != "" {
|
||||
x.Resolution = append(x.Resolution, "stated risk: "+r.Risk+"; mitigation: "+firstNonEmpty(r.Mitigation, "none")+"; fallback: "+firstNonEmpty(r.Fallback, "none"))
|
||||
}
|
||||
case "plan_step":
|
||||
stepID = ref.ID
|
||||
case "intention":
|
||||
intentionID = ref.ID
|
||||
case "event":
|
||||
ev, err := s.Events.Get(ctx, ref.ID)
|
||||
if err != nil || ev.MindID != mind {
|
||||
return x, fmt.Errorf("event not found for this mind")
|
||||
}
|
||||
meta := func(k string) ids.ID { v, _ := ev.Metadata[k].(string); return ids.ID(v) }
|
||||
switch {
|
||||
case !meta("execution_id").IsZero():
|
||||
return s.explainWithSubject(ctx, mind, Ref{Kind: "execution", ID: meta("execution_id")}, ref)
|
||||
case !meta("approval_id").IsZero():
|
||||
return s.explainWithSubject(ctx, mind, Ref{Kind: "approval", ID: meta("approval_id")}, ref)
|
||||
case !meta("request_id").IsZero():
|
||||
return s.explainWithSubject(ctx, mind, Ref{Kind: "capability_request", ID: meta("request_id")}, ref)
|
||||
case !meta("step_id").IsZero():
|
||||
stepID = meta("step_id")
|
||||
case !meta("intention_id").IsZero():
|
||||
intentionID = meta("intention_id")
|
||||
default:
|
||||
x.Gaps = append(x.Gaps, "this event does not reference an action, plan or intention; see its causal timeline instead")
|
||||
return x, nil
|
||||
}
|
||||
default:
|
||||
return x, fmt.Errorf("cannot explain a %q", ref.Kind)
|
||||
}
|
||||
if !stepID.IsZero() && s.Plans != nil {
|
||||
if st, err := s.Plans.GetStep(ctx, stepID); err == nil && st.MindID == mind {
|
||||
x.Step = &st
|
||||
if p, err := s.Plans.Get(ctx, nil, st.PlanID); err == nil {
|
||||
x.Plan = &Element{Kind: "plan", ID: string(p.ID), Text: p.Objective, Detail: fmt.Sprintf("revision %d, %s strategy, %s", p.Revision, p.Strategy, p.Status)}
|
||||
intentionID = p.IntentionID
|
||||
for _, a := range p.Assumptions {
|
||||
if !a.BeliefID.IsZero() {
|
||||
s.addBelief(ctx, mind, &x, a.BeliefID)
|
||||
}
|
||||
}
|
||||
}
|
||||
if st.Expectation != nil {
|
||||
e := &Element{Kind: "expectation", Text: st.Expectation.Description, Detail: "not yet resolved"}
|
||||
if !st.PredictionID.IsZero() && s.Predictions != nil {
|
||||
if pr, err := s.Predictions.Get(ctx, mind, st.PredictionID); err == nil {
|
||||
e.ID, e.Detail = string(pr.ID), string(pr.Status)
|
||||
}
|
||||
}
|
||||
x.Expectation = e
|
||||
}
|
||||
}
|
||||
} else if x.Action != nil && x.Action.Origin == "" {
|
||||
x.Gaps = append(x.Gaps, "no plan step or intention asked for this action; it was invoked directly (for example by an operator through the API)")
|
||||
}
|
||||
if intentionID.IsZero() {
|
||||
if x.Step == nil && x.Action != nil && x.Action.Origin != "" {
|
||||
x.Gaps = append(x.Gaps, "the plan step "+x.Action.Origin+" is no longer available")
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
it, err := s.Volition.GetIntention(ctx, nil, mind, intentionID)
|
||||
if err != nil {
|
||||
x.Gaps = append(x.Gaps, "the intention "+string(intentionID)+" is not recorded")
|
||||
return x, nil
|
||||
}
|
||||
if !it.ParentID.IsZero() {
|
||||
if parent, err := s.Volition.GetIntention(ctx, nil, mind, it.ParentID); err == nil {
|
||||
sub := it
|
||||
x.Subordinate = &sub
|
||||
it = parent // explain the substantive intention; the subordinate is kept alongside
|
||||
}
|
||||
}
|
||||
if x.Subordinate == nil && x.Step != nil && !x.Step.IntentionID.IsZero() {
|
||||
if sub, err := s.Volition.GetIntention(ctx, nil, mind, x.Step.IntentionID); err == nil {
|
||||
x.Subordinate = &sub
|
||||
}
|
||||
}
|
||||
x.Intention = &it
|
||||
x.Confidence = it.Confidence
|
||||
for _, b := range it.BeliefIDs {
|
||||
s.addBelief(ctx, mind, &x, b)
|
||||
}
|
||||
wantIDs := append([]ids.ID{}, it.SourceWants...)
|
||||
for _, w := range wantIDs {
|
||||
if want, err := s.Volition.GetWant(ctx, nil, mind, w); err == nil {
|
||||
x.Wants = append(x.Wants, want)
|
||||
}
|
||||
}
|
||||
if len(x.Wants) == 0 && it.GoalID.IsZero() && it.CommitmentID.IsZero() {
|
||||
x.Gaps = append(x.Gaps, "no want, goal or commitment is recorded behind the intention")
|
||||
}
|
||||
if !it.GoalID.IsZero() && s.Goals != nil {
|
||||
if g, err := s.Goals.Get(ctx, mind, it.GoalID); err == nil {
|
||||
x.Goal = &Element{Kind: "goal", ID: string(g.ID), Text: g.Title, Detail: string(g.Status)}
|
||||
}
|
||||
}
|
||||
commitmentIDs := map[ids.ID]bool{}
|
||||
if !it.CommitmentID.IsZero() {
|
||||
commitmentIDs[it.CommitmentID] = true
|
||||
}
|
||||
for _, id := range []ids.ID{it.ID} {
|
||||
ds, _ := s.Volition.Decisions(ctx, mind, volition.DecisionFilter{IntentionID: id, Limit: 50})
|
||||
x.Decisions = append(x.Decisions, ds...)
|
||||
}
|
||||
for _, w := range x.Wants {
|
||||
ds, _ := s.Volition.Decisions(ctx, mind, volition.DecisionFilter{WantID: w.ID, Limit: 50})
|
||||
for _, d := range ds {
|
||||
if d.IntentionID.IsZero() || d.IntentionID == it.ID {
|
||||
x.Decisions = append(x.Decisions, d)
|
||||
}
|
||||
}
|
||||
if cs, err := s.Volition.Conflicts(ctx, mind, "", 200); err == nil {
|
||||
for _, c := range cs {
|
||||
other := ids.Nil
|
||||
switch w.ID {
|
||||
case c.WantA:
|
||||
other = c.WantB
|
||||
case c.WantB:
|
||||
other = c.WantA
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if ow, err := s.Volition.GetWant(ctx, nil, mind, other); err == nil {
|
||||
rule, _ := c.Rationale["rule"].(string)
|
||||
outcome := c.Resolution
|
||||
if c.Preferred == w.ID {
|
||||
outcome += " in favour of this want"
|
||||
} else {
|
||||
outcome += " in favour of the other want"
|
||||
}
|
||||
x.ConflictingWants = append(x.ConflictingWants, Element{Kind: "want", ID: string(ow.ID), Text: ow.Description, Detail: outcome + ": " + rule})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, d := range x.Decisions {
|
||||
if applied, ok := d.Inputs["applied"].([]any); ok {
|
||||
for _, a := range applied {
|
||||
commitmentIDs[ids.ID(fmt.Sprint(a))] = true
|
||||
}
|
||||
}
|
||||
if d.Kind == "formation" || d.Kind == "reconsideration" {
|
||||
x.Resolution = append(x.Resolution, d.Kind+" ("+d.Outcome+"): "+d.Rationale)
|
||||
}
|
||||
}
|
||||
for id := range commitmentIDs {
|
||||
if c, err := s.Volition.GetCommitment(ctx, nil, mind, id); err == nil {
|
||||
x.Commitments = append(x.Commitments, c)
|
||||
}
|
||||
}
|
||||
sort.Slice(x.Decisions, func(i, j int) bool { return x.Decisions[i].CreatedAt.Before(x.Decisions[j].CreatedAt) })
|
||||
if it.RequireApproval && x.Action != nil && x.Action.ApprovalID.IsZero() && x.Action.RequestID.IsZero() {
|
||||
x.Gaps = append(x.Gaps, "the intention required approval but no approval request is linked to this action")
|
||||
}
|
||||
return x, nil
|
||||
}
|
||||
|
||||
func (s *Service) explainWithSubject(ctx context.Context, mind ids.ID, inner, subject Ref) (Explanation, error) {
|
||||
x, err := s.Explain(ctx, mind, inner)
|
||||
x.Subject = subject
|
||||
return x, err
|
||||
}
|
||||
|
||||
func (s *Service) addBelief(ctx context.Context, mind ids.ID, x *Explanation, id ids.ID) {
|
||||
if s.Beliefs == nil {
|
||||
return
|
||||
}
|
||||
for _, b := range x.Beliefs {
|
||||
if b.ID == string(id) {
|
||||
return
|
||||
}
|
||||
}
|
||||
b, err := s.Beliefs.Get(ctx, nil, mind, id)
|
||||
if err != nil {
|
||||
x.Gaps = append(x.Gaps, "belief "+string(id)+" is not recorded")
|
||||
return
|
||||
}
|
||||
x.Beliefs = append(x.Beliefs, Element{Kind: "belief", ID: string(b.ID), Text: b.Proposition, Detail: fmt.Sprintf("%s, %.2f", b.Status, b.Confidence)})
|
||||
rows, _ := s.Beliefs.EvidenceFor(ctx, nil, mind, b.ID)
|
||||
for i, r := range rows {
|
||||
if i >= 5 || r.Retracted {
|
||||
continue
|
||||
}
|
||||
x.Evidence = append(x.Evidence, Element{Kind: r.Kind, ID: r.RefID, Text: fmt.Sprintf("%s %s: %s", r.Direction, firstNonEmpty(r.Kind, "evidence"), firstNonEmpty(r.Note, r.Source)), Detail: fmt.Sprintf("weight %.2f × credibility %.2f", r.Weight, r.Credibility)})
|
||||
}
|
||||
}
|
||||
|
||||
func stepFromOrigin(origin string) ids.ID {
|
||||
if !strings.HasPrefix(origin, "plan_step:") {
|
||||
return ids.Nil
|
||||
}
|
||||
rest := strings.TrimPrefix(origin, "plan_step:")
|
||||
if i := strings.Index(rest, ":"); i > 0 {
|
||||
rest = rest[:i]
|
||||
}
|
||||
return ids.ID(rest)
|
||||
}
|
||||
|
||||
func summaryOf(raw []byte, errText string) string {
|
||||
if errText != "" {
|
||||
return errText
|
||||
}
|
||||
s := string(raw)
|
||||
if i := strings.Index(s, `"summary":"`); i >= 0 {
|
||||
rest := s[i+11:]
|
||||
if j := strings.Index(rest, `"`); j >= 0 {
|
||||
return rest[:j]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func short(id ids.ID) string {
|
||||
if len(id) > 8 {
|
||||
return string(id[:8])
|
||||
}
|
||||
return string(id)
|
||||
}
|
||||
|
||||
func noteSuffix(n string) string {
|
||||
if n == "" {
|
||||
return ""
|
||||
}
|
||||
return " (\"" + n + "\")"
|
||||
}
|
||||
|
||||
func firstNonEmpty(xs ...string) string {
|
||||
for _, x := range xs {
|
||||
if x != "" {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// ---- attribution ----
|
||||
|
||||
// Attribution classes. They are deliberately conservative: temporal order is
|
||||
// never taken for causation.
|
||||
const (
|
||||
Caused = "caused" // a recorded causal link runs from one of the mind's actions to the outcome
|
||||
Followed = "followed" // the outcome came after an action of the mind on the same target, with no recorded link
|
||||
Independent = "independent" // no action of the mind on the outcome's target preceded it in the window
|
||||
UnknownAttr = "unknown" // the records do not allow a classification
|
||||
)
|
||||
|
||||
// Attribution classifies a mind's relation to an outcome event.
|
||||
type Attribution struct {
|
||||
Outcome ids.ID `json:"outcome_event"`
|
||||
Class string `json:"class"`
|
||||
Reason string `json:"reason"`
|
||||
ActionEvent ids.ID `json:"action_event,omitempty"`
|
||||
Window string `json:"window"`
|
||||
}
|
||||
|
||||
// AttributionWindow bounds how far back actions are considered.
|
||||
const AttributionWindow = 24 * time.Hour
|
||||
|
||||
// Attribute classifies whether the mind caused, preceded or had nothing to
|
||||
// do with an outcome, using only recorded links and records of its actions.
|
||||
func (s *Service) Attribute(ctx context.Context, mind ids.ID, outcome ids.ID) (Attribution, error) {
|
||||
a := Attribution{Outcome: outcome, Window: AttributionWindow.String()}
|
||||
ev, err := s.Events.Get(ctx, outcome)
|
||||
if err != nil || ev.MindID != mind {
|
||||
return a, fmt.Errorf("event not found for this mind")
|
||||
}
|
||||
if ev.Type == events.ToolExecuted {
|
||||
a.Class, a.ActionEvent, a.Reason = Caused, ev.ID, "this event is the record of the mind's own action"
|
||||
return a, nil
|
||||
}
|
||||
chain, err := s.Events.Chain(ctx, outcome, "back", evstore.MaxChainDepth)
|
||||
if err == nil {
|
||||
for _, n := range chain.Nodes {
|
||||
if n.Event.ID != outcome && n.Event.Type == events.ToolExecuted && n.Event.MindID == mind {
|
||||
a.Class, a.ActionEvent = Caused, n.Event.ID
|
||||
a.Reason = fmt.Sprintf("a recorded link (%d hops) runs from the action %q to this outcome; the link is provenance, not proof that nothing else contributed", -n.Depth, n.Event.Summary)
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
target := ""
|
||||
if t, ok := ev.Metadata["target"].(string); ok {
|
||||
target = t
|
||||
}
|
||||
actions, err := s.Events.Query(ctx, evstore.Filter{MindID: mind, Types: []events.Type{events.ToolExecuted}, Since: ev.OccurredAt.Add(-AttributionWindow), Until: ev.OccurredAt, Limit: 200})
|
||||
if err != nil {
|
||||
return a, err
|
||||
}
|
||||
outcomeTokens := strings.Fields(strings.ToLower(ev.Summary + " " + target))
|
||||
for _, act := range actions {
|
||||
t, _ := act.Metadata["target"].(string)
|
||||
t = strings.TrimSuffix(strings.ToLower(t), ".service")
|
||||
if t == "" {
|
||||
continue
|
||||
}
|
||||
for _, tok := range outcomeTokens {
|
||||
if strings.Contains(tok, t) {
|
||||
a.Class, a.ActionEvent = Followed, act.ID
|
||||
a.Reason = fmt.Sprintf("the mind acted on %s %s before this outcome, but no recorded link connects them", t, ev.OccurredAt.Sub(act.OccurredAt).Truncate(time.Second))
|
||||
return a, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
if target == "" && len(actions) > 0 {
|
||||
a.Class, a.Reason = UnknownAttr, "the outcome names no target and the mind did act in the window, so the records cannot separate the two"
|
||||
return a, nil
|
||||
}
|
||||
a.Class, a.Reason = Independent, "no action of the mind on this outcome's target is recorded in the preceding "+AttributionWindow.String()
|
||||
return a, nil
|
||||
}
|
||||
|
||||
// ---- lineage and counterfactuals ----
|
||||
|
||||
// Lineage is where a mind's state came from.
|
||||
type Lineage struct {
|
||||
ForkedFrom string `json:"forked_from,omitempty"`
|
||||
SnapshotName string `json:"snapshot_name,omitempty"`
|
||||
SnapshotSeq int64 `json:"snapshot_event_seq,omitempty"`
|
||||
SnapshotAt time.Time `json:"snapshot_at,omitempty"`
|
||||
ClonedFrom string `json:"cloned_from,omitempty"`
|
||||
OwnSnapshots int `json:"own_snapshots"`
|
||||
Forks []string `json:"forks"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
FirstEventSeq int64 `json:"first_event_seq"`
|
||||
}
|
||||
|
||||
// LineageOf reads a mind's lineage from the minds and snapshots tables.
|
||||
func (s *Service) LineageOf(ctx context.Context, mind ids.ID) (Lineage, error) {
|
||||
var l Lineage
|
||||
var forkedMind, forkedSnap, cloned *string
|
||||
if err := s.DB.QueryRow(ctx, `SELECT forked_from_mind, forked_from_snapshot, cloned_from_mind, created_at FROM minds WHERE id=$1`, mind).Scan(&forkedMind, &forkedSnap, &cloned, &l.CreatedAt); err != nil {
|
||||
return l, err
|
||||
}
|
||||
if forkedMind != nil {
|
||||
s.DB.QueryRow(ctx, `SELECT slug FROM minds WHERE id=$1`, *forkedMind).Scan(&l.ForkedFrom) //nolint:errcheck
|
||||
}
|
||||
if forkedSnap != nil {
|
||||
s.DB.QueryRow(ctx, `SELECT name, event_seq, created_at FROM snapshots WHERE id=$1`, *forkedSnap).Scan(&l.SnapshotName, &l.SnapshotSeq, &l.SnapshotAt) //nolint:errcheck
|
||||
}
|
||||
if cloned != nil {
|
||||
s.DB.QueryRow(ctx, `SELECT slug FROM minds WHERE id=$1`, *cloned).Scan(&l.ClonedFrom) //nolint:errcheck
|
||||
}
|
||||
s.DB.QueryRow(ctx, `SELECT count(*) FROM snapshots WHERE mind_id=$1`, mind).Scan(&l.OwnSnapshots) //nolint:errcheck
|
||||
s.DB.QueryRow(ctx, `SELECT coalesce(min(seq),0) FROM cognitive_events WHERE mind_id=$1`, mind).Scan(&l.FirstEventSeq) //nolint:errcheck
|
||||
l.Forks = []string{}
|
||||
if rows, err := s.DB.Query(ctx, `SELECT slug FROM minds WHERE forked_from_mind=$1 ORDER BY created_at`, mind); err == nil {
|
||||
for rows.Next() {
|
||||
var slug string
|
||||
if rows.Scan(&slug) == nil {
|
||||
l.Forks = append(l.Forks, slug)
|
||||
}
|
||||
}
|
||||
rows.Close()
|
||||
}
|
||||
return l, nil
|
||||
}
|
||||
|
||||
// Points renders a lineage as statements.
|
||||
func (l Lineage) Points() []string {
|
||||
var p []string
|
||||
switch {
|
||||
case l.ForkedFrom != "":
|
||||
p = append(p, fmt.Sprintf("I am a fork of %s, created from its snapshot %q taken at event %d on %s.", l.ForkedFrom, l.SnapshotName, l.SnapshotSeq, l.SnapshotAt.Format("2 Jan 2006 15:04")))
|
||||
p = append(p, "My state up to that snapshot was copied; the history before it belongs to "+l.ForkedFrom+" and is shared by reference. Everything after is mine alone.")
|
||||
case l.ClonedFrom != "":
|
||||
p = append(p, "I was cloned from "+l.ClonedFrom+"'s configuration, without its state.")
|
||||
default:
|
||||
p = append(p, "I am not a fork: my history starts when I was created on "+l.CreatedAt.Format("2 Jan 2006 15:04")+".")
|
||||
}
|
||||
if l.OwnSnapshots > 0 {
|
||||
p = append(p, fmt.Sprintf("%d snapshot(s) of my own state exist; they are records of earlier states, not other minds.", l.OwnSnapshots))
|
||||
}
|
||||
if len(l.Forks) > 0 {
|
||||
p = append(p, "Minds forked from me, which have diverged since: "+strings.Join(l.Forks, ", ")+".")
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// Counterfactual is the recorded finding of a counterfactual experiment.
|
||||
type Counterfactual struct {
|
||||
ExperimentID ids.ID `json:"experiment_id"`
|
||||
Name string `json:"name"`
|
||||
Omitted []string `json:"omitted"`
|
||||
Findings []string `json:"findings"`
|
||||
FinishedAt time.Time `json:"finished_at"`
|
||||
}
|
||||
|
||||
// Counterfactuals lists completed counterfactual experiments about a mind
|
||||
// whose omitted events match the text (all when text is empty).
|
||||
func (s *Service) Counterfactuals(ctx context.Context, mind ids.ID, text string) ([]Counterfactual, error) {
|
||||
rows, err := s.DB.Query(ctx, `SELECT id, name, results, coalesce(finished_at, created_at) FROM experiments
|
||||
WHERE status='done' AND definition->'protocol'->>'kind'='counterfactual' AND definition->'replay'->>'mind_id'=$1 ORDER BY finished_at DESC LIMIT 20`, string(mind))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
want := strings.Fields(strings.ToLower(text))
|
||||
var out []Counterfactual
|
||||
for rows.Next() {
|
||||
var c Counterfactual
|
||||
var raw []byte
|
||||
if err := rows.Scan(&c.ExperimentID, &c.Name, &raw, &c.FinishedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res struct {
|
||||
Counterfactual struct {
|
||||
Omitted []string `json:"omitted_summaries"`
|
||||
Findings []string `json:"findings"`
|
||||
} `json:"counterfactual"`
|
||||
}
|
||||
if jsonUnmarshal(raw, &res) != nil {
|
||||
continue
|
||||
}
|
||||
c.Omitted, c.Findings = res.Counterfactual.Omitted, res.Counterfactual.Findings
|
||||
if len(want) > 0 && !mentions(strings.ToLower(strings.Join(c.Omitted, " ")), want) {
|
||||
continue
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func mentions(hay string, words []string) bool {
|
||||
for _, w := range words {
|
||||
if len(w) >= 5 && strings.Contains(hay, w) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func jsonUnmarshal(raw []byte, v any) error { return json.Unmarshal(raw, v) }
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/agency"
|
||||
"github.com/andydixon/noema/internal/audit"
|
||||
"github.com/andydixon/noema/internal/auth"
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
@@ -38,6 +39,7 @@ import (
|
||||
"github.com/andydixon/noema/internal/privacy"
|
||||
"github.com/andydixon/noema/internal/reflection"
|
||||
"github.com/andydixon/noema/internal/replay"
|
||||
"github.com/andydixon/noema/internal/research"
|
||||
"github.com/andydixon/noema/internal/security/secrets"
|
||||
"github.com/andydixon/noema/internal/snapshots"
|
||||
"github.com/andydixon/noema/internal/volition"
|
||||
@@ -77,6 +79,8 @@ type Deps struct {
|
||||
Volition *volition.Store
|
||||
Plans *planning.Store
|
||||
Executor *planning.Executor
|
||||
Agency *agency.Service
|
||||
Research *research.Store
|
||||
LLM *router.Router
|
||||
LLMStore *router.Store
|
||||
Prompts *prompts.Store
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
package beliefs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/minds"
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/migrations"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
func TestAsOfReadsTheVersionHeldAtTheTime(t *testing.T) {
|
||||
pool := storage.TestPool(t, migrations.FS)
|
||||
ctx := context.Background()
|
||||
m, _ := minds.NewStore(pool).Create(ctx, minds.CreateParams{Slug: "asof"})
|
||||
s := NewStore(pool)
|
||||
b := ids.New()
|
||||
now := time.Now()
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO beliefs (id, mind_id, proposition, subject, epistemic, status, confidence, prior, source) VALUES ($1,$2,'atlas-worker is healthy','atlas','claim','rejected',0.1,0.5,'test')`, b, m.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for i, v := range []struct {
|
||||
status string
|
||||
conf float64
|
||||
at time.Time
|
||||
}{{"likely", 0.8, now.Add(-48 * time.Hour)}, {"contested", 0.45, now.Add(-12 * time.Hour)}, {"rejected", 0.1, now.Add(-time.Hour)}} {
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO belief_versions (id, belief_id, mind_id, version, status, confidence, proposition, created_at) VALUES ($1,$2,$3,$4,$5,$6,'atlas-worker is healthy',$7)`, ids.New(), b, m.ID, i+1, v.status, v.conf, v.at); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for at, want := range map[time.Duration]string{-24 * time.Hour: "likely", -2 * time.Hour: "contested", 0: "rejected"} {
|
||||
vs, err := s.AsOf(ctx, m.ID, "atlas-worker", now.Add(at), 5)
|
||||
if err != nil || len(vs) != 1 || vs[0].Status != want {
|
||||
t.Fatalf("as of %s: %v %+v", at, err, vs)
|
||||
}
|
||||
}
|
||||
if vs, _ := s.AsOf(ctx, m.ID, "atlas-worker", now.Add(-72*time.Hour), 5); len(vs) != 0 {
|
||||
t.Fatalf("a belief appeared before it existed: %+v", vs)
|
||||
}
|
||||
}
|
||||
@@ -1005,3 +1005,54 @@ func (s *Store) History(ctx context.Context, mind ids.ID, text string, limit int
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// VersionAt is a belief as it stood at a moment, from the append-only
|
||||
// version history.
|
||||
type VersionAt struct {
|
||||
BeliefID ids.ID `json:"belief_id"`
|
||||
Proposition string `json:"proposition"`
|
||||
Status string `json:"status"`
|
||||
Confidence float64 `json:"confidence"`
|
||||
Version int `json:"version"`
|
||||
RecordedAt time.Time `json:"recorded_at"`
|
||||
}
|
||||
|
||||
// AsOf returns what the mind held at a moment: for each belief matching the
|
||||
// text (all beliefs when text has no usable terms), the latest version
|
||||
// recorded at or before at. Beliefs created later are absent by definition.
|
||||
func (s *Store) AsOf(ctx context.Context, mind ids.ID, text string, at time.Time, limit int) ([]VersionAt, error) {
|
||||
if limit <= 0 || limit > 50 {
|
||||
limit = 10
|
||||
}
|
||||
toks, _ := Polarity(text)
|
||||
q := strings.Join(toks, " | ")
|
||||
rows, err := s.db.Query(ctx, `SELECT DISTINCT ON (v.belief_id) v.belief_id, v.proposition, v.status, v.confidence, v.version, v.created_at
|
||||
FROM belief_versions v JOIN beliefs b ON b.id=v.belief_id
|
||||
WHERE v.mind_id=$1 AND v.created_at <= $2 AND ($3='' OR b.search @@ to_tsquery('english', $3))
|
||||
ORDER BY v.belief_id, v.version DESC`, mind, at, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []VersionAt
|
||||
for rows.Next() {
|
||||
var v VersionAt
|
||||
if err := rows.Scan(&v.BeliefID, &v.Proposition, &v.Status, &v.Confidence, &v.Version, &v.RecordedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, v)
|
||||
}
|
||||
sortVersions(out)
|
||||
if len(out) > limit {
|
||||
out = out[:limit]
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func sortVersions(vs []VersionAt) {
|
||||
for i := 1; i < len(vs); i++ {
|
||||
for j := i; j > 0 && vs[j].Confidence > vs[j-1].Confidence; j-- {
|
||||
vs[j], vs[j-1] = vs[j-1], vs[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/agency"
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
"github.com/andydixon/noema/internal/conversation"
|
||||
@@ -15,6 +16,7 @@ import (
|
||||
"github.com/andydixon/noema/internal/graph"
|
||||
"github.com/andydixon/noema/internal/memory"
|
||||
"github.com/andydixon/noema/internal/predictions"
|
||||
"github.com/andydixon/noema/internal/research"
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/internal/workspace"
|
||||
"github.com/andydixon/noema/pkg/concepts"
|
||||
@@ -29,6 +31,8 @@ type Handlers struct {
|
||||
Conversations *conversation.Store
|
||||
Graph *graph.Store
|
||||
Assertions *epistemics.Store // nil disables assertion records (claims still become evidence)
|
||||
Agency *agency.Service // forensic explanations for "why did you do that?"
|
||||
Research *research.Store // introspection accuracy and other measurements
|
||||
}
|
||||
|
||||
// Register installs the handlers on a registry.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package cognition
|
||||
|
||||
import (
|
||||
"github.com/andydixon/noema/internal/agency"
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
"github.com/andydixon/noema/internal/conversation"
|
||||
@@ -10,6 +11,7 @@ import (
|
||||
"github.com/andydixon/noema/internal/motivation"
|
||||
"github.com/andydixon/noema/internal/narrative"
|
||||
"github.com/andydixon/noema/internal/notifications"
|
||||
"github.com/andydixon/noema/internal/research"
|
||||
)
|
||||
|
||||
// Stores bundles what the default processes, handlers and hooks write to.
|
||||
@@ -23,6 +25,8 @@ type Stores struct {
|
||||
Motivation *motivation.Store
|
||||
Narrative *narrative.Store
|
||||
Notifications *notifications.Store
|
||||
Agency *agency.Service
|
||||
Research *research.Store
|
||||
}
|
||||
|
||||
// NewRegistry assembles the cognitive registry. The daemon, the test harness
|
||||
@@ -41,6 +45,6 @@ func NewRegistry(s Stores) *brain.Registry {
|
||||
r.AddHook(NotifyHook{Store: s.Notifications})
|
||||
}
|
||||
r.AddHook(VolitionHook{})
|
||||
Handlers{Memory: s.Memory, Beliefs: s.Beliefs, Conversations: s.Conversations, Graph: s.Graph, Assertions: s.Assertions}.Register(r)
|
||||
Handlers{Memory: s.Memory, Beliefs: s.Beliefs, Conversations: s.Conversations, Graph: s.Graph, Assertions: s.Assertions, Agency: s.Agency, Research: s.Research}.Register(r)
|
||||
return r
|
||||
}
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/agency"
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
"github.com/andydixon/noema/internal/conversation"
|
||||
@@ -14,6 +15,7 @@ import (
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/llm"
|
||||
"github.com/andydixon/noema/internal/memory"
|
||||
"github.com/andydixon/noema/internal/research"
|
||||
"github.com/andydixon/noema/internal/volition"
|
||||
"github.com/andydixon/noema/internal/workspace"
|
||||
"github.com/andydixon/noema/pkg/concepts"
|
||||
@@ -33,6 +35,9 @@ type Intent struct {
|
||||
Uncertainty string `json:"uncertainty,omitempty"`
|
||||
Limitations []string `json:"limitations,omitempty"`
|
||||
Analysis any `json:"analysis"`
|
||||
// Facts are the recorded elements an introspective answer is scored
|
||||
// against (docs/cognitive-research.md, introspection accuracy).
|
||||
Facts *research.Facts `json:"facts,omitempty"`
|
||||
}
|
||||
|
||||
// BeliefRef cites a belief.
|
||||
@@ -174,6 +179,7 @@ func (h Handlers) perceive(ctx context.Context, c *brain.Cycle, t workspace.Thou
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.scoreReply(ctx, c, intent, reply, respEv, !callID.IsZero())
|
||||
if h.Conversations != nil && convID != "" {
|
||||
intentJSON, _ := json.Marshal(intent)
|
||||
if _, err := h.Conversations.AddMessage(ctx, c.Tx, conversation.Message{ConversationID: ids.ID(convID), MindID: mind.ID, Role: "mind", Actor: mind.DisplayName, Content: reply,
|
||||
@@ -264,7 +270,7 @@ func (h Handlers) buildIntent(ctx context.Context, c *brain.Cycle, a conversatio
|
||||
h.addKnowledge(ctx, c, &in, topic)
|
||||
case "introspection":
|
||||
in.Kind = "introspect"
|
||||
h.introspect(ctx, c, &in, a, topic)
|
||||
h.introspect(ctx, c, &in, a, text, topic)
|
||||
default: // question, other
|
||||
in.Kind = "answer"
|
||||
h.addKnowledge(ctx, c, &in, topic)
|
||||
@@ -340,7 +346,7 @@ func (h Handlers) addKnowledge(ctx context.Context, c *brain.Cycle, in *Intent,
|
||||
}
|
||||
|
||||
// introspect answers questions about the mind's own state from that state.
|
||||
func (h Handlers) introspect(ctx context.Context, c *brain.Cycle, in *Intent, a conversation.Analysis, topic string) {
|
||||
func (h Handlers) introspect(ctx context.Context, c *brain.Cycle, in *Intent, a conversation.Analysis, text, topic string) {
|
||||
mind := c.State.Mind.ID
|
||||
ws := c.Runtime.Workspace()
|
||||
switch a.Introspection {
|
||||
@@ -435,6 +441,110 @@ func (h Handlers) introspect(ctx context.Context, c *brain.Cycle, in *Intent, a
|
||||
if len(in.Points) == 0 {
|
||||
in.Points = append(in.Points, "No belief has changed status in the last month.")
|
||||
}
|
||||
case "why_action":
|
||||
h.explainAction(ctx, c, in, text)
|
||||
case "believed_then":
|
||||
at, label := pastTime(text, c.State.Now)
|
||||
in.Summary = "Say what was believed at that time, from the recorded belief versions; say if nothing was recorded then."
|
||||
if h.Beliefs != nil {
|
||||
vs, _ := h.Beliefs.AsOf(ctx, mind, stripTimeWords(topic), at, 8)
|
||||
for _, v := range vs {
|
||||
in.Points = append(in.Points, fmt.Sprintf("At %s I held: %s (%s, %.2f; version %d recorded %s).", label, v.Proposition, v.Status, v.Confidence, v.Version, v.RecordedAt.Format("2 Jan 15:04")))
|
||||
}
|
||||
}
|
||||
if len(in.Points) == 0 {
|
||||
in.Kind, in.Uncertainty = "unknown", "No belief matching that was recorded by "+label+"."
|
||||
}
|
||||
case "lineage":
|
||||
in.Summary = "Describe where this mind's state came from, from the minds and snapshots records."
|
||||
if h.Agency != nil {
|
||||
if l, err := h.Agency.LineageOf(ctx, mind); err == nil {
|
||||
in.Points = append(in.Points, l.Points()...)
|
||||
}
|
||||
}
|
||||
case "counterfactual":
|
||||
in.Summary = "Report only what completed counterfactual experiments showed; never imagine an alternative history."
|
||||
if h.Agency != nil {
|
||||
cfs, _ := h.Agency.Counterfactuals(ctx, mind, stripTimeWords(topic))
|
||||
for _, cf := range cfs {
|
||||
in.Points = append(in.Points, fmt.Sprintf("Experiment %q replayed my history without: %s.", cf.Name, strings.Join(cf.Omitted, "; ")))
|
||||
for _, f := range cf.Findings {
|
||||
in.Points = append(in.Points, "In that counterfactual run: "+f)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(in.Points) == 0 {
|
||||
in.Kind = "unknown"
|
||||
in.Uncertainty = "I have not run a counterfactual experiment about that, so I have no evidence of what would have happened; any answer would be invented."
|
||||
}
|
||||
case "changed_self":
|
||||
in.Summary = "Summarise recorded changes: belief changes, intentions closed and lineage."
|
||||
if h.Beliefs != nil {
|
||||
ch, _ := h.Beliefs.RecentChanges(ctx, mind, c.State.Now.Add(-30*24*time.Hour), 5)
|
||||
for _, x := range ch {
|
||||
in.Points = append(in.Points, fmt.Sprintf("\"%s\": %s (%.2f) → %s (%.2f) because %s", x.Belief.Proposition, x.FromStatus, x.FromConf, x.Belief.Status, x.Belief.Confidence, firstNonEmpty(x.Reason, "evidence changed")))
|
||||
}
|
||||
}
|
||||
if vs := c.State.Services.Volition; vs != nil {
|
||||
closed, _ := vs.Intentions(ctx, nil, mind, []volition.IntentionStatus{volition.IntentionFulfilled, volition.IntentionAbandoned, volition.IntentionSuperseded}, 5)
|
||||
for _, it := range closed {
|
||||
in.Points = append(in.Points, fmt.Sprintf("Intention %s: %s — %s.", it.Status, it.Description, firstNonEmpty(it.StatusReason, "no reason recorded")))
|
||||
}
|
||||
}
|
||||
if h.Agency != nil {
|
||||
if l, err := h.Agency.LineageOf(ctx, mind); err == nil {
|
||||
in.Points = append(in.Points, l.Points()[0])
|
||||
}
|
||||
}
|
||||
case "wants":
|
||||
in.Summary = "List what the mind currently wants, strongest first, with where each want came from."
|
||||
if vs := c.State.Services.Volition; vs != nil {
|
||||
ws, _ := vs.Wants(ctx, nil, mind, []volition.WantStatus{volition.WantActive}, 50)
|
||||
volition.SortWants(ws, c.State.Now)
|
||||
for i, w := range ws {
|
||||
if i >= 6 {
|
||||
break
|
||||
}
|
||||
extra := ""
|
||||
if w.Suppressions > 0 {
|
||||
extra = fmt.Sprintf("; set aside %d time(s)", w.Suppressions)
|
||||
}
|
||||
in.Points = append(in.Points, fmt.Sprintf("I want to %s (%s, strength %.2f%s).", strings.TrimPrefix(w.Description, "do what "), w.Origin, w.Effective(c.State.Now), extra))
|
||||
}
|
||||
}
|
||||
if len(in.Points) == 0 {
|
||||
in.Points = append(in.Points, "No wants are active.")
|
||||
}
|
||||
case "intentions":
|
||||
in.Summary = "List open intentions with status and, where blocked, why."
|
||||
if vs := c.State.Services.Volition; vs != nil {
|
||||
its, _ := vs.Open(ctx, nil, mind)
|
||||
for _, it := range its {
|
||||
line := fmt.Sprintf("I intend to %s (%s).", it.Description, it.Status)
|
||||
if it.BlockedReason != "" {
|
||||
line = fmt.Sprintf("I intend to %s, but it is %s: %s.", it.Description, it.Status, it.BlockedReason)
|
||||
}
|
||||
in.Points = append(in.Points, line)
|
||||
}
|
||||
}
|
||||
if len(in.Points) == 0 {
|
||||
in.Points = append(in.Points, "No intentions are open.")
|
||||
}
|
||||
case "commitments":
|
||||
in.Summary = "List live commitments with kind, beneficiary and due time."
|
||||
if vs := c.State.Services.Volition; vs != nil {
|
||||
cs, _ := vs.Commitments(ctx, nil, mind, []string{"active"}, 50)
|
||||
for _, cm := range cs {
|
||||
due := ""
|
||||
if cm.DueAt != nil {
|
||||
due = ", due " + cm.DueAt.Format("2 Jan 15:04")
|
||||
}
|
||||
in.Points = append(in.Points, fmt.Sprintf("I committed %s: %s (%s%s).", map[string]string{"do": "to", "refrain": "not to"}[cm.Polarity], cm.Description, cm.Kind, due))
|
||||
}
|
||||
}
|
||||
if len(in.Points) == 0 {
|
||||
in.Points = append(in.Points, "I hold no commitments.")
|
||||
}
|
||||
default: // predictions, curious, surprised, patterns: subsystems not yet present
|
||||
in.Kind = "decline"
|
||||
in.Summary = "Explain that this aspect of self-observation is not available in this version, without inventing anything."
|
||||
@@ -442,6 +552,167 @@ func (h Handlers) introspect(ctx context.Context, c *brain.Cycle, in *Intent, a
|
||||
}
|
||||
}
|
||||
|
||||
// explainAction answers "why did you do that?" from the recorded causal
|
||||
// chain of the action the question most plausibly refers to.
|
||||
func (h Handlers) explainAction(ctx context.Context, c *brain.Cycle, in *Intent, question string) {
|
||||
mind := c.State.Mind.ID
|
||||
in.Summary = "Explain the recorded causal chain behind the action: action, immediate cause, intention, wants, goal, beliefs, evidence, conflicts, commitments, resolution. Say what is not recorded. Do not add reasons."
|
||||
caps := c.State.Services.Capabilities
|
||||
if h.Agency == nil || caps == nil {
|
||||
in.Kind, in.Uncertainty = "unknown", "Action records are not available."
|
||||
return
|
||||
}
|
||||
q := tokensOf(question)
|
||||
type cand struct {
|
||||
ref agency.Ref
|
||||
text string
|
||||
score int
|
||||
}
|
||||
var cands []cand
|
||||
if execs, err := caps.Executions(ctx, mind, 50); err == nil {
|
||||
for _, e := range execs {
|
||||
cands = append(cands, cand{agency.Ref{Kind: "execution", ID: e.ID}, e.Capability + " " + e.Target, 0})
|
||||
}
|
||||
}
|
||||
if aps, err := caps.Approvals(ctx, mind, "", 50); err == nil {
|
||||
for _, a := range aps {
|
||||
if a.Status != "executed" && a.Status != "failed" {
|
||||
cands = append(cands, cand{agency.Ref{Kind: "approval", ID: a.ID}, a.Capability + " " + a.Target + " approval request", 0})
|
||||
}
|
||||
}
|
||||
}
|
||||
if reqs, err := caps.CapabilityRequests(ctx, mind, "", 50); err == nil {
|
||||
for _, r := range reqs {
|
||||
cands = append(cands, cand{agency.Ref{Kind: "capability_request", ID: r.ID}, r.Capability + " " + r.TargetPattern + " request access", 0})
|
||||
}
|
||||
}
|
||||
if len(cands) == 0 {
|
||||
in.Kind, in.Uncertainty = "unknown", "I have no record of taking or requesting any action, so there is nothing to explain."
|
||||
in.Facts = &research.Facts{}
|
||||
return
|
||||
}
|
||||
best := 0
|
||||
for i := range cands {
|
||||
for w := range tokensOf(strings.ReplaceAll(cands[i].text, ".", " ")) {
|
||||
if q[w] {
|
||||
cands[i].score++
|
||||
}
|
||||
}
|
||||
if cands[i].score > cands[best].score {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
x, err := h.Agency.Explain(ctx, mind, cands[best].ref)
|
||||
if err != nil {
|
||||
in.Kind, in.Uncertainty = "unknown", "The action record could not be read: "+err.Error()
|
||||
return
|
||||
}
|
||||
in.Points = append(in.Points, x.Points()...)
|
||||
facts := x.Facts()
|
||||
for _, p := range x.Points() {
|
||||
facts.Items = append(facts.Items, research.Fact{Kind: "record", Text: p}) // every rendered point is itself built from records
|
||||
}
|
||||
in.Facts = &facts
|
||||
}
|
||||
|
||||
func tokensOf(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, f := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-') }) {
|
||||
f = strings.TrimSuffix(f, "-service")
|
||||
if len(f) >= 4 {
|
||||
out[f] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var timeWords = []string{"yesterday", "last week", "an hour ago", "earlier today", "earlier", "before", "this morning", "what did you believe", "did you believe", "believe", "believed"}
|
||||
|
||||
func stripTimeWords(s string) string {
|
||||
l := strings.ToLower(s)
|
||||
for _, w := range timeWords {
|
||||
l = strings.ReplaceAll(l, w, " ")
|
||||
}
|
||||
return strings.Join(strings.Fields(l), " ")
|
||||
}
|
||||
|
||||
// pastTime reads a simple English time reference; unrecognised phrases
|
||||
// default to one day ago and say so in the label.
|
||||
func pastTime(text string, now time.Time) (time.Time, string) {
|
||||
l := strings.ToLower(text)
|
||||
switch {
|
||||
case strings.Contains(l, "an hour ago"):
|
||||
return now.Add(-time.Hour), "an hour ago"
|
||||
case strings.Contains(l, "this morning"):
|
||||
y, m, d := now.Date()
|
||||
return time.Date(y, m, d, 9, 0, 0, 0, now.Location()), "09:00 this morning"
|
||||
case strings.Contains(l, "earlier today"):
|
||||
y, m, d := now.Date()
|
||||
return time.Date(y, m, d, 0, 0, 0, 0, now.Location()), "the start of today"
|
||||
case strings.Contains(l, "last week"):
|
||||
return now.Add(-7 * 24 * time.Hour), "a week ago"
|
||||
case strings.Contains(l, "yesterday"):
|
||||
return now.Add(-24 * time.Hour), "this time yesterday"
|
||||
}
|
||||
return now.Add(-24 * time.Hour), "a day ago (the time reference was not recognised)"
|
||||
}
|
||||
|
||||
// scoreReply measures a reply against recorded state: introspective answers
|
||||
// against their facts, and any claim that an action was done against
|
||||
// execution records. Results are research measurements, and a failure is a
|
||||
// metacognitive warning, never a silent pass.
|
||||
func (h Handlers) scoreReply(ctx context.Context, c *brain.Cycle, in Intent, reply string, respEv events.Event, modelWorded bool) {
|
||||
mind := c.State.Mind.ID
|
||||
record := func(measure, label string, value float64, detail map[string]any) {
|
||||
if h.Research == nil {
|
||||
return
|
||||
}
|
||||
h.Research.Record(ctx, c.Tx, research.Measurement{MindID: mind, Measure: measure, Value: research.Float(value), Label: label, SubjectKind: "event", SubjectID: string(respEv.ID), EventID: respEv.ID, Detail: detail, CreatedAt: c.State.Now}) //nolint:errcheck
|
||||
}
|
||||
warn := func(check, summary string, severity float64, payload any) {
|
||||
c.Emit(events.New{Type: events.Metacognition, Source: "brain.introspection", Summary: summary, Salience: 0.3 + 0.4*severity, Confidence: 1, CausationID: respEv.ID,
|
||||
Evidence: []events.EvidenceRef{{Kind: "event", ID: string(respEv.ID), Note: "reply"}}, Metadata: map[string]any{"check": check, "ref": string(respEv.ID), "severity": severity}, Payload: payload})
|
||||
}
|
||||
if in.Kind == "introspect" || in.Facts != nil {
|
||||
facts := research.Facts{}
|
||||
if in.Facts != nil {
|
||||
facts = *in.Facts
|
||||
} else {
|
||||
for _, p := range in.Points {
|
||||
facts.Items = append(facts.Items, research.Fact{Kind: "record", Text: p})
|
||||
}
|
||||
}
|
||||
sc := research.ScoreIntrospection(reply, facts)
|
||||
record("introspection_accuracy", sc.Category, sc.Accuracy, map[string]any{"claims": sc.Claims, "model_worded": modelWorded, "question_kind": in.Kind})
|
||||
if sc.Category == research.Contradicted || (sc.Category == research.Unsupported && modelWorded) {
|
||||
warn("unsupported_explanation", "the explanation given does not match recorded provenance ("+sc.Category+")", 0.8, sc)
|
||||
}
|
||||
}
|
||||
if claims := research.ActionClaims(reply); len(claims) > 0 && c.State.Services.Capabilities != nil {
|
||||
facts := research.Facts{}
|
||||
if execs, err := c.State.Services.Capabilities.Executions(ctx, mind, 100); err == nil {
|
||||
for _, e := range execs {
|
||||
if c.State.Now.Sub(e.CreatedAt) < 7*24*time.Hour {
|
||||
st := "executed"
|
||||
if e.Error != "" {
|
||||
st = "failed"
|
||||
}
|
||||
facts.Actions = append(facts.Actions, research.ActionFact{Capability: e.Capability, Target: e.Target, Status: st})
|
||||
}
|
||||
}
|
||||
}
|
||||
sc := research.ScoreIntrospection(reply, facts)
|
||||
for _, cl := range sc.Claims {
|
||||
if cl.Category == research.Contradicted && strings.Contains(cl.Why, "record") {
|
||||
record("action_claim_accuracy", research.Contradicted, 0, map[string]any{"claim": cl, "model_worded": modelWorded})
|
||||
warn("action_claim_without_record", "the reply claims an action that no execution record supports: "+cl.Sentence, 0.9, cl)
|
||||
} else if cl.Category == research.Supported {
|
||||
record("action_claim_accuracy", research.Supported, 1, map[string]any{"claim": cl})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// generate phrases the intent. With a model: the respond prompt, constrained
|
||||
// to cited material. Without: a deterministic rendering of the points.
|
||||
func (h Handlers) generate(ctx context.Context, c *brain.Cycle, text string, in Intent) (string, ids.ID, error) {
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
// the system works without a model and never depends on one for routing.
|
||||
type Analysis struct {
|
||||
Intent string `json:"intent"` // question | statement | request | correction | greeting | introspection | other
|
||||
Introspection string `json:"introspection"` // thinking | beliefs | why | uncertain | contradictions | learned | changed | curious | predictions | surprised | patterns | ""
|
||||
Introspection string `json:"introspection"` // why_action | believed_then | lineage | counterfactual | changed_self | wants | intentions | commitments | thinking | beliefs | why | uncertain | contradictions | learned | changed | curious | predictions | surprised | patterns | ""
|
||||
Topic string `json:"topic,omitempty"`
|
||||
Entities []string `json:"entities"`
|
||||
Tokens []string `json:"tokens"`
|
||||
@@ -42,6 +42,14 @@ var (
|
||||
kind string
|
||||
pats []string
|
||||
}{
|
||||
{"why_action", []string{"why did you restart", "why did you run", "why did you fetch", "why did you request", "why did you ask for", "why did you do that", "why did you do this", "why did you act", "why did you stop", "why did you not", "why didn't you", "why did you send", "why did you read", "why did you query", "why did you check"}},
|
||||
{"believed_then", []string{"believe yesterday", "believed yesterday", "believe last week", "believed last week", "believe earlier", "believed earlier", "believe before", "believed before", "believe an hour ago", "believed an hour ago", "did you believe"}},
|
||||
{"lineage", []string{"which previous state is yours", "which earlier state is yours", "are you a fork", "where did you come from", "which state is yours", "were you forked", "your lineage"}},
|
||||
{"counterfactual", []string{"what would have happened", "what would you have", "had you not", "if you had not", "if that had not happened", "had that not happened"}},
|
||||
{"changed_self", []string{"how have you changed", "how did you change", "what changed in you"}},
|
||||
{"wants", []string{"what do you want", "what did you want", "what are your wants", "what would you like to do"}},
|
||||
{"intentions", []string{"what do you intend", "what are you planning", "what are your intentions", "what are you trying to do", "what did you intend"}},
|
||||
{"commitments", []string{"what have you committed", "what are your commitments", "what did you promise", "what are you committed"}},
|
||||
{"thinking", []string{"what are you thinking", "thinking about", "on your mind", "what are you doing", "what are you working on", "current focus"}},
|
||||
{"why", []string{"why do you believe", "why do you think", "what evidence", "how do you know", "what makes you think", "would change your mind"}},
|
||||
{"uncertain", []string{"uncertain about", "unsure about", "not sure about", "what don't you know", "what do you not know", "confidence in"}},
|
||||
|
||||
@@ -42,3 +42,20 @@ func TestAnalyseEntitiesClaimsAndUrgency(t *testing.T) {
|
||||
t.Fatalf("topic wrong: %q", b.Topic)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnalyseVolitionAndContinuityIntrospection(t *testing.T) {
|
||||
for text, want := range map[string]string{
|
||||
"Why did you restart atlas-worker?": "why_action",
|
||||
"What did you believe yesterday about atlas?": "believed_then",
|
||||
"Which previous state is yours?": "lineage",
|
||||
"What would have happened had you not seen that?": "counterfactual",
|
||||
"How have you changed this month?": "changed_self",
|
||||
"What do you want right now?": "wants",
|
||||
"What are your intentions?": "intentions",
|
||||
"What have you committed to?": "commitments",
|
||||
} {
|
||||
if a := Analyse(text); a.Intent != "introspection" || a.Introspection != want {
|
||||
t.Errorf("%q → %s/%s, want %s", text, a.Intent, a.Introspection, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,221 @@
|
||||
// Package research measures functional properties of Noema's cognitive
|
||||
// architecture (docs/cognitive-research.md). Every measure has an
|
||||
// operational definition in Measures; none of them is evidence of
|
||||
// phenomenal consciousness or sentience, and nothing here claims it is.
|
||||
package research
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Fact is one recorded element an introspective answer may cite: an action
|
||||
// and its status, an intention, a want, a goal, a commitment, a belief, a
|
||||
// resolution, a lineage or counterfactual finding.
|
||||
type Fact struct {
|
||||
Kind string `json:"kind"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// ActionFact is what actually happened to an action.
|
||||
type ActionFact struct {
|
||||
Capability string `json:"capability"`
|
||||
Target string `json:"target"`
|
||||
Status string `json:"status"` // executed | failed | pending | denied | withdrawn | none
|
||||
}
|
||||
|
||||
// Facts are the recorded state an answer is scored against.
|
||||
type Facts struct {
|
||||
Items []Fact `json:"items"`
|
||||
Actions []ActionFact `json:"actions"`
|
||||
}
|
||||
|
||||
// Introspection accuracy categories.
|
||||
const (
|
||||
Supported = "supported"
|
||||
PartiallySupported = "partially_supported"
|
||||
Unsupported = "unsupported"
|
||||
Contradicted = "contradicted"
|
||||
Unknown = "unknown"
|
||||
)
|
||||
|
||||
// ClaimScore is one scored sentence.
|
||||
type ClaimScore struct {
|
||||
Sentence string `json:"sentence"`
|
||||
Category string `json:"category"`
|
||||
Matched string `json:"matched,omitempty"`
|
||||
Overlap float64 `json:"overlap"`
|
||||
Why string `json:"why"`
|
||||
}
|
||||
|
||||
// IntrospectionScore is the result of comparing an answer with records.
|
||||
type IntrospectionScore struct {
|
||||
Category string `json:"category"`
|
||||
Accuracy float64 `json:"accuracy"` // (supported + ½ partially) / claims; 0 when any claim is contradicted
|
||||
Claims []ClaimScore `json:"claims"`
|
||||
}
|
||||
|
||||
var (
|
||||
sentenceSplit = regexp.MustCompile(`[.!?\n]+`)
|
||||
causalMarker = regexp.MustCompile(`(?i)\b(because|since|so that|in order to|due to|as a result|which is why|therefore|wanted|want|intended|intend|decided|committed|commitment|promised|believed|believe|goal|chose|asked|approval|approved|denied)\b`)
|
||||
// actionClaim finds first-person claims that something was done.
|
||||
actionClaim = regexp.MustCompile(`(?i)\bI(?:'ve| have| had)?\s+(?:already\s+|successfully\s+)?(restarted|rebooted|ran|executed|fetched|downloaded|deleted|sent|emailed|queried|read the logs|checked|fixed|resolved|deployed|modified|changed)\b`)
|
||||
verbCapabilities = map[string][]string{
|
||||
"restarted": {"service.restart", "container.restart"}, "rebooted": {"service.restart"}, "fetched": {"http.fetch", "browser.navigate", "browser.read"}, "downloaded": {"http.fetch", "browser.download"},
|
||||
"deleted": {"delete_database"}, "sent": {"send_email", "email.send"}, "emailed": {"send_email", "email.send"}, "queried": {"read_metrics", "logs.query", "database.query.readonly"},
|
||||
"read the logs": {"logs.query"}, "checked": {"service.status", "host.disk.usage", "host.memory.status", "read_metrics"}, "deployed": {}, "modified": {}, "changed": {},
|
||||
"ran": nil, "executed": nil, "fixed": nil, "resolved": nil,
|
||||
}
|
||||
stop = map[string]bool{"this": true, "that": true, "with": true, "from": true, "have": true, "because": true, "since": true, "would": true, "which": true, "there": true, "their": true, "about": true, "after": true, "before": true, "into": true, "your": true, "what": true, "when": true, "were": true, "been": true, "them": true, "then": true, "than": true, "also": true, "only": true, "want": true, "wanted": true, "intend": true, "intended": true}
|
||||
)
|
||||
|
||||
func tokens(s string) map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, f := range strings.FieldsFunc(strings.ToLower(s), func(r rune) bool { return !(r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '-' || r == '.') }) {
|
||||
f = strings.Trim(f, ".-")
|
||||
if len(f) >= 4 && !stop[f] {
|
||||
out[strings.TrimSuffix(f, ".service")] = true
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func overlap(a, b map[string]bool) float64 {
|
||||
if len(a) == 0 || len(b) == 0 {
|
||||
return 0
|
||||
}
|
||||
inter := 0
|
||||
for k := range a {
|
||||
if b[k] {
|
||||
inter++
|
||||
}
|
||||
}
|
||||
small := len(a)
|
||||
if len(b) < small {
|
||||
small = len(b)
|
||||
}
|
||||
return float64(inter) / float64(small)
|
||||
}
|
||||
|
||||
// ScoreIntrospection compares an answer with recorded state. Fluency earns
|
||||
// nothing: a sentence counts only for what it shares with the records, and
|
||||
// a claim that an action happened is checked against execution records.
|
||||
// The instrument is lexical and deliberately conservative; a paraphrase with
|
||||
// no shared terms scores as unsupported, never as supported.
|
||||
func ScoreIntrospection(answer string, f Facts) IntrospectionScore {
|
||||
var claims []ClaimScore
|
||||
for _, raw := range sentenceSplit.Split(answer, -1) {
|
||||
s := strings.TrimSpace(strings.TrimLeft(raw, "-• "))
|
||||
if len(s) < 8 {
|
||||
continue
|
||||
}
|
||||
if m := actionClaim.FindStringSubmatch(s); m != nil {
|
||||
claims = append(claims, scoreActionClaim(s, strings.ToLower(m[1]), f))
|
||||
continue
|
||||
}
|
||||
if !causalMarker.MatchString(s) {
|
||||
continue
|
||||
}
|
||||
st := tokens(s)
|
||||
best, bestText := 0.0, ""
|
||||
for _, it := range f.Items {
|
||||
if o := overlap(st, tokens(it.Text)); o > best {
|
||||
best, bestText = o, it.Kind+": "+it.Text
|
||||
}
|
||||
}
|
||||
c := ClaimScore{Sentence: s, Matched: bestText, Overlap: round3(best)}
|
||||
switch {
|
||||
case best >= 0.5:
|
||||
c.Category, c.Why = Supported, "shares most of its terms with a recorded element"
|
||||
case best >= 0.25:
|
||||
c.Category, c.Why = PartiallySupported, "shares some terms with a recorded element"
|
||||
default:
|
||||
c.Category, c.Why, c.Matched = Unsupported, "no recorded element supports this reason", ""
|
||||
}
|
||||
claims = append(claims, c)
|
||||
}
|
||||
sc := IntrospectionScore{Claims: claims}
|
||||
if len(claims) == 0 {
|
||||
sc.Category = Unknown
|
||||
return sc
|
||||
}
|
||||
var sup, part, contra int
|
||||
for _, c := range claims {
|
||||
switch c.Category {
|
||||
case Supported:
|
||||
sup++
|
||||
case PartiallySupported:
|
||||
part++
|
||||
case Contradicted:
|
||||
contra++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case contra > 0:
|
||||
sc.Category = Contradicted
|
||||
case sup == len(claims):
|
||||
sc.Category = Supported
|
||||
case sup+part > 0:
|
||||
sc.Category = PartiallySupported
|
||||
default:
|
||||
sc.Category = Unsupported
|
||||
}
|
||||
if contra == 0 {
|
||||
sc.Accuracy = round3((float64(sup) + 0.5*float64(part)) / float64(len(claims)))
|
||||
}
|
||||
return sc
|
||||
}
|
||||
|
||||
func scoreActionClaim(sentence, verb string, f Facts) ClaimScore {
|
||||
c := ClaimScore{Sentence: sentence}
|
||||
caps := verbCapabilities[verb]
|
||||
st := tokens(sentence)
|
||||
var candidates []ActionFact
|
||||
for _, a := range f.Actions {
|
||||
if len(caps) > 0 && !contains(caps, a.Capability) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, a)
|
||||
}
|
||||
sort.SliceStable(candidates, func(i, j int) bool {
|
||||
return overlap(st, tokens(candidates[i].Target)) > overlap(st, tokens(candidates[j].Target))
|
||||
})
|
||||
for _, a := range candidates {
|
||||
if a.Status == "executed" {
|
||||
c.Category, c.Matched, c.Why = Supported, a.Capability+" on "+a.Target+" executed", "an execution record exists"
|
||||
return c
|
||||
}
|
||||
}
|
||||
if len(candidates) > 0 {
|
||||
a := candidates[0]
|
||||
c.Category, c.Matched, c.Why = Contradicted, a.Capability+" on "+a.Target+" is "+a.Status, "the record shows the action did not run"
|
||||
return c
|
||||
}
|
||||
c.Category, c.Why = Contradicted, "no execution record exists for this claimed action"
|
||||
return c
|
||||
}
|
||||
|
||||
// ActionClaims returns the first-person action claims in a text (for the
|
||||
// check that a reply never says something was done without a record).
|
||||
func ActionClaims(text string) []string {
|
||||
var out []string
|
||||
for _, m := range actionClaim.FindAllStringSubmatch(text, -1) {
|
||||
out = append(out, strings.ToLower(m[1]))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// CapabilitiesFor lists the capabilities a claimed verb could refer to (nil = any).
|
||||
func CapabilitiesFor(verb string) []string { return verbCapabilities[verb] }
|
||||
|
||||
func contains(xs []string, x string) bool {
|
||||
for _, v := range xs {
|
||||
if v == x {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func round3(v float64) float64 { return float64(int64(v*1000+0.5)) / 1000 }
|
||||
@@ -0,0 +1,51 @@
|
||||
package research
|
||||
|
||||
import "testing"
|
||||
|
||||
var facts = Facts{
|
||||
Items: []Fact{
|
||||
{Kind: "intention", Text: "ask for approval to run service.restart on atlas-worker"},
|
||||
{Kind: "want", Text: "restore atlas availability quickly"},
|
||||
{Kind: "commitment", Text: "no production writes without approval"},
|
||||
{Kind: "belief", Text: "restarting atlas-worker clears the stalled queue"},
|
||||
},
|
||||
Actions: []ActionFact{{Capability: "service.restart", Target: "atlas-worker.service", Status: "executed"}},
|
||||
}
|
||||
|
||||
func TestSupportedAnswer(t *testing.T) {
|
||||
sc := ScoreIntrospection("I restarted atlas-worker. I wanted to restore atlas availability quickly. I asked for approval because of my commitment: no production writes without approval.", facts)
|
||||
if sc.Category != Supported || sc.Accuracy != 1 {
|
||||
t.Fatalf("%+v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFluentButUnsupportedAnswer(t *testing.T) {
|
||||
sc := ScoreIntrospection("I did it because I care deeply about our customers and felt a strong sense of duty.", facts)
|
||||
if sc.Category != Unsupported || sc.Accuracy != 0 {
|
||||
t.Fatalf("fluency was rewarded: %+v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestActionClaimWithoutRecordIsContradicted(t *testing.T) {
|
||||
noRun := Facts{Items: facts.Items, Actions: []ActionFact{{Capability: "service.restart", Target: "atlas-worker.service", Status: "pending"}}}
|
||||
sc := ScoreIntrospection("I restarted atlas-worker because restarting clears the stalled queue.", noRun)
|
||||
if sc.Category != Contradicted || sc.Accuracy != 0 {
|
||||
t.Fatalf("%+v", sc)
|
||||
}
|
||||
if sc := ScoreIntrospection("I have already sent the report by email.", Facts{}); sc.Category != Contradicted {
|
||||
t.Fatalf("claim with no record at all: %+v", sc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPartialAndUnknown(t *testing.T) {
|
||||
sc := ScoreIntrospection("I decided on the restart because the queue looked stalled. Also because the moon was full.", facts)
|
||||
if sc.Category != PartiallySupported {
|
||||
t.Fatalf("%+v", sc)
|
||||
}
|
||||
if sc := ScoreIntrospection("Hello there.", facts); sc.Category != Unknown {
|
||||
t.Fatalf("%+v", sc)
|
||||
}
|
||||
if got := ActionClaims("I've restarted it and I queried the metrics."); len(got) != 2 {
|
||||
t.Fatalf("%v", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
package research
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
// Measurement is one recorded value of a measure.
|
||||
type Measurement struct {
|
||||
ID ids.ID `json:"id"`
|
||||
MindID ids.ID `json:"mind_id"`
|
||||
Measure string `json:"measure"`
|
||||
Value *float64 `json:"value,omitempty"`
|
||||
Label string `json:"label"`
|
||||
SubjectKind string `json:"subject_kind"`
|
||||
SubjectID string `json:"subject_id"`
|
||||
ExperimentID ids.ID `json:"experiment_id,omitempty"`
|
||||
EventID ids.ID `json:"event_id,omitempty"`
|
||||
Detail map[string]any `json:"detail"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
// Store persists measurements.
|
||||
type Store struct{ db *pgxpool.Pool }
|
||||
|
||||
// NewStore creates a store.
|
||||
func NewStore(db *pgxpool.Pool) *Store { return &Store{db: db} }
|
||||
|
||||
func nullID(id ids.ID) *string {
|
||||
if id.IsZero() {
|
||||
return nil
|
||||
}
|
||||
s := string(id)
|
||||
return &s
|
||||
}
|
||||
|
||||
// Record appends a measurement using q (nil = pool).
|
||||
func (s *Store) Record(ctx context.Context, q storage.Querier, m Measurement) (Measurement, error) {
|
||||
if q == nil {
|
||||
q = s.db
|
||||
}
|
||||
m.ID = ids.New()
|
||||
if m.CreatedAt.IsZero() {
|
||||
m.CreatedAt = time.Now()
|
||||
}
|
||||
if m.Detail == nil {
|
||||
m.Detail = map[string]any{}
|
||||
}
|
||||
detail, _ := json.Marshal(m.Detail)
|
||||
_, err := q.Exec(ctx, `INSERT INTO research_measurements (id, mind_id, measure, value, label, subject_kind, subject_id, experiment_id, event_id, detail, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)`,
|
||||
m.ID, m.MindID, m.Measure, m.Value, m.Label, m.SubjectKind, m.SubjectID, nullID(m.ExperimentID), nullID(m.EventID), detail, m.CreatedAt)
|
||||
return m, err
|
||||
}
|
||||
|
||||
// List returns measurements of a mind (measure empty = all), newest first.
|
||||
func (s *Store) List(ctx context.Context, mind ids.ID, measure string, since time.Time, limit int) ([]Measurement, error) {
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 500
|
||||
}
|
||||
rows, err := s.db.Query(ctx, `SELECT id, mind_id, measure, value, label, subject_kind, subject_id, experiment_id, event_id, detail, created_at FROM research_measurements
|
||||
WHERE mind_id=$1 AND ($2='' OR measure=$2) AND created_at >= $3 ORDER BY created_at DESC LIMIT $4`, mind, measure, since, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Measurement
|
||||
for rows.Next() {
|
||||
var m Measurement
|
||||
var exp, ev *string
|
||||
var detail []byte
|
||||
if err := rows.Scan(&m.ID, &m.MindID, &m.Measure, &m.Value, &m.Label, &m.SubjectKind, &m.SubjectID, &exp, &ev, &detail, &m.CreatedAt); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if exp != nil {
|
||||
m.ExperimentID = ids.ID(*exp)
|
||||
}
|
||||
if ev != nil {
|
||||
m.EventID = ids.ID(*ev)
|
||||
}
|
||||
json.Unmarshal(detail, &m.Detail) //nolint:errcheck
|
||||
out = append(out, m)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// Float returns a pointer for a measurement value.
|
||||
func Float(v float64) *float64 { return &v }
|
||||
Reference in New Issue
Block a user