Phase 15: snapshots, forks, export/import, cognitive replay, experiments
- snapshots: checksummed JSON archive of every mind-scoped state table plus identity/personality/config and the event position; stored, downloadable - fork: restore into a new mind with all ids remapped; event references are kept so history before the fork is shared by reference - export/import: same archive; import is hostile-input validated (size, allow-listed tables, row caps, column names, checksum, forced mind_id) - compare: deterministic counts, belief diffs by proposition, personality and config diffs, event mix - replay: Runtime.Step for synchronous drivers; sandbox fork driven one stimulus at a time under a forbidden capability context; recorded model responses via a replaying router; steps labelled exact / approximate / diverged with recorded-vs-replayed digests - experiments: definition (overrides, stimuli, expectations, seed, duration), background run in a disabled subject mind, stored and exportable results - migration 0015, API, UI (Snapshots and Replay tabs, Experiments section, compare page), noemactl snapshots/compare/replay/experiments, docs
This commit is contained in:
Andy Dixon
@@ -0,0 +1,159 @@
|
||||
// Package replay re-runs recorded cognition (spec §41) and drives sandbox
|
||||
// minds for experiments (spec §57). A Driver owns a brain runtime that is
|
||||
// never started as a loop: the driver is the clock, feeding one stimulus at
|
||||
// a time and stepping cycles until the mind settles. Every step runs under a
|
||||
// forbidden capability context, so replays and experiments cannot act.
|
||||
package replay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
"github.com/andydixon/noema/internal/capabilities"
|
||||
"github.com/andydixon/noema/internal/minds"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
// Driver steps a sandbox mind synchronously.
|
||||
type Driver struct {
|
||||
rt *brain.Runtime
|
||||
cfg brain.Config
|
||||
mind minds.Mind
|
||||
why string
|
||||
}
|
||||
|
||||
// NewDriver builds a runtime for the sandbox mind. cfg.Services.LLM should
|
||||
// be a replaying router when recorded responses are wanted.
|
||||
func NewDriver(ctx context.Context, cfg brain.Config, m minds.Mind, why string) (*Driver, error) {
|
||||
rt, err := brain.NewRuntime(ctx, cfg, m)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Driver{rt: rt, cfg: cfg, mind: m, why: why}, nil
|
||||
}
|
||||
|
||||
// MaxStepsPerStimulus bounds how long the driver waits for a mind to settle.
|
||||
const MaxStepsPerStimulus = 8
|
||||
|
||||
// bookkeeping event types are produced by every cycle and carry no cognitive content for comparison.
|
||||
var bookkeeping = map[events.Type]bool{events.CycleCompleted: true, events.WorkspaceBroadcast: true, events.AttentionShift: true, events.ModeChanged: true,
|
||||
events.LLMRequest: true, events.LLMResponse: true, events.PolicyDecision: true, events.ThoughtCandidate: true}
|
||||
|
||||
// Settle runs cycles until one produces nothing but bookkeeping, or the cap
|
||||
// is reached, and returns every event committed.
|
||||
func (d *Driver) Settle(ctx context.Context) ([]events.Event, int, error) {
|
||||
ctx = capabilities.WithForbidden(ctx, d.why)
|
||||
var all []events.Event
|
||||
steps := 0
|
||||
for steps < MaxStepsPerStimulus {
|
||||
// Let the just-committed events pass the runtime's settle window.
|
||||
select {
|
||||
case <-time.After(brain.SettleWindow):
|
||||
case <-ctx.Done():
|
||||
return all, steps, ctx.Err()
|
||||
}
|
||||
out, err := d.rt.Step(ctx, d.why)
|
||||
if err != nil {
|
||||
return all, steps, err
|
||||
}
|
||||
steps++
|
||||
all = append(all, out...)
|
||||
if !hasContent(out) {
|
||||
break
|
||||
}
|
||||
}
|
||||
return all, steps, nil
|
||||
}
|
||||
|
||||
func hasContent(evs []events.Event) bool {
|
||||
for _, e := range evs {
|
||||
if !bookkeeping[e.Type] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Inject appends a stimulus copy to the sandbox mind and returns it.
|
||||
func (d *Driver) Inject(ctx context.Context, src events.Event, tag map[string]any) (events.Event, error) {
|
||||
meta := map[string]any{}
|
||||
for k, v := range src.Metadata {
|
||||
meta[k] = v
|
||||
}
|
||||
for k, v := range tag {
|
||||
meta[k] = v
|
||||
}
|
||||
var payload any = src.Payload
|
||||
out, err := d.cfg.Events.Append(ctx, events.New{MindID: d.mind.ID, Type: src.Type, Source: src.Source, OccurredAt: time.Now(), Salience: src.Salience, Confidence: src.Confidence,
|
||||
Valence: src.Valence, Arousal: src.Arousal, Evidence: src.Evidence, Summary: src.Summary, Metadata: meta, Payload: payload})
|
||||
if err != nil {
|
||||
return events.Event{}, err
|
||||
}
|
||||
return out[0], nil
|
||||
}
|
||||
|
||||
// Observe appends a fresh observation stimulus (experiments).
|
||||
func (d *Driver) Observe(ctx context.Context, source, text string, salience float64, meta map[string]any) (events.Event, error) {
|
||||
out, err := d.cfg.Events.Append(ctx, events.New{MindID: d.mind.ID, Type: events.Observation, Source: source, OccurredAt: time.Now(), Salience: salience, Confidence: 0.9, Summary: text, Metadata: meta,
|
||||
Payload: events.ObservationPayload{Text: text, ContentType: "text/plain", Channel: source, Trust: events.TrustOperator}})
|
||||
if err != nil {
|
||||
return events.Event{}, err
|
||||
}
|
||||
return out[0], nil
|
||||
}
|
||||
|
||||
// Mind returns the sandbox mind.
|
||||
func (d *Driver) Mind() minds.Mind { return d.mind }
|
||||
|
||||
// Digest is the comparable content of a list of events: type + normalised
|
||||
// summary with ids and numbers removed, sorted.
|
||||
func Digest(evs []events.Event) []string {
|
||||
var out []string
|
||||
for _, e := range evs {
|
||||
if bookkeeping[e.Type] {
|
||||
continue
|
||||
}
|
||||
out = append(out, string(e.Type)+": "+Normalise(e.Summary))
|
||||
}
|
||||
sortStrings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
var (
|
||||
uuidRe = regexp.MustCompile(`[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}`)
|
||||
numRe = regexp.MustCompile(`\d+(\.\d+)?`)
|
||||
)
|
||||
|
||||
// Normalise strips ids, numbers and case so summaries from different minds compare.
|
||||
func Normalise(s string) string {
|
||||
s = uuidRe.ReplaceAllString(strings.ToLower(s), "<id>")
|
||||
s = numRe.ReplaceAllString(s, "<n>")
|
||||
return strings.Join(strings.Fields(s), " ")
|
||||
}
|
||||
|
||||
func sortStrings(xs []string) {
|
||||
for i := 1; i < len(xs); i++ {
|
||||
for j := i; j > 0 && xs[j] < xs[j-1]; j-- {
|
||||
xs[j], xs[j-1] = xs[j-1], xs[j]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Same reports whether two digests match as multisets.
|
||||
func Same(a, b []string) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for i := range a {
|
||||
if a[i] != b[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
var _ = ids.Nil
|
||||
@@ -0,0 +1,293 @@
|
||||
package replay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/minds"
|
||||
"github.com/andydixon/noema/internal/snapshots"
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
)
|
||||
|
||||
// Replay is one recorded re-run.
|
||||
type Replay struct {
|
||||
ID ids.ID `json:"id"`
|
||||
MindID ids.ID `json:"mind_id"`
|
||||
SnapshotID ids.ID `json:"snapshot_id"`
|
||||
ReplayMindID *ids.ID `json:"replay_mind_id,omitempty"`
|
||||
FromSeq int64 `json:"from_seq"`
|
||||
ToSeq int64 `json:"to_seq"`
|
||||
Status string `json:"status"`
|
||||
Summary Summary `json:"summary"`
|
||||
Steps json.RawMessage `json:"steps"`
|
||||
Error string `json:"error"`
|
||||
CreatedBy string `json:"created_by"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
FinishedAt *time.Time `json:"finished_at,omitempty"`
|
||||
}
|
||||
|
||||
// Summary counts step labels.
|
||||
type Summary struct {
|
||||
Steps int `json:"steps"`
|
||||
Exact int `json:"exact"`
|
||||
Approximate int `json:"approximate"`
|
||||
Diverged int `json:"diverged"`
|
||||
LLMLive int `json:"llm_live"`
|
||||
LLMReplayed int `json:"llm_replayed"`
|
||||
Cycles int `json:"cycles"`
|
||||
}
|
||||
|
||||
// Step is the outcome of replaying one stimulus.
|
||||
type Step struct {
|
||||
Index int `json:"index"`
|
||||
StimulusID ids.ID `json:"stimulus_id"`
|
||||
InjectedID ids.ID `json:"injected_id"`
|
||||
Summary string `json:"summary"`
|
||||
Original []string `json:"original"`
|
||||
Replayed []string `json:"replayed"`
|
||||
Label string `json:"label"` // exact | approximate | diverged
|
||||
Note string `json:"note"`
|
||||
Cycles int `json:"cycles"`
|
||||
LLMLive int `json:"llm_live"`
|
||||
LLMReplayed int `json:"llm_replayed"`
|
||||
}
|
||||
|
||||
// Service runs and stores replays.
|
||||
type Service struct {
|
||||
DB *pgxpool.Pool
|
||||
Events *evstore.Store
|
||||
Minds *minds.Store
|
||||
Snapshots *snapshots.Store
|
||||
Brain brain.Config // template; Services.LLM must be a replaying router
|
||||
Log *slog.Logger
|
||||
}
|
||||
|
||||
// MaxStimuli bounds one replay.
|
||||
const MaxStimuli = 200
|
||||
|
||||
// Run replays the source mind's perception events in (from, to] starting
|
||||
// from a snapshot, inside a fresh disabled sandbox mind.
|
||||
func (s *Service) Run(ctx context.Context, src minds.Mind, snapshotID ids.ID, from, to int64, by string) (Replay, error) {
|
||||
r, err := s.Begin(ctx, src, snapshotID, from, to, by)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
return s.Execute(ctx, r)
|
||||
}
|
||||
|
||||
// Begin validates and records a replay so callers can run Execute in the
|
||||
// background and hand the id back immediately.
|
||||
func (s *Service) Begin(ctx context.Context, src minds.Mind, snapshotID ids.ID, from, to int64, by string) (Replay, error) {
|
||||
r := Replay{ID: ids.New(), MindID: src.ID, SnapshotID: snapshotID, FromSeq: from, ToSeq: to, Status: "running", CreatedBy: by, CreatedAt: time.Now(), Steps: []byte("[]"), Summary: Summary{}}
|
||||
sn, err := s.Snapshots.Get(ctx, snapshotID)
|
||||
if err != nil || sn.MindID != src.ID {
|
||||
return r, fmt.Errorf("snapshot not found for this mind")
|
||||
}
|
||||
if from < sn.EventSeq {
|
||||
r.FromSeq = sn.EventSeq
|
||||
}
|
||||
if to <= r.FromSeq {
|
||||
if last, err := s.Events.Query(ctx, evstore.Filter{MindID: src.ID, Limit: 1}); err == nil && len(last) > 0 {
|
||||
r.ToSeq = last[0].Seq
|
||||
}
|
||||
}
|
||||
if r.ToSeq <= r.FromSeq {
|
||||
return r, fmt.Errorf("nothing to replay after event %d", r.FromSeq)
|
||||
}
|
||||
_, err = s.DB.Exec(ctx, `INSERT INTO replays (id, mind_id, snapshot_id, from_seq, to_seq, status, created_by) VALUES ($1,$2,$3,$4,$5,'running',$6)`, r.ID, src.ID, snapshotID, r.FromSeq, r.ToSeq, by)
|
||||
return r, err
|
||||
}
|
||||
|
||||
// Execute performs a replay recorded by Begin.
|
||||
func (s *Service) Execute(ctx context.Context, r Replay) (Replay, error) {
|
||||
src, err := s.Minds.Get(ctx, r.MindID)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
snapshotID, from, to, by := r.SnapshotID, r.FromSeq, r.ToSeq, r.CreatedBy
|
||||
defer func() {
|
||||
if p := recover(); p != nil {
|
||||
s.Log.Error("replay panicked", "replay", r.ID, "panic", p)
|
||||
s.DB.Exec(context.WithoutCancel(ctx), `UPDATE replays SET status='failed', error='internal error', finished_at=now() WHERE id=$1`, r.ID) //nolint:errcheck
|
||||
}
|
||||
}()
|
||||
finish := func(status, errText string, steps []Step) (Replay, error) {
|
||||
if steps == nil {
|
||||
steps = []Step{}
|
||||
}
|
||||
sj, _ := json.Marshal(steps)
|
||||
sum, _ := json.Marshal(r.Summary)
|
||||
_, err := s.DB.Exec(ctx, `UPDATE replays SET status=$2, error=$3, steps=$4, summary=$5, replay_mind_id=$6, finished_at=now() WHERE id=$1`, r.ID, status, errText, sj, sum, r.ReplayMindID)
|
||||
r.Status, r.Error, r.Steps = status, errText, sj
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
if errText != "" {
|
||||
return r, fmt.Errorf("%s", errText)
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
sn, err := s.Snapshots.Get(ctx, snapshotID)
|
||||
if err != nil || sn.MindID != src.ID {
|
||||
return finish("failed", "snapshot not found for this mind", nil)
|
||||
}
|
||||
stimuli, err := s.Events.Query(ctx, evstore.Filter{MindID: src.ID, Categories: StimulusCategories, AfterSeq: from, BeforeSeq: to + 1, Ascending: true, Limit: MaxStimuli + 1})
|
||||
if err != nil {
|
||||
return finish("failed", err.Error(), nil)
|
||||
}
|
||||
if len(stimuli) > MaxStimuli {
|
||||
return finish("failed", fmt.Sprintf("more than %d stimuli in range; narrow it", MaxStimuli), nil)
|
||||
}
|
||||
var external []events.Event
|
||||
for _, e := range stimuli {
|
||||
if isStimulus(e) {
|
||||
external = append(external, e)
|
||||
}
|
||||
}
|
||||
if len(external) == 0 {
|
||||
return finish("failed", "no perception or operator events in range", nil)
|
||||
}
|
||||
slug := "replay-" + strings.ToLower(r.ID.String()[len(r.ID.String())-8:])
|
||||
sandbox, err := s.Snapshots.Fork(ctx, snapshotID, snapshots.ForkOptions{Slug: slug, DisplayName: "Replay of " + src.DisplayName, Status: minds.StatusDisabled, By: by,
|
||||
Notes: fmt.Sprintf("replay %s of %s from event %d to %d (snapshot %s)", r.ID, src.Slug, from, to, sn.Name)})
|
||||
if err != nil {
|
||||
return finish("failed", "fork sandbox: "+err.Error(), nil)
|
||||
}
|
||||
r.ReplayMindID = &sandbox.ID
|
||||
drv, err := NewDriver(ctx, s.Brain, sandbox, "replay")
|
||||
if err != nil {
|
||||
return finish("failed", "runtime: "+err.Error(), nil)
|
||||
}
|
||||
var steps []Step
|
||||
for i, stim := range external {
|
||||
st := Step{Index: i, StimulusID: stim.ID, Summary: stim.Summary}
|
||||
// What the source produced between this stimulus and the next one.
|
||||
upper := to + 1
|
||||
if i+1 < len(external) {
|
||||
upper = external[i+1].Seq
|
||||
}
|
||||
orig, _ := s.Events.Query(ctx, evstore.Filter{MindID: src.ID, AfterSeq: stim.Seq, BeforeSeq: upper, Ascending: true, Limit: 2000})
|
||||
st.Original = Digest(withoutPerception(orig))
|
||||
startedAt := time.Now()
|
||||
inj, err := drv.Inject(ctx, stim, map[string]any{"replay_of": stim.ID, "replay_id": r.ID})
|
||||
if err != nil {
|
||||
st.Label, st.Note = "diverged", "inject failed: "+err.Error()
|
||||
steps = append(steps, st)
|
||||
continue
|
||||
}
|
||||
st.InjectedID = inj.ID
|
||||
produced, cycles, err := drv.Settle(ctx)
|
||||
st.Cycles = cycles
|
||||
r.Summary.Cycles += cycles
|
||||
if err != nil {
|
||||
st.Note = "cycle error: " + err.Error()
|
||||
}
|
||||
st.Replayed = Digest(produced)
|
||||
st.LLMLive, st.LLMReplayed = s.llmCounts(ctx, sandbox.ID, startedAt)
|
||||
r.Summary.LLMLive += st.LLMLive
|
||||
r.Summary.LLMReplayed += st.LLMReplayed
|
||||
switch {
|
||||
case st.LLMLive > 0:
|
||||
st.Label = "approximate"
|
||||
st.Note = strings.TrimSpace(st.Note + " live model call(s); recorded response not found")
|
||||
r.Summary.Approximate++
|
||||
case Same(st.Original, st.Replayed):
|
||||
st.Label = "exact"
|
||||
r.Summary.Exact++
|
||||
default:
|
||||
st.Label = "diverged"
|
||||
st.Note = strings.TrimSpace(st.Note + " deterministic run produced different events; state or code differs from the recording")
|
||||
r.Summary.Diverged++
|
||||
}
|
||||
steps = append(steps, st)
|
||||
}
|
||||
r.Summary.Steps = len(steps)
|
||||
return finish("done", "", steps)
|
||||
}
|
||||
|
||||
// StimulusCategories are the event categories a mind receives from outside:
|
||||
// perceptions (messages, integrations, observations) and operator actions.
|
||||
var StimulusCategories = []events.Category{events.CatPerception, events.CatOperator}
|
||||
|
||||
func isStimulus(e events.Event) bool {
|
||||
if strings.HasPrefix(e.Source, "brain") {
|
||||
return false
|
||||
}
|
||||
for _, c := range StimulusCategories {
|
||||
if e.Category == c {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func withoutPerception(evs []events.Event) []events.Event {
|
||||
var out []events.Event
|
||||
for _, e := range evs {
|
||||
if !isStimulus(e) {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Service) llmCounts(ctx context.Context, mind ids.ID, since time.Time) (live, replayed int) {
|
||||
s.DB.QueryRow(ctx, `SELECT count(*) FILTER (WHERE NOT replayed), count(*) FILTER (WHERE replayed) FROM llm_calls WHERE mind_id=$1 AND created_at >= $2`, mind, since).Scan(&live, &replayed) //nolint:errcheck
|
||||
return
|
||||
}
|
||||
|
||||
const replayCols = `id, mind_id, snapshot_id, replay_mind_id, from_seq, to_seq, status, summary, steps, error, created_by, created_at, finished_at`
|
||||
|
||||
func scan(row pgx.Row) (Replay, error) {
|
||||
var r Replay
|
||||
var sum []byte
|
||||
if err := row.Scan(&r.ID, &r.MindID, &r.SnapshotID, &r.ReplayMindID, &r.FromSeq, &r.ToSeq, &r.Status, &sum, &r.Steps, &r.Error, &r.CreatedBy, &r.CreatedAt, &r.FinishedAt); err != nil {
|
||||
return r, storage.Translate(err)
|
||||
}
|
||||
json.Unmarshal(sum, &r.Summary) //nolint:errcheck
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// Get returns one replay.
|
||||
func (s *Service) Get(ctx context.Context, id ids.ID) (Replay, error) {
|
||||
return scan(s.DB.QueryRow(ctx, `SELECT `+replayCols+` FROM replays WHERE id=$1`, id))
|
||||
}
|
||||
|
||||
// List returns replays for a mind, newest first.
|
||||
func (s *Service) List(ctx context.Context, mind ids.ID, limit int) ([]Replay, error) {
|
||||
if limit <= 0 || limit > 200 {
|
||||
limit = 20
|
||||
}
|
||||
rows, err := s.DB.Query(ctx, `SELECT `+replayCols+` FROM replays WHERE mind_id=$1 ORDER BY created_at DESC LIMIT $2`, mind, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []Replay
|
||||
for rows.Next() {
|
||||
r, err := scan(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// StepsOf decodes the steps of a replay.
|
||||
func StepsOf(r Replay) []Step {
|
||||
var out []Step
|
||||
json.Unmarshal(r.Steps, &out) //nolint:errcheck
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
package replay
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/minds"
|
||||
"github.com/andydixon/noema/internal/replay/replaytest"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
)
|
||||
|
||||
func TestReplayIsExactWithoutModel(t *testing.T) {
|
||||
h := replaytest.NewHarness(t)
|
||||
ctx := context.Background()
|
||||
src, err := h.Minds.Create(ctx, minds.CreateParams{Slug: "src", DisplayName: "Source"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Live run on the source, driven synchronously: one stimulus, settle.
|
||||
live, err := NewDriver(ctx, h.Brain, src, "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sn, err := h.Snaps.Create(ctx, src, "t0", "", "test")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := live.Observe(ctx, "operator", "Andy said the payments service failed after the migration", 0.8, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
produced, cycles, err := live.Settle(ctx)
|
||||
if err != nil || cycles == 0 || len(Digest(produced)) == 0 {
|
||||
t.Fatalf("live run produced nothing: %v cycles=%d", err, cycles)
|
||||
}
|
||||
last, _ := h.Events.Query(ctx, evstore.Filter{MindID: src.ID, Limit: 1})
|
||||
svc := &Service{DB: h.Pool, Events: h.Events, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Log: slog.New(slog.DiscardHandler)}
|
||||
r, err := svc.Run(ctx, src, sn.ID, sn.EventSeq, last[0].Seq, "test")
|
||||
if err != nil {
|
||||
t.Fatalf("replay: %v %+v", err, r)
|
||||
}
|
||||
if r.Status != "done" || r.Summary.Steps != 1 || r.Summary.Exact != 1 || r.Summary.LLMLive != 0 {
|
||||
t.Fatalf("expected one exact step: %+v steps=%s", r.Summary, string(r.Steps))
|
||||
}
|
||||
steps := StepsOf(r)
|
||||
if len(steps) != 1 || !Same(steps[0].Original, steps[0].Replayed) || len(steps[0].Replayed) == 0 {
|
||||
t.Fatalf("step mismatch: %+v", steps)
|
||||
}
|
||||
sandbox, err := h.Minds.Get(ctx, *r.ReplayMindID)
|
||||
if err != nil || sandbox.Status != minds.StatusDisabled || sandbox.ForkedFromSnapshot == nil {
|
||||
t.Fatalf("sandbox mind wrong: %+v %v", sandbox, err)
|
||||
}
|
||||
// The source's history is untouched by the replay.
|
||||
after, _ := h.Events.Query(ctx, evstore.Filter{MindID: src.ID, Limit: 1})
|
||||
if after[0].Seq != last[0].Seq {
|
||||
t.Fatal("replay wrote into the source mind")
|
||||
}
|
||||
// Digest normalisation ignores ids and numbers.
|
||||
if Normalise("selected 3 thoughts for 01a0a148-9034-7685-9794-cf5ec0c25fb7") != "selected <n> thoughts for <id>" {
|
||||
t.Fatal(Normalise("selected 3 thoughts for 01a0a148-9034-7685-9794-cf5ec0c25fb7"))
|
||||
}
|
||||
_ = events.Observation
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
// Package replaytest wires a deterministic cognitive stack for replay and
|
||||
// experiment tests.
|
||||
package replaytest
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/brain"
|
||||
"github.com/andydixon/noema/internal/capabilities"
|
||||
"github.com/andydixon/noema/internal/cognition"
|
||||
"github.com/andydixon/noema/internal/conversation"
|
||||
evstore "github.com/andydixon/noema/internal/events"
|
||||
"github.com/andydixon/noema/internal/goals"
|
||||
"github.com/andydixon/noema/internal/graph"
|
||||
"github.com/andydixon/noema/internal/memory"
|
||||
"github.com/andydixon/noema/internal/minds"
|
||||
"github.com/andydixon/noema/internal/motivation"
|
||||
"github.com/andydixon/noema/internal/observability"
|
||||
"github.com/andydixon/noema/internal/predictions"
|
||||
"github.com/andydixon/noema/internal/scheduler"
|
||||
"github.com/andydixon/noema/internal/snapshots"
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/migrations"
|
||||
)
|
||||
|
||||
// Harness wires the deterministic cognitive registry without a model.
|
||||
type Harness struct {
|
||||
Pool *pgxpool.Pool
|
||||
Events *evstore.Store
|
||||
Minds *minds.Store
|
||||
Snaps *snapshots.Store
|
||||
Brain brain.Config
|
||||
}
|
||||
|
||||
// NewHarness is shared with the experiments tests.
|
||||
func NewHarness(t *testing.T) *Harness {
|
||||
t.Helper()
|
||||
pool := storage.TestPool(t, migrations.FS)
|
||||
ctx := context.Background()
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
es := evstore.NewStore(pool, evstore.NewBus(64), nil)
|
||||
ms := minds.NewStore(pool)
|
||||
mem := memory.NewStore(ctx, pool, nil)
|
||||
bs := beliefs.NewStore(pool)
|
||||
caps := &capabilities.Service{Registry: capabilities.NewRegistry(), DB: pool, Events: es, Log: log}
|
||||
for _, c := range capabilities.Builtins() {
|
||||
caps.Registry.Add(c)
|
||||
}
|
||||
reg := brain.NewRegistry().AddProcess(cognition.Attend{}).AddProcess(cognition.Retrieve{}).AddProcess(cognition.Associate{}).AddProcess(cognition.Perceive{}).
|
||||
AddProcess(cognition.GoalMonitor{}).AddProcess(cognition.Curiosity{}).AddProcess(cognition.Metacognition{}).AddProcess(cognition.Act{})
|
||||
cognition.Handlers{Memory: mem, Beliefs: bs, Conversations: conversation.NewStore(pool), Graph: graph.NewStore(pool)}.Register(reg)
|
||||
cfg := brain.Config{DB: pool, Events: es, Minds: ms, Registry: reg, Pool: scheduler.NewPool(2), Log: log, Metrics: observability.NewMetrics(),
|
||||
Services: brain.Services{Events: es, Memory: mem, Beliefs: bs, Goals: goals.NewStore(pool), Motivation: motivation.NewStore(pool), Predictions: predictions.NewStore(pool), Capabilities: caps}}
|
||||
return &Harness{Pool: pool, Events: es, Minds: ms, Snaps: &snapshots.Store{DB: pool, Events: es, Minds: ms}, Brain: cfg}
|
||||
}
|
||||
Reference in New Issue
Block a user