andy/noema
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(replay): record runtime identity; label compatibility; analyse first divergence

Browse Source 
Snapshots, replays and experiments now record the runtime they ran under:
build (module version, VCS commit, modified tree, commit time, Go version,
platform), schema version, a digest of the non-secret configuration that
ignores secret and credential changes, personality version, policy digest,
model routing including per-mind overrides, prompt versions, deterministic
subsystem versions and timezone. A replay compares its identity with the
snapshot's and is an exact-candidate, a different-build (differences
listed) or incompatible (schema differs; the replay stops before running).
The first non-exact step is analysed into events only in the recording,
events only in the replay, and the components they belong to, labelled as
inferred; live model calls are named as their own source of difference.
Migration 0022 is additive; unrecorded identities are reported as such.
This commit is contained in:
Andy Dixon
2026-08-20 21:11:27 +01:00
parent 37b9f0e2f5
commit 522de3b026
18 changed files with 648 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/replay/replay_test.go
+27 -1
View File
@@ -2,12 +2,14 @@ package replay
import (
"context"
"encoding/json"
"log/slog"
"sort"
"strings"
"testing"
"github.com/andydixon/noema/internal/beliefs"
"github.com/andydixon/noema/internal/identity"
evstore "github.com/andydixon/noema/internal/events"
"github.com/andydixon/noema/internal/minds"
@@ -40,7 +42,7 @@ func TestReplayIsExactWithoutModel(t *testing.T) {
t.Fatalf("live run produced nothing: %v cycles=%d", err, cycles)
}
last, _ := h.Events.Query(ctx, evstore.Filter{MindID: src.ID, Limit: 1})
svc := &Service{DB: h.Pool, Events: h.Events, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Log: slog.New(slog.DiscardHandler)}
svc := &Service{DB: h.Pool, Events: h.Events, Minds: h.Minds, Snapshots: h.Snaps, Brain: h.Brain, Identity: h.Identity, Log: slog.New(slog.DiscardHandler)}
r, err := svc.Run(ctx, src, sn.ID, sn.EventSeq, last[0].Seq, "test")
if err != nil {
t.Fatalf("replay: %v %+v", err, r)
@@ -48,6 +50,25 @@ func TestReplayIsExactWithoutModel(t *testing.T) {
if r.Status != "done" || r.Summary.Steps != 1 || r.Summary.Exact != 1 || r.Summary.LLMLive != 0 {
t.Fatalf("expected one exact step: %+v steps=%s", r.Summary, string(r.Steps))
}
// Reproducibility record: the runtime is recorded and compared with the
// snapshot's; same process, same schema means an exact candidate with no
// first divergence.
if r.Identity.SchemaVersion == 0 || r.Identity.Build.GoVersion == "" || len(r.Identity.Deterministic) == 0 {
t.Fatalf("identity not recorded: %+v", r.Identity)
}
if r.Compatibility != identity.CompatExactCandidate || r.Divergence != nil {
t.Fatalf("compatibility %q diff %v divergence %+v", r.Compatibility, r.IdentityDiff, r.Divergence)
}
// A snapshot taken under another schema cannot be replayed faithfully.
old, _ := h.Snaps.Get(ctx, sn.ID)
old.Identity.SchemaVersion = 1
oj, _ := json.Marshal(old.Identity)
h.Pool.Exec(ctx, `UPDATE snapshots SET identity=$2 WHERE id=$1`, sn.ID, oj) //nolint:errcheck
inc, err := svc.Run(ctx, src, sn.ID, sn.EventSeq, last[0].Seq, "test")
if inc.Status != "incompatible" || inc.Compatibility != identity.CompatIncompatible || inc.Summary.Steps != 0 {
t.Fatalf("incompatible replay should stop before running: %+v %v", inc, err)
}
h.Pool.Exec(ctx, `UPDATE snapshots SET identity=$2 WHERE id=$1`, sn.ID, mustJSON(r.Identity)) //nolint:errcheck
steps := StepsOf(r)
if len(steps) != 1 || !Same(steps[0].Original, steps[0].Replayed) || len(steps[0].Replayed) == 0 {
t.Fatalf("step mismatch: %+v", steps)
@@ -96,3 +117,8 @@ func provenanceDigest(t *testing.T, h *replaytest.Harness, mind ids.ID) string {
sort.Strings(lines)
return strings.Join(lines, "\n")
}
func mustJSON(v any) []byte {
b, _ := json.Marshal(v)
return b
}