andy/noema
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(replay): record runtime identity; label compatibility; analyse first divergence

Browse Source 
Snapshots, replays and experiments now record the runtime they ran under:
build (module version, VCS commit, modified tree, commit time, Go version,
platform), schema version, a digest of the non-secret configuration that
ignores secret and credential changes, personality version, policy digest,
model routing including per-mind overrides, prompt versions, deterministic
subsystem versions and timezone. A replay compares its identity with the
snapshot's and is an exact-candidate, a different-build (differences
listed) or incompatible (schema differs; the replay stops before running).
The first non-exact step is analysed into events only in the recording,
events only in the replay, and the components they belong to, labelled as
inferred; live model calls are named as their own source of difference.
Migration 0022 is additive; unrecorded identities are reported as such.
This commit is contained in:
Andy Dixon
2026-08-20 21:11:27 +01:00
parent 37b9f0e2f5
commit 522de3b026
18 changed files with 648 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/experiments/experiments.go
+17 -13
View File
@@ -17,6 +17,7 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
"github.com/andydixon/noema/internal/brain"
"github.com/andydixon/noema/internal/identity"
"github.com/andydixon/noema/internal/minds"
"github.com/andydixon/noema/internal/replay"
"github.com/andydixon/noema/internal/snapshots"
@@ -68,19 +69,20 @@ type Experiment struct {
// Results is what a run records.
type Results struct {
Subject ids.ID `json:"subject_mind_id"`
SubjectSlug string `json:"subject_slug"`
Steps []StepResult `json:"steps"`
EventsByType map[string]int `json:"events_by_type"`
Beliefs []BeliefRow `json:"beliefs"`
Memories int `json:"memories"`
Goals int `json:"goals"`
Expectations []ExpectResult `json:"expectations"`
LLMCalls map[string]int `json:"llm_calls"` // live / replayed
Routing []map[string]any `json:"routing,omitempty"`
DurationMS int64 `json:"duration_ms"`
Cycles int `json:"cycles"`
TruncatedAfter int `json:"truncated_after_stimulus,omitempty"`
Subject ids.ID `json:"subject_mind_id"`
SubjectSlug string `json:"subject_slug"`
Steps []StepResult `json:"steps"`
EventsByType map[string]int `json:"events_by_type"`
Beliefs []BeliefRow `json:"beliefs"`
Memories int `json:"memories"`
Goals int `json:"goals"`
Expectations []ExpectResult `json:"expectations"`
LLMCalls map[string]int `json:"llm_calls"` // live / replayed
Routing []map[string]any `json:"routing,omitempty"`
DurationMS int64 `json:"duration_ms"`
Identity identity.Identity `json:"identity"` // the runtime that produced these results
Cycles int `json:"cycles"`
TruncatedAfter int `json:"truncated_after_stimulus,omitempty"`
}
// StepResult records one stimulus.
@@ -114,6 +116,7 @@ type Service struct {
Snapshots *snapshots.Store
Brain brain.Config
Routing func(ctx context.Context) []map[string]any // current LLM role routing, recorded with results
Identity *identity.Capturer
Log *slog.Logger
}
@@ -219,6 +222,7 @@ func (s *Service) execute(ctx context.Context, e *Experiment, by string) (Result
if s.Routing != nil {
res.Routing = s.Routing(ctx)
}
res.Identity = s.Identity.Capture(ctx, subject.ID)
drv, err := replay.NewDriver(ctx, s.Brain, subject, "experiment")
if err != nil {
return res, err