Andy Dixon
844de93c8b
A mind's configuration may override model routing per role (llm_roles); the router honours it for that mind only and fails closed when the named model is missing or disabled, so a comparison between providers can never silently fall back to the default. Experiments may declare variants: each becomes its own fork of the same snapshot with its own configuration, personality or routing, receives identical stimuli, and is measured. The results carry one entry per variant and a deterministic comparison: first divergent stimulus, belief and attention overlap, mean score difference on shared propositions, and the ranges of memory counts, goal counts, cost and capability requests. Single-subject definitions are unchanged.
27 lines
5.9 KiB
HTML
27 lines
5.9 KiB
HTML
{{define "content"}}
|
||
{{$e := .Data.E}}{{$r := .Data.Results}}
|
||
<div class="pagehead mindhead"><div><p class="muted small"><a href="/experiments">Experiments</a></p><h1>{{$e.Name}} <span class="badge {{if eq $e.Status "done"}}ok{{else if eq $e.Status "failed"}}danger{{else if eq $e.Status "running"}}warn{{end}}">{{$e.Status}}</span></h1>
|
||
<p class="muted">{{$e.Hypothesis}}</p><p class="small">Baseline: <a href="/minds/{{.Data.Base.Slug}}">{{.Data.Base.DisplayName}}</a> snapshot <strong>{{.Data.Snapshot.Name}}</strong> (event {{.Data.Snapshot.EventSeq}}){{with .Data.Subject}} · Subject: <a href="/minds/{{.Slug}}">{{.Slug}}</a> · <a href="/minds/{{$.Data.Base.Slug}}/compare?with={{.Slug}}">compare with baseline</a>{{end}}{{with $e.Error}} · <span class="danger">{{.}}</span>{{end}}</p></div>
|
||
<div class="row">{{if .Principal.User.Role.AtLeast "operator"}}<form method="post" action="/experiments/{{$e.ID}}/run"><input type="hidden" name="_csrf" value="{{.CSRF}}"><button class="btn btn-primary" {{if eq $e.Status "running"}}disabled{{end}}>{{if eq $e.Status "draft"}}Run{{else}}Run again{{end}}</button></form>{{end}}
|
||
{{if ne $e.Status "draft"}}<a class="btn" href="/api/v1/experiments/{{$e.ID}}/export">Export results</a>{{end}}
|
||
{{if .Principal.User.Role.AtLeast "administrator"}}<form method="post" action="/experiments/{{$e.ID}}/delete"><input type="hidden" name="_csrf" value="{{.CSRF}}"><button class="btn btn-danger" data-confirm="Delete this experiment? The subject mind is archived.">Delete</button></form>{{end}}</div></div>
|
||
{{if eq $e.Status "running"}}<p class="muted small" data-live-refresh="5000">Running… this page refreshes.</p>{{end}}
|
||
<section class="grid cols-2">
|
||
<div class="card"><h2>Definition</h2><pre class="json">{{.Data.DefinitionJSON}}</pre></div>
|
||
<div class="card"><h2>Results</h2>
|
||
{{if eq $e.Status "done" "failed"}}
|
||
<table class="kv-table"><tr><th>Duration</th><td>{{$r.DurationMS}} ms · {{$r.Cycles}} cycles{{if $r.TruncatedAfter}} · <span class="danger">stopped after stimulus {{$r.TruncatedAfter}} (duration cap)</span>{{end}}</td></tr><tr><th>Memories / goals</th><td>{{$r.Memories}} / {{$r.Goals}}</td></tr><tr><th>Model calls</th><td>{{index $r.LLMCalls "live"}} live · {{index $r.LLMCalls "replayed"}} replayed</td></tr></table>
|
||
{{if $r.Expectations}}<h3>Expectations</h3><table class="table compact"><tbody>{{range $r.Expectations}}<tr><td><span class="badge {{if .Met}}ok{{else}}danger{{end}}">{{if .Met}}met{{else}}not met{{end}}</span></td><td class="mono small">{{.EventType}}{{with .Contains}} ∋ "{{.}}"{{end}}{{if .Absent}} (absent){{end}}</td><td class="small muted">{{trunc .Matched 90}}</td></tr>{{end}}</tbody></table>{{end}}
|
||
{{if $r.EventsByType}}<h3>Events produced</h3><p class="small mono">{{range $t, $n := $r.EventsByType}}{{$t}} {{$n}} · {{end}}</p>{{end}}
|
||
{{if $r.Beliefs}}<h3>Beliefs of the subject</h3><table class="table compact"><tbody>{{range $r.Beliefs}}<tr><td>{{.Proposition}}</td><td class="num">{{conf .Confidence}}</td><td class="small muted">{{.Status}} · {{.Source}}</td></tr>{{end}}</tbody></table>{{end}}
|
||
{{else}}<p class="muted">Not run yet.</p>{{end}}
|
||
</div>
|
||
</section>
|
||
{{if $r.Variants}}{{if gt (len $r.Variants) 1}}<div class="card"><h2>Variants</h2>
|
||
<table class="table compact"><thead><tr><th>Variant</th><th>Subject</th><th>Routing</th><th class="num">Beliefs</th><th class="num">Memories</th><th class="num">Goals</th><th class="num">Cycles</th><th class="num">Model calls</th><th class="num">Tokens</th><th class="num">Cost</th><th class="num">Capability requests</th><th>Expectations</th></tr></thead><tbody>
|
||
{{range $r.Variants}}<tr><td><strong>{{.Name}}</strong>{{with .Error}}<div class="small danger">{{.}}</div>{{end}}</td><td><a href="/minds/{{.SubjectSlug}}">{{.SubjectSlug}}</a></td><td class="small">{{range $k, $v := .Routing}}{{$k}} → {{$v}}<br>{{else}}default{{end}}</td><td class="num">{{len .Beliefs}}</td><td class="num">{{.Memories}}</td><td class="num">{{.Goals}}</td><td class="num">{{.Cycles}}</td><td class="num">{{index .LLMCalls "live"}} live / {{index .LLMCalls "replayed"}} replayed</td><td class="num">{{.Tokens}}</td><td class="num">{{printf "%.4f" .Cost}}</td><td class="num">{{.CapabilityRequests}}</td><td class="small">{{range .Expectations}}<span class="badge small {{if .Met}}ok{{else}}danger{{end}}">{{.EventType}}</span> {{end}}</td></tr>{{end}}</tbody></table>
|
||
{{with $r.Comparison}}<h3>Divergence</h3><table class="kv-table small"><tr><th>First divergent stimulus</th><td>{{if lt .FirstDivergentStimulus 0}}none{{else}}{{.FirstDivergentStimulus}}{{end}}</td></tr><tr><th>Belief overlap</th><td>{{printf "%.2f" .BeliefOverlap}}</td></tr><tr><th>Mean score difference (shared beliefs)</th><td>{{printf "%.3f" .MeanConfidenceDiff}}</td></tr><tr><th>Attention overlap</th><td>{{printf "%.2f" .AttentionOverlap}}</td></tr><tr><th>Memories</th><td>{{index .MemoryCountRange 0}}–{{index .MemoryCountRange 1}}</td></tr><tr><th>Goals</th><td>{{index .GoalCountRange 0}}–{{index .GoalCountRange 1}}</td></tr><tr><th>Cost</th><td>{{printf "%.4f" (index .CostRange 0)}}–{{printf "%.4f" (index .CostRange 1)}}</td></tr><tr><th>Capability requests</th><td>{{index .CapabilityRequestRange 0}}–{{index .CapabilityRequestRange 1}}</td></tr></table><ul class="small muted">{{range .Notes}}<li>{{.}}</li>{{end}}</ul>{{end}}
|
||
<p class="small muted">Compare any two subjects in detail: {{range $i, $v := $r.Variants}}{{if $i}} · {{end}}<a href="/minds/{{$.Data.Base.Slug}}/compare?with={{$v.SubjectSlug}}">baseline vs {{$v.Name}}</a>{{end}}</p></div>{{end}}{{end}}
|
||
{{if $r.Steps}}<div class="card"><h2>Stimuli</h2>{{range $r.Steps}}<div class="replay-step"><div><strong>{{.Index}}.</strong> <a href="/events/{{.EventID}}">{{.Text}}</a> <span class="small muted">· {{.Cycles}} cycle(s)</span></div><ul class="small mono">{{range .Produced}}<li>{{.}}</li>{{else}}<li class="muted">nothing produced</li>{{end}}</ul></div>{{end}}</div>{{end}}
|
||
{{end}}
|