feat(beliefs): state what a score means; gated calibration; primary uncertainty
Every breakdown now carries its meaning: an uncalibrated epistemic score derived from recorded evidence, not a probability of truth. Belief detail adds a calibrated estimate read from the mind's resolved predictions when at least twenty were decided in the score's range, and otherwise says why none is shown; and names the primary uncertainty deterministically (no live evidence, conflicting evidence, second-hand only, unclear reading, a single source family, little evidence). The belief list heading becomes Score.
This commit is contained in:
Andy Dixon
@@ -139,6 +139,10 @@ type BeliefDetail struct {
|
||||
Events []events.Event `json:"events"`
|
||||
Origin *events.Event `json:"origin,omitempty"`
|
||||
Assertions []epistemics.Assertion `json:"assertions"` // who stated this proposition, with what stance
|
||||
// Score semantics: the raw number, its empirical reading where prediction
|
||||
// outcomes allow one, and the strongest deterministic reason to doubt it.
|
||||
Calibration beliefs.Calibrated `json:"calibration"`
|
||||
PrimaryUncertainty string `json:"primary_uncertainty"`
|
||||
}
|
||||
|
||||
// BeliefDetail assembles the detail; shared with the UI.
|
||||
@@ -164,6 +168,12 @@ func (s *Server) BeliefDetail(r *http.Request, mind, id ids.ID) (BeliefDetail, e
|
||||
if d.Assertions == nil {
|
||||
d.Assertions = []epistemics.Assertion{}
|
||||
}
|
||||
if s.deps.Predictions != nil {
|
||||
d.Calibration = s.deps.Predictions.CalibratedScore(ctx, mind, b.Confidence)
|
||||
} else {
|
||||
d.Calibration = beliefs.CalibrateWith(b.Confidence, nil)
|
||||
}
|
||||
d.PrimaryUncertainty = beliefs.PrimaryUncertainty(d.Breakdown, d.Evidence)
|
||||
d.Events, _ = s.deps.Events.DerivedFrom(ctx, "belief", string(id), 100)
|
||||
if !b.OriginEvent.IsZero() {
|
||||
if e, err := s.deps.Events.Get(ctx, b.OriginEvent); err == nil {
|
||||
|
||||
@@ -5,6 +5,7 @@ package beliefs
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
@@ -86,6 +87,94 @@ type Evidence struct {
|
||||
ProvenanceStatus string `json:"provenance_status,omitempty"` // recorded | legacy_unknown
|
||||
}
|
||||
|
||||
// ScoreMeaning states, on every breakdown, what the confidence number is.
|
||||
// It is an uncalibrated epistemic score: the deterministic output of the
|
||||
// evidence arithmetic, comparable between one mind's beliefs, and not a
|
||||
// probability that the proposition is true. Only resolved predictions can
|
||||
// say how a score has tracked reality; see Calibrated.
|
||||
const ScoreMeaning = "uncalibrated epistemic score derived from recorded evidence; not a probability of truth"
|
||||
|
||||
// Calibrated is the empirical reading of a score, when there is one: the
|
||||
// hit rate of this mind's resolved predictions whose stated confidence fell
|
||||
// in the same range. It is offered only above a minimum sample so that no
|
||||
// false precision is shown.
|
||||
type Calibrated struct {
|
||||
Raw float64 `json:"raw"`
|
||||
Estimate float64 `json:"estimate,omitempty"`
|
||||
SampleSize int `json:"sample_size"`
|
||||
Available bool `json:"available"`
|
||||
Basis string `json:"basis"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
}
|
||||
|
||||
// CalibrationBucket is one range of stated confidence with its outcomes.
|
||||
type CalibrationBucket struct {
|
||||
Low, High float64
|
||||
Decided int
|
||||
HitRate float64
|
||||
}
|
||||
|
||||
// MinCalibrationSample is the fewest decided predictions in a bucket before
|
||||
// a calibrated estimate is shown.
|
||||
const MinCalibrationSample = 20
|
||||
|
||||
// CalibrateWith maps a score onto the bucket containing it. Deterministic;
|
||||
// the buckets come from prediction outcomes.
|
||||
func CalibrateWith(score float64, buckets []CalibrationBucket) Calibrated {
|
||||
c := Calibrated{Raw: round4(score), Basis: "resolved predictions of this mind with stated confidence in the same range"}
|
||||
for _, b := range buckets {
|
||||
if score >= b.Low && score < b.High {
|
||||
c.SampleSize = b.Decided
|
||||
if b.Decided >= MinCalibrationSample {
|
||||
c.Available, c.Estimate = true, round4(b.HitRate)
|
||||
} else {
|
||||
c.Reason = fmt.Sprintf("insufficient data: %d decided prediction(s) in this range; %d needed", b.Decided, MinCalibrationSample)
|
||||
}
|
||||
return c
|
||||
}
|
||||
}
|
||||
c.Reason = "no prediction outcomes recorded in this range"
|
||||
return c
|
||||
}
|
||||
|
||||
// PrimaryUncertainty names, deterministically, the strongest reason to doubt
|
||||
// a score: what an operator should look at first.
|
||||
func PrimaryUncertainty(b Breakdown, ev []EvidenceRow) string {
|
||||
var live, reported int
|
||||
lowReading := false
|
||||
for _, e := range ev {
|
||||
if e.Retracted || e.Weight <= 0 || e.Direction == "against" {
|
||||
continue
|
||||
}
|
||||
live++
|
||||
if e.Derivation == DerivationAssertion && strings.Contains(e.Note, "reported") {
|
||||
reported++
|
||||
}
|
||||
if strings.Contains(e.Note, "interpretation confidence") {
|
||||
for _, low := range []string{"40%", "50%", "60%"} {
|
||||
if strings.Contains(e.Note, low) {
|
||||
lowReading = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case live == 0 && b.Against == 0:
|
||||
return "no live evidence"
|
||||
case b.For > 0 && b.Against > 0 && b.Against >= 0.5*b.For:
|
||||
return "conflicting evidence"
|
||||
case live > 0 && reported == live:
|
||||
return "second-hand only: nobody who saw it has told the mind"
|
||||
case lowReading:
|
||||
return "unclear reading of what was said"
|
||||
case b.SourcesFor == 1:
|
||||
return "evidence independence: a single source family"
|
||||
case b.For < 0.3:
|
||||
return "little evidence"
|
||||
}
|
||||
return "none dominant"
|
||||
}
|
||||
|
||||
// Derivations. Primary derivations bring information from outside the
|
||||
// belief system; derived ones re-express information already present.
|
||||
const (
|
||||
@@ -174,6 +263,7 @@ type FamilyShare struct {
|
||||
// Breakdown explains a confidence value.
|
||||
type Breakdown struct {
|
||||
Prior float64 `json:"prior"`
|
||||
Meaning string `json:"meaning"` // what the number is and is not
|
||||
Families []FamilyShare `json:"families,omitempty"` // per independent-source group
|
||||
For float64 `json:"for"` // Σ weight×credibility, supporting
|
||||
Against float64 `json:"against"` // Σ weight×credibility, opposing
|
||||
@@ -197,7 +287,7 @@ type Breakdown struct {
|
||||
// re-express the episodes rather than corroborate them. Diversity counts
|
||||
// families, not rows (T4: poisoned memory; review pass item 2).
|
||||
func Compute(prior float64, ev []Evidence) Breakdown {
|
||||
b := Breakdown{Prior: clampP(prior), Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"}
|
||||
b := Breakdown{Prior: clampP(prior), Meaning: ScoreMeaning, Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"}
|
||||
type group struct {
|
||||
primary, derived []float64
|
||||
items int
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package beliefs
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/noema/pkg/concepts"
|
||||
@@ -125,3 +126,45 @@ func TestProvenanceGroupingPreventsInflation(t *testing.T) {
|
||||
t.Fatal(f)
|
||||
}
|
||||
}
|
||||
|
||||
// TestCalibrationIsGatedAndDeterministic: a calibrated estimate appears only
|
||||
// with enough decided predictions in the score's range, and the primary
|
||||
// uncertainty names the right doubt.
|
||||
func TestCalibrationIsGatedAndDeterministic(t *testing.T) {
|
||||
buckets := []CalibrationBucket{{0, 0.5, 3, 0.33}, {0.5, 0.8, 25, 0.64}, {0.8, 1.01, 40, 0.9}}
|
||||
if c := CalibrateWith(0.3, buckets); c.Available || c.SampleSize != 3 || c.Reason == "" {
|
||||
t.Fatalf("small sample must not calibrate: %+v", c)
|
||||
}
|
||||
if c := CalibrateWith(0.59, buckets); !c.Available || c.Estimate != 0.64 || c.SampleSize != 25 {
|
||||
t.Fatalf("calibrated: %+v", c)
|
||||
}
|
||||
if c := CalibrateWith(0.59, nil); c.Available || c.Reason == "" {
|
||||
t.Fatalf("no buckets: %+v", c)
|
||||
}
|
||||
if b := Compute(0.5, nil); b.Meaning != ScoreMeaning {
|
||||
t.Fatal("breakdown must carry its meaning")
|
||||
}
|
||||
rows := func(rs ...EvidenceRow) ([]Evidence, []EvidenceRow) {
|
||||
var ev []Evidence
|
||||
for _, r := range rs {
|
||||
ev = append(ev, r.Evidence)
|
||||
}
|
||||
return ev, rs
|
||||
}
|
||||
one := EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:andy", Family: "actor:andy", Derivation: DerivationAssertion, Weight: 0.5, Credibility: 0.9}, Note: "andy asserted it (read with 85% interpretation confidence)"}
|
||||
ev, rs := rows(one)
|
||||
if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "evidence independence: a single source family" {
|
||||
t.Fatal(u)
|
||||
}
|
||||
ev, rs = rows(EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:alice", Family: "actor:alice", Derivation: DerivationAssertion, Weight: 0.2, Credibility: 0.6}, Note: "andy reported that alice holds this (asserted, read with 75% interpretation confidence)"})
|
||||
if u := PrimaryUncertainty(Compute(0.5, ev), rs); !strings.HasPrefix(u, "second-hand") {
|
||||
t.Fatal(u)
|
||||
}
|
||||
ev, rs = rows(one, EvidenceRow{Evidence: Evidence{Direction: "against", Kind: "operator", Source: "operator:x", Family: "operator:x", Weight: 0.5, Credibility: 0.9}})
|
||||
if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "conflicting evidence" {
|
||||
t.Fatal(u)
|
||||
}
|
||||
if u := PrimaryUncertainty(Compute(0.5, nil), nil); u != "no live evidence" {
|
||||
t.Fatal(u)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
|
||||
"github.com/andydixon/noema/internal/beliefs"
|
||||
"github.com/andydixon/noema/internal/storage"
|
||||
"github.com/andydixon/noema/pkg/events"
|
||||
"github.com/andydixon/noema/pkg/ids"
|
||||
@@ -454,3 +455,18 @@ func (s *Store) Stats(ctx context.Context, mind ids.ID) (map[string]int64, error
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// CalibratedScore reads a belief score against this mind's whole prediction
|
||||
// history. It is the only empirical calibration Noema offers, and it says
|
||||
// so when the sample is too small.
|
||||
func (s *Store) CalibratedScore(ctx context.Context, mind ids.ID, score float64) beliefs.Calibrated {
|
||||
c, err := s.Calibrate(ctx, mind, time.Time{})
|
||||
if err != nil {
|
||||
return beliefs.CalibrateWith(score, nil)
|
||||
}
|
||||
var buckets []beliefs.CalibrationBucket
|
||||
for _, b := range c.Buckets {
|
||||
buckets = append(buckets, beliefs.CalibrationBucket{Low: b.Low, High: b.High, Decided: b.Count, HitRate: b.HitRate})
|
||||
}
|
||||
return beliefs.CalibrateWith(score, buckets)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user