diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cfc3b3..3761aca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,8 @@ All notable changes to Noema are recorded here. Format follows Keep a Changelog; - Security suite: SQL-injection effect test, static allow-list scan for assembled SQL, deceived-mind authority test, garbage model output test, self-preservation vocabulary test; threat model rewritten so every verification cites a test that asserts the claim. ### Added +- Confidence semantics: every breakdown states that the number is an uncalibrated epistemic score; belief detail adds a calibrated estimate from the mind's resolved predictions when at least twenty were decided in the same range (otherwise says why not) and names the primary uncertainty deterministically. `docs/beliefs.md` defines every epistemic number. +- Explicit `unknown` belief status: no live evidence either way is neither tentative belief nor falsity. Migration 0019. - Assertions: every statement made to a mind is recorded first as an append-only assertion (attributed actor, stance, certainty, modality, negation, reported or quoted speech, hypothetical status, interpretation confidence and notes) before, and separately from, any belief about the proposition. Reported speech is attributed to the person reported and weighed as second-hand; hypotheticals, questions and imperatives create no belief; expressed doubt counts against. API `GET /api/v1/minds/{mind}/assertions`, belief pages show who said so. Migration 0018. ### Changed diff --git a/docs/site/pages/beliefs.md b/docs/site/pages/beliefs.md index 3db70da..9ff831b 100644 --- a/docs/site/pages/beliefs.md +++ b/docs/site/pages/beliefs.md @@ -8,7 +8,7 @@ A belief starts from a prior set by its epistemic kind (a hypothesis at 0.25, a confidence = sigmoid( logit(prior) + 1.5 × (for × diversity − against) ) ``` -where `for` and `against` are the sums of family contributions and `diversity` rises from 0.6 for a single family to 1.0 for three or more. The result is bounded to [0.02, 0.98]: a mind is never certain and never certain of the negation. Every evidence row carries its derivation, family and root ids; the breakdown, family by family, is stored with every version and shown on the belief page. Rows written before provenance was recorded are marked as legacy and grouped by their source string; no ancestry is invented for them. +where `for` and `against` are the sums of family contributions and `diversity` rises from 0.6 for a single family to 1.0 for three or more. The result is bounded to [0.02, 0.98]: a mind is never certain and never certain of the negation. The number is an **uncalibrated epistemic score**, not a probability of truth; the belief page also shows a calibrated estimate drawn from the mind's resolved predictions when at least twenty were decided in the same range, and names the primary uncertainty (a single source family, second-hand only, conflicting evidence, little evidence, an unclear reading). Every evidence row carries its derivation, family and root ids; the breakdown, family by family, is stored with every version and shown on the belief page. Rows written before provenance was recorded are marked as legacy and grouped by their source string; no ancestry is invented for them. ## Statuses diff --git a/internal/api/beliefs.go b/internal/api/beliefs.go index d48bc8c..160850c 100644 --- a/internal/api/beliefs.go +++ b/internal/api/beliefs.go @@ -139,6 +139,10 @@ type BeliefDetail struct { Events []events.Event `json:"events"` Origin *events.Event `json:"origin,omitempty"` Assertions []epistemics.Assertion `json:"assertions"` // who stated this proposition, with what stance + // Score semantics: the raw number, its empirical reading where prediction + // outcomes allow one, and the strongest deterministic reason to doubt it. + Calibration beliefs.Calibrated `json:"calibration"` + PrimaryUncertainty string `json:"primary_uncertainty"` } // BeliefDetail assembles the detail; shared with the UI. @@ -164,6 +168,12 @@ func (s *Server) BeliefDetail(r *http.Request, mind, id ids.ID) (BeliefDetail, e if d.Assertions == nil { d.Assertions = []epistemics.Assertion{} } + if s.deps.Predictions != nil { + d.Calibration = s.deps.Predictions.CalibratedScore(ctx, mind, b.Confidence) + } else { + d.Calibration = beliefs.CalibrateWith(b.Confidence, nil) + } + d.PrimaryUncertainty = beliefs.PrimaryUncertainty(d.Breakdown, d.Evidence) d.Events, _ = s.deps.Events.DerivedFrom(ctx, "belief", string(id), 100) if !b.OriginEvent.IsZero() { if e, err := s.deps.Events.Get(ctx, b.OriginEvent); err == nil { diff --git a/internal/beliefs/confidence.go b/internal/beliefs/confidence.go index 6bfe7a4..89ea560 100644 --- a/internal/beliefs/confidence.go +++ b/internal/beliefs/confidence.go @@ -5,6 +5,7 @@ package beliefs import ( "encoding/json" + "fmt" "math" "sort" "strings" @@ -86,6 +87,94 @@ type Evidence struct { ProvenanceStatus string `json:"provenance_status,omitempty"` // recorded | legacy_unknown } +// ScoreMeaning states, on every breakdown, what the confidence number is. +// It is an uncalibrated epistemic score: the deterministic output of the +// evidence arithmetic, comparable between one mind's beliefs, and not a +// probability that the proposition is true. Only resolved predictions can +// say how a score has tracked reality; see Calibrated. +const ScoreMeaning = "uncalibrated epistemic score derived from recorded evidence; not a probability of truth" + +// Calibrated is the empirical reading of a score, when there is one: the +// hit rate of this mind's resolved predictions whose stated confidence fell +// in the same range. It is offered only above a minimum sample so that no +// false precision is shown. +type Calibrated struct { + Raw float64 `json:"raw"` + Estimate float64 `json:"estimate,omitempty"` + SampleSize int `json:"sample_size"` + Available bool `json:"available"` + Basis string `json:"basis"` + Reason string `json:"reason,omitempty"` +} + +// CalibrationBucket is one range of stated confidence with its outcomes. +type CalibrationBucket struct { + Low, High float64 + Decided int + HitRate float64 +} + +// MinCalibrationSample is the fewest decided predictions in a bucket before +// a calibrated estimate is shown. +const MinCalibrationSample = 20 + +// CalibrateWith maps a score onto the bucket containing it. Deterministic; +// the buckets come from prediction outcomes. +func CalibrateWith(score float64, buckets []CalibrationBucket) Calibrated { + c := Calibrated{Raw: round4(score), Basis: "resolved predictions of this mind with stated confidence in the same range"} + for _, b := range buckets { + if score >= b.Low && score < b.High { + c.SampleSize = b.Decided + if b.Decided >= MinCalibrationSample { + c.Available, c.Estimate = true, round4(b.HitRate) + } else { + c.Reason = fmt.Sprintf("insufficient data: %d decided prediction(s) in this range; %d needed", b.Decided, MinCalibrationSample) + } + return c + } + } + c.Reason = "no prediction outcomes recorded in this range" + return c +} + +// PrimaryUncertainty names, deterministically, the strongest reason to doubt +// a score: what an operator should look at first. +func PrimaryUncertainty(b Breakdown, ev []EvidenceRow) string { + var live, reported int + lowReading := false + for _, e := range ev { + if e.Retracted || e.Weight <= 0 || e.Direction == "against" { + continue + } + live++ + if e.Derivation == DerivationAssertion && strings.Contains(e.Note, "reported") { + reported++ + } + if strings.Contains(e.Note, "interpretation confidence") { + for _, low := range []string{"40%", "50%", "60%"} { + if strings.Contains(e.Note, low) { + lowReading = true + } + } + } + } + switch { + case live == 0 && b.Against == 0: + return "no live evidence" + case b.For > 0 && b.Against > 0 && b.Against >= 0.5*b.For: + return "conflicting evidence" + case live > 0 && reported == live: + return "second-hand only: nobody who saw it has told the mind" + case lowReading: + return "unclear reading of what was said" + case b.SourcesFor == 1: + return "evidence independence: a single source family" + case b.For < 0.3: + return "little evidence" + } + return "none dominant" +} + // Derivations. Primary derivations bring information from outside the // belief system; derived ones re-express information already present. const ( @@ -174,6 +263,7 @@ type FamilyShare struct { // Breakdown explains a confidence value. type Breakdown struct { Prior float64 `json:"prior"` + Meaning string `json:"meaning"` // what the number is and is not Families []FamilyShare `json:"families,omitempty"` // per independent-source group For float64 `json:"for"` // Σ weight×credibility, supporting Against float64 `json:"against"` // Σ weight×credibility, opposing @@ -197,7 +287,7 @@ type Breakdown struct { // re-express the episodes rather than corroborate them. Diversity counts // families, not rows (T4: poisoned memory; review pass item 2). func Compute(prior float64, ev []Evidence) Breakdown { - b := Breakdown{Prior: clampP(prior), Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"} + b := Breakdown{Prior: clampP(prior), Meaning: ScoreMeaning, Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"} type group struct { primary, derived []float64 items int diff --git a/internal/beliefs/confidence_test.go b/internal/beliefs/confidence_test.go index a1ff881..ca96f3e 100644 --- a/internal/beliefs/confidence_test.go +++ b/internal/beliefs/confidence_test.go @@ -1,6 +1,7 @@ package beliefs import ( + "strings" "testing" "github.com/andydixon/noema/pkg/concepts" @@ -125,3 +126,45 @@ func TestProvenanceGroupingPreventsInflation(t *testing.T) { t.Fatal(f) } } + +// TestCalibrationIsGatedAndDeterministic: a calibrated estimate appears only +// with enough decided predictions in the score's range, and the primary +// uncertainty names the right doubt. +func TestCalibrationIsGatedAndDeterministic(t *testing.T) { + buckets := []CalibrationBucket{{0, 0.5, 3, 0.33}, {0.5, 0.8, 25, 0.64}, {0.8, 1.01, 40, 0.9}} + if c := CalibrateWith(0.3, buckets); c.Available || c.SampleSize != 3 || c.Reason == "" { + t.Fatalf("small sample must not calibrate: %+v", c) + } + if c := CalibrateWith(0.59, buckets); !c.Available || c.Estimate != 0.64 || c.SampleSize != 25 { + t.Fatalf("calibrated: %+v", c) + } + if c := CalibrateWith(0.59, nil); c.Available || c.Reason == "" { + t.Fatalf("no buckets: %+v", c) + } + if b := Compute(0.5, nil); b.Meaning != ScoreMeaning { + t.Fatal("breakdown must carry its meaning") + } + rows := func(rs ...EvidenceRow) ([]Evidence, []EvidenceRow) { + var ev []Evidence + for _, r := range rs { + ev = append(ev, r.Evidence) + } + return ev, rs + } + one := EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:andy", Family: "actor:andy", Derivation: DerivationAssertion, Weight: 0.5, Credibility: 0.9}, Note: "andy asserted it (read with 85% interpretation confidence)"} + ev, rs := rows(one) + if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "evidence independence: a single source family" { + t.Fatal(u) + } + ev, rs = rows(EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:alice", Family: "actor:alice", Derivation: DerivationAssertion, Weight: 0.2, Credibility: 0.6}, Note: "andy reported that alice holds this (asserted, read with 75% interpretation confidence)"}) + if u := PrimaryUncertainty(Compute(0.5, ev), rs); !strings.HasPrefix(u, "second-hand") { + t.Fatal(u) + } + ev, rs = rows(one, EvidenceRow{Evidence: Evidence{Direction: "against", Kind: "operator", Source: "operator:x", Family: "operator:x", Weight: 0.5, Credibility: 0.9}}) + if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "conflicting evidence" { + t.Fatal(u) + } + if u := PrimaryUncertainty(Compute(0.5, nil), nil); u != "no live evidence" { + t.Fatal(u) + } +} diff --git a/internal/predictions/predictions.go b/internal/predictions/predictions.go index 660ab2c..29fd415 100644 --- a/internal/predictions/predictions.go +++ b/internal/predictions/predictions.go @@ -16,6 +16,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgxpool" + "github.com/andydixon/noema/internal/beliefs" "github.com/andydixon/noema/internal/storage" "github.com/andydixon/noema/pkg/events" "github.com/andydixon/noema/pkg/ids" @@ -454,3 +455,18 @@ func (s *Store) Stats(ctx context.Context, mind ids.ID) (map[string]int64, error } return out, rows.Err() } + +// CalibratedScore reads a belief score against this mind's whole prediction +// history. It is the only empirical calibration Noema offers, and it says +// so when the sample is too small. +func (s *Store) CalibratedScore(ctx context.Context, mind ids.ID, score float64) beliefs.Calibrated { + c, err := s.Calibrate(ctx, mind, time.Time{}) + if err != nil { + return beliefs.CalibrateWith(score, nil) + } + var buckets []beliefs.CalibrationBucket + for _, b := range c.Buckets { + buckets = append(buckets, beliefs.CalibrationBucket{Low: b.Low, High: b.High, Decided: b.Count, HitRate: b.HitRate}) + } + return beliefs.CalibrateWith(score, buckets) +} diff --git a/web/templates/pages/belief_detail.html b/web/templates/pages/belief_detail.html index eb95eaa..6f3f935 100644 --- a/web/templates/pages/belief_detail.html +++ b/web/templates/pages/belief_detail.html @@ -3,6 +3,9 @@

{{$m.DisplayName}} › beliefs

{{$b.Status}} {{$b.Proposition}}

+

Epistemic score {{conf $b.Confidence}} (uncalibrated: derived from recorded evidence, not a probability of truth) + · calibrated: {{if $d.Calibration.Available}}{{conf $d.Calibration.Estimate}} (hit rate of {{$d.Calibration.SampleSize}} decided predictions in this range){{else}}not available ({{$d.Calibration.Reason}}){{end}} + · main uncertainty: {{$d.PrimaryUncertainty}}

{{$b.Epistemic}} · version {{$b.Version}} · first {{date $b.FirstCreated}} · evaluated {{$b.Evaluations}}× · source {{$b.Source}}{{if $b.Subject}} · subject {{$b.Subject}}{{end}}{{if $b.PinnedStatus}} · status pinned by operator{{end}}

diff --git a/web/templates/pages/beliefs.html b/web/templates/pages/beliefs.html index 7f750e1..9bec3a8 100644 --- a/web/templates/pages/beliefs.html +++ b/web/templates/pages/beliefs.html @@ -15,7 +15,7 @@
{{if .Data.Beliefs}} - + {{range .Data.Beliefs}}
StatusPropositionConfidenceKindEvaluated
StatusPropositionScoreKindEvaluated
{{.Status}} {{.Proposition}}{{if .Subject}} {{.Subject}}{{end}}