andy/noema
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(beliefs): state what a score means; gated calibration; primary uncertainty

Browse Source 
Every breakdown now carries its meaning: an uncalibrated epistemic score
derived from recorded evidence, not a probability of truth. Belief detail
adds a calibrated estimate read from the mind's resolved predictions when
at least twenty were decided in the score's range, and otherwise says why
none is shown; and names the primary uncertainty deterministically (no live
evidence, conflicting evidence, second-hand only, unclear reading, a single
source family, little evidence). The belief list heading becomes Score.
This commit is contained in:
Andy Dixon
2026-08-09 19:14:53 +01:00
parent 08a265f0fa
commit b661496128
8 changed files with 167 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/beliefs/confidence.go
+91 -1
View File
@@ -5,6 +5,7 @@ package beliefs
import (
"encoding/json"
"fmt"
"math"
"sort"
"strings"
@@ -86,6 +87,94 @@ type Evidence struct {
ProvenanceStatus string `json:"provenance_status,omitempty"` // recorded | legacy_unknown
}
// ScoreMeaning states, on every breakdown, what the confidence number is.
// It is an uncalibrated epistemic score: the deterministic output of the
// evidence arithmetic, comparable between one mind's beliefs, and not a
// probability that the proposition is true. Only resolved predictions can
// say how a score has tracked reality; see Calibrated.
const ScoreMeaning = "uncalibrated epistemic score derived from recorded evidence; not a probability of truth"
// Calibrated is the empirical reading of a score, when there is one: the
// hit rate of this mind's resolved predictions whose stated confidence fell
// in the same range. It is offered only above a minimum sample so that no
// false precision is shown.
type Calibrated struct {
Raw float64 `json:"raw"`
Estimate float64 `json:"estimate,omitempty"`
SampleSize int `json:"sample_size"`
Available bool `json:"available"`
Basis string `json:"basis"`
Reason string `json:"reason,omitempty"`
}
// CalibrationBucket is one range of stated confidence with its outcomes.
type CalibrationBucket struct {
Low, High float64
Decided int
HitRate float64
}
// MinCalibrationSample is the fewest decided predictions in a bucket before
// a calibrated estimate is shown.
const MinCalibrationSample = 20
// CalibrateWith maps a score onto the bucket containing it. Deterministic;
// the buckets come from prediction outcomes.
func CalibrateWith(score float64, buckets []CalibrationBucket) Calibrated {
c := Calibrated{Raw: round4(score), Basis: "resolved predictions of this mind with stated confidence in the same range"}
for _, b := range buckets {
if score >= b.Low && score < b.High {
c.SampleSize = b.Decided
if b.Decided >= MinCalibrationSample {
c.Available, c.Estimate = true, round4(b.HitRate)
} else {
c.Reason = fmt.Sprintf("insufficient data: %d decided prediction(s) in this range; %d needed", b.Decided, MinCalibrationSample)
}
return c
}
}
c.Reason = "no prediction outcomes recorded in this range"
return c
}
// PrimaryUncertainty names, deterministically, the strongest reason to doubt
// a score: what an operator should look at first.
func PrimaryUncertainty(b Breakdown, ev []EvidenceRow) string {
var live, reported int
lowReading := false
for _, e := range ev {
if e.Retracted || e.Weight <= 0 || e.Direction == "against" {
continue
}
live++
if e.Derivation == DerivationAssertion && strings.Contains(e.Note, "reported") {
reported++
}
if strings.Contains(e.Note, "interpretation confidence") {
for _, low := range []string{"40%", "50%", "60%"} {
if strings.Contains(e.Note, low) {
lowReading = true
}
}
}
}
switch {
case live == 0 && b.Against == 0:
return "no live evidence"
case b.For > 0 && b.Against > 0 && b.Against >= 0.5*b.For:
return "conflicting evidence"
case live > 0 && reported == live:
return "second-hand only: nobody who saw it has told the mind"
case lowReading:
return "unclear reading of what was said"
case b.SourcesFor == 1:
return "evidence independence: a single source family"
case b.For < 0.3:
return "little evidence"
}
return "none dominant"
}
// Derivations. Primary derivations bring information from outside the
// belief system; derived ones re-express information already present.
const (
@@ -174,6 +263,7 @@ type FamilyShare struct {
// Breakdown explains a confidence value.
type Breakdown struct {
Prior float64 `json:"prior"`
Meaning string `json:"meaning"` // what the number is and is not
Families []FamilyShare `json:"families,omitempty"` // per independent-source group
For float64 `json:"for"` // Σ weight×credibility, supporting
Against float64 `json:"against"` // Σ weight×credibility, opposing
@@ -197,7 +287,7 @@ type Breakdown struct {
// re-express the episodes rather than corroborate them. Diversity counts
// families, not rows (T4: poisoned memory; review pass item 2).
func Compute(prior float64, ev []Evidence) Breakdown {
b := Breakdown{Prior: clampP(prior), Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"}
b := Breakdown{Prior: clampP(prior), Meaning: ScoreMeaning, Formula: "confidence = sigmoid(logit(prior) + slope × (for × diversity − against)); for/against sum per-family contributions (strongest primary + ¼ of the rest, capped at 1.5×; derived items only when no primary exists)"}
type group struct {
primary, derived []float64
items int
internal/beliefs/confidence_test.go
+43
View File
@@ -1,6 +1,7 @@
package beliefs
import (
"strings"
"testing"
"github.com/andydixon/noema/pkg/concepts"
@@ -125,3 +126,45 @@ func TestProvenanceGroupingPreventsInflation(t *testing.T) {
t.Fatal(f)
}
}
// TestCalibrationIsGatedAndDeterministic: a calibrated estimate appears only
// with enough decided predictions in the score's range, and the primary
// uncertainty names the right doubt.
func TestCalibrationIsGatedAndDeterministic(t *testing.T) {
buckets := []CalibrationBucket{{0, 0.5, 3, 0.33}, {0.5, 0.8, 25, 0.64}, {0.8, 1.01, 40, 0.9}}
if c := CalibrateWith(0.3, buckets); c.Available || c.SampleSize != 3 || c.Reason == "" {
t.Fatalf("small sample must not calibrate: %+v", c)
}
if c := CalibrateWith(0.59, buckets); !c.Available || c.Estimate != 0.64 || c.SampleSize != 25 {
t.Fatalf("calibrated: %+v", c)
}
if c := CalibrateWith(0.59, nil); c.Available || c.Reason == "" {
t.Fatalf("no buckets: %+v", c)
}
if b := Compute(0.5, nil); b.Meaning != ScoreMeaning {
t.Fatal("breakdown must carry its meaning")
}
rows := func(rs ...EvidenceRow) ([]Evidence, []EvidenceRow) {
var ev []Evidence
for _, r := range rs {
ev = append(ev, r.Evidence)
}
return ev, rs
}
one := EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:andy", Family: "actor:andy", Derivation: DerivationAssertion, Weight: 0.5, Credibility: 0.9}, Note: "andy asserted it (read with 85% interpretation confidence)"}
ev, rs := rows(one)
if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "evidence independence: a single source family" {
t.Fatal(u)
}
ev, rs = rows(EvidenceRow{Evidence: Evidence{Direction: "for", Kind: "assertion", Source: "asserted-by:alice", Family: "actor:alice", Derivation: DerivationAssertion, Weight: 0.2, Credibility: 0.6}, Note: "andy reported that alice holds this (asserted, read with 75% interpretation confidence)"})
if u := PrimaryUncertainty(Compute(0.5, ev), rs); !strings.HasPrefix(u, "second-hand") {
t.Fatal(u)
}
ev, rs = rows(one, EvidenceRow{Evidence: Evidence{Direction: "against", Kind: "operator", Source: "operator:x", Family: "operator:x", Weight: 0.5, Credibility: 0.9}})
if u := PrimaryUncertainty(Compute(0.5, ev), rs); u != "conflicting evidence" {
t.Fatal(u)
}
if u := PrimaryUncertainty(Compute(0.5, nil), nil); u != "no live evidence" {
t.Fatal(u)
}
}