Files
Andy Dixon 7d608c2783 feat(privacy): audited redaction that keeps history's shape and removes its content
Rows in append-only history are never deleted; their content can now be
made irrecoverable through one audited path. A redaction replaces the
content columns of an event, memory, message, model call, belief or
assertion with a fixed marker, keeps identifiers, links, timestamps,
scores and breakdowns, writes an append-only ledger row with a SHA-256 of
the removed content, appends a redaction event, optionally cascades to
derived rows and to the events that quote them, and scrubs stored snapshot
archives before re-signing them. noema_redact() is the only write the
append-only trigger admits, for UPDATE only, under a transaction-local flag
the application never sets; plain updates and all deletes stay refused.
Replay labels redacted stimuli instead of inventing them.

Cryptographic erasure was considered and deferred with reasons (ADR-0009);
docs/privacy.md defines retained, redacted, irrecoverable and deleted and
states what backups and earlier exports are not covered. Migration 0023.
2026-08-23 23:52:07 +01:00

478 lines
18 KiB
Go

// Package memory implements long-term memory: episodic, semantic, procedural
// and autobiographical records with explicit epistemic status, credibility,
// importance and decaying activation, plus deterministic fused retrieval
// (ADR-0003). Nothing here is deleted by decay.
package memory
import (
"context"
"encoding/json"
"errors"
"fmt"
"math"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/andydixon/noema/internal/embeddings"
"github.com/andydixon/noema/internal/storage"
"github.com/andydixon/noema/pkg/concepts"
"github.com/andydixon/noema/pkg/events"
"github.com/andydixon/noema/pkg/ids"
)
// Status of a memory record. Retracted and superseded memories stay
// retrievable for provenance but are excluded from normal recall.
type Status string
// Statuses.
const (
StatusActive Status = "active"
StatusRetracted Status = "retracted"
StatusSuperseded Status = "superseded"
StatusArchived Status = "archived"
)
// Memory is one long-term record.
type Memory struct {
ID ids.ID `json:"id"`
MindID ids.ID `json:"mind_id"`
Kind concepts.MemoryKind `json:"kind"`
Epistemic concepts.Epistemic `json:"epistemic"`
Status Status `json:"status"`
Content string `json:"content"`
Detail map[string]any `json:"detail"`
Source string `json:"source"`
Trust events.Trust `json:"trust"`
Credibility float64 `json:"credibility"`
Confidence float64 `json:"confidence"`
Importance float64 `json:"importance"`
Salience float64 `json:"salience"`
Activation float64 `json:"activation"`
Valence *float64 `json:"valence,omitempty"`
Arousal *float64 `json:"arousal,omitempty"`
OccurredAt time.Time `json:"occurred_at"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
RetrievalCount int `json:"retrieval_count"`
LastRetrievedAt *time.Time `json:"last_retrieved_at,omitempty"`
OriginEvent ids.ID `json:"origin_event,omitempty"`
Supersedes ids.ID `json:"supersedes,omitempty"`
RedactedAt *time.Time `json:"redacted_at,omitempty"`
}
// New describes a memory to store.
type New struct {
MindID ids.ID
Kind concepts.MemoryKind
Epistemic concepts.Epistemic
Content string
Detail map[string]any
Source string
Trust events.Trust
Confidence float64
Importance float64
Salience float64
Valence *float64
Arousal *float64
OccurredAt time.Time
OriginEvent ids.ID
Supersedes ids.ID
}
// Relationship is an edge between two memories.
type Relationship struct {
ID ids.ID `json:"id"`
FromMemory ids.ID `json:"from_memory"`
ToMemory ids.ID `json:"to_memory"`
Relation concepts.Relation `json:"relation"`
Confidence float64 `json:"confidence"`
Source string `json:"source"`
CreatedAt time.Time `json:"created_at"`
}
// ErrInvalid marks bad input.
var ErrInvalid = errors.New("invalid memory")
// Store persists memories.
type Store struct {
db *pgxpool.Pool
embedder embeddings.Embedder
vectors bool // memory_embeddings exists (pgvector available)
}
// NewStore creates a store and detects vector support.
func NewStore(ctx context.Context, db *pgxpool.Pool, embedder embeddings.Embedder) *Store {
s := &Store{db: db, embedder: embedder}
if embedder == nil {
s.embedder = embeddings.None{}
}
db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_embeddings')`).Scan(&s.vectors) //nolint:errcheck
return s
}
// VectorsEnabled reports whether pgvector storage is present.
func (s *Store) VectorsEnabled() bool { return s.vectors }
// SemanticEnabled reports whether semantic (embedding) retrieval can run.
func (s *Store) SemanticEnabled() bool { return s.vectors && s.embedder.Model() != "" }
// SetEmbedder swaps the embedder (providers are configured at runtime).
func (s *Store) SetEmbedder(e embeddings.Embedder) {
if e == nil {
e = embeddings.None{}
}
s.embedder = e
}
func (n *New) validate() error {
if n.MindID.IsZero() {
return fmt.Errorf("%w: mind id required", ErrInvalid)
}
if !n.Kind.Valid() {
return fmt.Errorf("%w: unknown kind %q", ErrInvalid, n.Kind)
}
if !n.Epistemic.Valid() {
return fmt.Errorf("%w: unknown epistemic kind %q", ErrInvalid, n.Epistemic)
}
n.Content = strings.TrimSpace(n.Content)
if n.Content == "" || len(n.Content) > 20000 {
return fmt.Errorf("%w: content required (max 20000 chars)", ErrInvalid)
}
if !n.Trust.Valid() {
n.Trust = events.TrustLow
}
n.Confidence, n.Importance, n.Salience = clamp(n.Confidence), clamp(n.Importance), clamp(n.Salience)
if n.OccurredAt.IsZero() {
n.OccurredAt = time.Now()
}
if n.Detail == nil {
n.Detail = map[string]any{}
}
return nil
}
func clamp(v float64) float64 {
if math.IsNaN(v) || v < 0 {
return 0
}
if v > 1 {
return 1
}
return v
}
// Create stores a memory using q (a pool or transaction). Embedding happens
// best-effort after the row exists so a slow provider never blocks encoding.
func (s *Store) Create(ctx context.Context, q storage.Querier, n New) (Memory, error) {
if err := n.validate(); err != nil {
return Memory{}, err
}
m := Memory{ID: ids.New(), MindID: n.MindID, Kind: n.Kind, Epistemic: n.Epistemic, Status: StatusActive, Content: n.Content, Detail: n.Detail, Source: n.Source,
Trust: n.Trust, Credibility: n.Trust.Credibility(), Confidence: n.Confidence, Importance: n.Importance, Salience: n.Salience, Activation: 1,
Valence: n.Valence, Arousal: n.Arousal, OccurredAt: n.OccurredAt, CreatedAt: time.Now(), UpdatedAt: time.Now(), OriginEvent: n.OriginEvent, Supersedes: n.Supersedes}
detail, _ := json.Marshal(n.Detail)
_, err := q.Exec(ctx, `INSERT INTO memories (id, mind_id, kind, epistemic, content, detail, source, trust, credibility, confidence, importance, salience, valence, arousal, occurred_at, origin_event, supersedes)
VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)`,
m.ID, m.MindID, string(m.Kind), string(m.Epistemic), m.Content, detail, m.Source, string(m.Trust), m.Credibility, m.Confidence, m.Importance, m.Salience, m.Valence, m.Arousal, m.OccurredAt, nullID(m.OriginEvent), nullID(m.Supersedes))
if err != nil {
return Memory{}, storage.Translate(err)
}
if !n.Supersedes.IsZero() {
q.Exec(ctx, `UPDATE memories SET status='superseded', updated_at=now() WHERE id=$1 AND mind_id=$2`, n.Supersedes, n.MindID) //nolint:errcheck
}
return m, nil
}
// Embed computes and stores the embedding for a memory if a provider exists.
// Safe to call from a background task; errors are returned for logging only.
func (s *Store) Embed(ctx context.Context, m Memory) error {
if !s.SemanticEnabled() {
return nil
}
vecs, err := s.embedder.Embed(ctx, []string{m.Content})
if err != nil || len(vecs) != 1 {
return fmt.Errorf("embed: %w", err)
}
_, err = s.db.Exec(ctx, `INSERT INTO memory_embeddings (memory_id, model, dims, embedding) VALUES ($1,$2,$3,$4::vector)
ON CONFLICT (memory_id, model) DO UPDATE SET embedding=excluded.embedding, dims=excluded.dims, created_at=now()`, m.ID, s.embedder.Model(), len(vecs[0]), embeddings.Format(vecs[0]))
return err
}
// EmbedMissing embeds up to limit memories lacking a vector for the current model.
func (s *Store) EmbedMissing(ctx context.Context, limit int) (int, error) {
if !s.SemanticEnabled() {
return 0, nil
}
rows, err := s.db.Query(ctx, `SELECT `+cols+` FROM memories m WHERE status='active' AND NOT EXISTS (SELECT 1 FROM memory_embeddings e WHERE e.memory_id=m.id AND e.model=$1) ORDER BY created_at DESC LIMIT $2`, s.embedder.Model(), limit)
if err != nil {
return 0, err
}
var list []Memory
for rows.Next() {
m, err := scan(rows)
if err != nil {
rows.Close()
return 0, err
}
list = append(list, m)
}
rows.Close()
n := 0
for _, m := range list {
if err := s.Embed(ctx, m); err != nil {
return n, err
}
n++
}
return n, nil
}
func nullID(id ids.ID) *string {
if id.IsZero() {
return nil
}
s := string(id)
return &s
}
const cols = `m.id, m.mind_id, m.kind, m.epistemic, m.status, m.content, m.detail, m.source, m.trust, m.credibility, m.confidence, m.importance, m.salience, m.activation, m.valence, m.arousal, m.occurred_at, m.created_at, m.updated_at, m.retrieval_count, m.last_retrieved_at, m.origin_event, m.supersedes, m.redacted_at`
func scan(row pgx.Row) (Memory, error) {
var m Memory
var kind, epi, status, trust string
var detail []byte
var origin, supersedes *string
err := row.Scan(&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt)
if err != nil {
return m, storage.Translate(err)
}
m.Kind, m.Epistemic, m.Status, m.Trust = concepts.MemoryKind(kind), concepts.Epistemic(epi), Status(status), events.Trust(trust)
json.Unmarshal(detail, &m.Detail) //nolint:errcheck
if m.Detail == nil {
m.Detail = map[string]any{}
}
if origin != nil {
m.OriginEvent = ids.ID(*origin)
}
if supersedes != nil {
m.Supersedes = ids.ID(*supersedes)
}
return m, nil
}
// Get fetches a memory scoped to a mind (cross-mind reads are impossible by construction).
func (s *Store) Get(ctx context.Context, mind, id ids.ID) (Memory, error) {
return scan(s.db.QueryRow(ctx, `SELECT `+cols+` FROM memories m WHERE m.id=$1 AND m.mind_id=$2`, id, mind))
}
// Filter selects memories for listing.
type Filter struct {
Kinds []concepts.MemoryKind
Epistemic []concepts.Epistemic
Statuses []Status // default: active
MinConfidence float64
MinImportance float64
Source string
Since, Until time.Time
Search string // plain text search (websearch syntax)
Before time.Time
Limit int
}
// List returns memories newest first (by occurred_at) with keyset paging on Before.
func (s *Store) List(ctx context.Context, mind ids.ID, f Filter) ([]Memory, error) {
where := []string{"m.mind_id = $1"}
args := []any{mind}
arg := func(v any) string { args = append(args, v); return fmt.Sprintf("$%d", len(args)) }
if len(f.Kinds) > 0 {
where = append(where, "m.kind = ANY("+arg(strs(f.Kinds))+")")
}
if len(f.Epistemic) > 0 {
where = append(where, "m.epistemic = ANY("+arg(strs(f.Epistemic))+")")
}
if len(f.Statuses) > 0 {
where = append(where, "m.status = ANY("+arg(strs(f.Statuses))+")")
} else {
where = append(where, "m.status = 'active'")
}
if f.MinConfidence > 0 {
where = append(where, "m.confidence >= "+arg(f.MinConfidence))
}
if f.MinImportance > 0 {
where = append(where, "m.importance >= "+arg(f.MinImportance))
}
if f.Source != "" {
where = append(where, "m.source = "+arg(f.Source))
}
if !f.Since.IsZero() {
where = append(where, "m.occurred_at >= "+arg(f.Since))
}
if !f.Until.IsZero() {
where = append(where, "m.occurred_at <= "+arg(f.Until))
}
if !f.Before.IsZero() {
where = append(where, "m.occurred_at < "+arg(f.Before))
}
if q := strings.TrimSpace(f.Search); q != "" {
where = append(where, "m.search @@ websearch_to_tsquery('english', "+arg(q)+")")
}
limit := f.Limit
if limit <= 0 || limit > 500 {
limit = 100
}
rows, err := s.db.Query(ctx, `SELECT `+cols+` FROM memories m WHERE `+strings.Join(where, " AND ")+` ORDER BY m.occurred_at DESC, m.id DESC LIMIT `+arg(limit), args...)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Memory
for rows.Next() {
m, err := scan(rows)
if err != nil {
return nil, err
}
out = append(out, m)
}
return out, rows.Err()
}
func strs[T ~string](xs []T) []string {
out := make([]string, len(xs))
for i, x := range xs {
out[i] = string(x)
}
return out
}
// SetStatus changes status (retract, archive). It never deletes.
func (s *Store) SetStatus(ctx context.Context, mind, id ids.ID, st Status) error {
tag, err := s.db.Exec(ctx, `UPDATE memories SET status=$3, updated_at=now() WHERE id=$1 AND mind_id=$2`, id, mind, string(st))
if err == nil && tag.RowsAffected() == 0 {
return storage.ErrNotFound
}
return err
}
// UpdateConfidence sets a new confidence (used by consolidation and corrections).
func (s *Store) UpdateConfidence(ctx context.Context, q storage.Querier, mind, id ids.ID, confidence float64) error {
tag, err := q.Exec(ctx, `UPDATE memories SET confidence=$3, updated_at=now() WHERE id=$1 AND mind_id=$2`, id, mind, clamp(confidence))
if err == nil && tag.RowsAffected() == 0 {
return storage.ErrNotFound
}
return err
}
// Reinforce records a retrieval: count, timestamp and activation towards 1.
func (s *Store) Reinforce(ctx context.Context, q storage.Querier, mind, id ids.ID) error {
_, err := q.Exec(ctx, `UPDATE memories SET retrieval_count = retrieval_count + 1, last_retrieved_at = now(), activation = LEAST(1, activation + (1 - activation) * 0.5), updated_at = now() WHERE id=$1 AND mind_id=$2`, id, mind)
return err
}
// Decay lowers activation of memories not retrieved recently. Importance slows
// decay; nothing is deleted. Returns the number of rows touched.
func (s *Store) Decay(ctx context.Context, mind ids.ID, halfLife time.Duration) (int64, error) {
if halfLife <= 0 {
halfLife = 14 * 24 * time.Hour
}
// activation *= 0.5 ^ (age / (halfLife * (1 + importance)))
tag, err := s.db.Exec(ctx, `UPDATE memories SET activation = GREATEST(0.01, activation * power(0.5, EXTRACT(EPOCH FROM (now() - coalesce(last_retrieved_at, created_at))) / ($2 * (1 + importance)))), updated_at = now()
WHERE mind_id=$1 AND status='active' AND activation > 0.01 AND coalesce(last_retrieved_at, created_at) < now() - interval '1 hour'`, mind, halfLife.Seconds())
if err != nil {
return 0, err
}
return tag.RowsAffected(), nil
}
// Relate adds an edge between two memories of the same mind.
func (s *Store) Relate(ctx context.Context, q storage.Querier, mind, from, to ids.ID, rel concepts.Relation, confidence float64, source string) (Relationship, error) {
if !rel.Valid() {
return Relationship{}, fmt.Errorf("%w: unknown relation %q", ErrInvalid, rel)
}
if from == to {
return Relationship{}, fmt.Errorf("%w: self relation", ErrInvalid)
}
r := Relationship{ID: ids.New(), FromMemory: from, ToMemory: to, Relation: rel, Confidence: clamp(confidence), Source: source, CreatedAt: time.Now()}
// Both endpoints must belong to the mind; the subquery enforces it.
tag, err := q.Exec(ctx, `INSERT INTO memory_relationships (id, mind_id, from_memory, to_memory, relation, confidence, source)
SELECT $1, $2, $3, $4, $5, $6, $7 WHERE (SELECT count(*) FROM memories WHERE id IN ($3,$4) AND mind_id=$2) = 2
ON CONFLICT (from_memory, to_memory, relation) DO UPDATE SET confidence = GREATEST(memory_relationships.confidence, excluded.confidence)`,
r.ID, mind, from, to, string(rel), r.Confidence, source)
if err != nil {
return Relationship{}, err
}
if tag.RowsAffected() == 0 {
return Relationship{}, fmt.Errorf("%w: memories not found in mind", ErrInvalid)
}
return r, nil
}
// Related lists edges touching a memory, with the other endpoint loaded.
type Related struct {
Relationship
Other Memory `json:"other"`
Direction string `json:"direction"` // out | in
}
// Relationships returns edges in both directions.
func (s *Store) Relationships(ctx context.Context, mind, id ids.ID) ([]Related, error) {
rows, err := s.db.Query(ctx, `SELECT r.id, r.from_memory, r.to_memory, r.relation, r.confidence, r.source, r.created_at, CASE WHEN r.from_memory=$1 THEN 'out' ELSE 'in' END, `+cols+`
FROM memory_relationships r JOIN memories m ON m.id = CASE WHEN r.from_memory=$1 THEN r.to_memory ELSE r.from_memory END
WHERE r.mind_id=$2 AND (r.from_memory=$1 OR r.to_memory=$1) ORDER BY r.created_at`, id, mind)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Related
for rows.Next() {
var r Related
var rel string
var kind, epi, status, trust string
var detail []byte
var origin, supersedes *string
m := &r.Other
if err := rows.Scan(&r.ID, &r.FromMemory, &r.ToMemory, &rel, &r.Confidence, &r.Source, &r.CreatedAt, &r.Direction,
&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt); err != nil {
return nil, err
}
r.Relation = concepts.Relation(rel)
m.Kind, m.Epistemic, m.Status, m.Trust = concepts.MemoryKind(kind), concepts.Epistemic(epi), Status(status), events.Trust(trust)
json.Unmarshal(detail, &m.Detail) //nolint:errcheck
if origin != nil {
m.OriginEvent = ids.ID(*origin)
}
out = append(out, r)
}
return out, rows.Err()
}
// Stats summarises a mind's memory for overview pages.
type Stats struct {
Total int64 `json:"total"`
ByKind map[string]int64 `json:"by_kind"`
}
// Stats counts active memories by kind.
func (s *Store) Stats(ctx context.Context, mind ids.ID) (Stats, error) {
st := Stats{ByKind: map[string]int64{}}
rows, err := s.db.Query(ctx, `SELECT kind, count(*) FROM memories WHERE mind_id=$1 AND status='active' GROUP BY kind`, mind)
if err != nil {
return st, err
}
defer rows.Close()
for rows.Next() {
var k string
var n int64
if err := rows.Scan(&k, &n); err != nil {
return st, err
}
st.ByKind[k] = n
st.Total += n
}
return st, rows.Err()
}