feat(privacy): audited redaction that keeps history's shape and removes its content

Rows in append-only history are never deleted; their content can now be
made irrecoverable through one audited path. A redaction replaces the
content columns of an event, memory, message, model call, belief or
assertion with a fixed marker, keeps identifiers, links, timestamps,
scores and breakdowns, writes an append-only ledger row with a SHA-256 of
the removed content, appends a redaction event, optionally cascades to
derived rows and to the events that quote them, and scrubs stored snapshot
archives before re-signing them. noema_redact() is the only write the
append-only trigger admits, for UPDATE only, under a transaction-local flag
the application never sets; plain updates and all deletes stay refused.
Replay labels redacted stimuli instead of inventing them.

Cryptographic erasure was considered and deferred with reasons (ADR-0009);
docs/privacy.md defines retained, redacted, irrecoverable and deleted and
states what backups and earlier exports are not covered. Migration 0023.
This commit is contained in:
Andy Dixon
2026-08-23 23:52:07 +01:00
parent 9eed6e2a18
commit 7d608c2783
33 changed files with 1245 additions and 45 deletions
+2
View File
@@ -34,6 +34,7 @@ import (
"github.com/andydixon/noema/internal/notifications"
"github.com/andydixon/noema/internal/observability"
"github.com/andydixon/noema/internal/predictions"
"github.com/andydixon/noema/internal/privacy"
"github.com/andydixon/noema/internal/reflection"
"github.com/andydixon/noema/internal/scheduler"
"github.com/andydixon/noema/internal/security/secrets"
@@ -65,6 +66,7 @@ func testServer(t *testing.T) (*httptest.Server, string) {
deps.Dreaming = &dreaming.Service{DB: pool, Memory: deps.Memory, Beliefs: deps.Beliefs, Events: deps.Events, Log: log}
deps.Snapshots = &snapshots.Store{DB: pool, Events: deps.Events, Minds: deps.Minds}
deps.Assertions = epistemics.NewStore(pool)
deps.Privacy = &privacy.Service{DB: pool, Events: deps.Events, Audit: deps.Audit, Snapshots: deps.Snapshots, Log: log}
deps.Narrative = narrative.NewStore(pool)
deps.Notifications = notifications.NewStore(pool)
deps.Consolidation = &consolidation.Service{DB: pool, Memory: deps.Memory, Beliefs: deps.Beliefs, Events: deps.Events, Minds: deps.Minds, Log: log}
+117
View File
@@ -6345,6 +6345,123 @@
}
}
},
"/api/v1/minds/{mind}/redactions": {
"get": {
"tags": [
"minds"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"401": {
"$ref": "#/components/responses/Error"
},
"403": {
"$ref": "#/components/responses/Error"
},
"404": {
"$ref": "#/components/responses/Error"
},
"201": {
"description": "Redacted; irreversible"
}
},
"parameters": [
{
"name": "mind",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"operationId": "redactionsList",
"summary": "list the redaction ledger",
"x-required-role": "operator"
},
"post": {
"tags": [
"minds"
],
"responses": {
"200": {
"description": "OK",
"content": {
"application/json": {
"schema": {
"type": "object"
}
}
}
},
"401": {
"$ref": "#/components/responses/Error"
},
"403": {
"$ref": "#/components/responses/Error"
},
"404": {
"$ref": "#/components/responses/Error"
},
"201": {
"description": "Redacted; irreversible"
}
},
"parameters": [
{
"name": "mind",
"in": "path",
"required": true,
"schema": {
"type": "string"
}
}
],
"operationId": "redactionCreate",
"summary": "redact a row's content irreversibly (event, memory, message, llm_call, belief, assertion), optionally cascading to derived rows",
"x-required-role": "administrator",
"requestBody": {
"content": {
"application/json": {
"schema": {
"type": "object",
"properties": {
"kind": {
"type": "string"
},
"id": {
"type": "string"
},
"reason": {
"type": "string"
},
"policy": {
"type": "string"
},
"cascade": {
"type": "boolean"
}
},
"required": [
"kind",
"id",
"reason"
]
}
}
}
}
}
},
"/api/v1/minds/{mind}/reflections": {
"get": {
"operationId": "reflectionsList",
+73
View File
@@ -0,0 +1,73 @@
package api
import (
"errors"
"net/http"
"github.com/andydixon/noema/internal/auth"
"github.com/andydixon/noema/internal/privacy"
"github.com/andydixon/noema/pkg/ids"
)
// privacyRoutes: the audited redaction path. Administrators only; every
// call is irreversible by design and says so in its response.
func (s *Server) privacyRoutes() {
m := s.mux
m.Handle("GET /api/v1/minds/{mind}/redactions", require(auth.RoleOperator, s.redactionsList))
m.Handle("POST /api/v1/minds/{mind}/redactions", require(auth.RoleAdministrator, s.redactionCreate))
}
func (s *Server) redactionsList(w http.ResponseWriter, r *http.Request) {
m, ok := s.mind(w, r)
if !ok {
return
}
list, err := s.deps.Privacy.List(r.Context(), m.ID, 200)
if err != nil {
fail(w, r, err)
return
}
if list == nil {
list = []privacy.Redaction{}
}
writeJSON(w, http.StatusOK, map[string]any{"items": list})
}
// RedactionInput is the request body (API and UI form).
type RedactionInput struct {
Kind string `json:"kind"`
ID ids.ID `json:"id"`
Reason string `json:"reason"`
Policy string `json:"policy"`
Cascade bool `json:"cascade"`
}
// Redact performs a redaction; shared with the UI.
func (s *Server) Redact(r *http.Request, mind ids.ID, in RedactionInput, by string) (privacy.Redaction, []privacy.Redaction, error) {
p, c, err := s.deps.Privacy.Redact(r.Context(), privacy.Request{MindID: mind, Kind: privacy.Kind(in.Kind), ID: in.ID, Reason: in.Reason, Policy: in.Policy, Cascade: in.Cascade, By: by})
switch {
case errors.Is(err, privacy.ErrInvalid):
return p, c, ValidationError(err.Error())
case errors.Is(err, privacy.ErrAlreadyRedacted):
return p, c, ValidationError("already redacted")
}
return p, c, err
}
func (s *Server) redactionCreate(w http.ResponseWriter, r *http.Request) {
m, ok := s.mind(w, r)
if !ok {
return
}
var in RedactionInput
if err := decode(r, &in); err != nil {
fail(w, r, err)
return
}
p, c, err := s.Redact(r, m.ID, in, actor(r))
if err != nil {
fail(w, r, err)
return
}
writeJSON(w, http.StatusCreated, map[string]any{"redaction": p, "cascaded": c, "note": "content is irrecoverable; the row, its links and a hash of the removed content remain"})
}
+1
View File
@@ -80,6 +80,7 @@ func (s *Server) routes() {
s.dreamRoutes()
s.studyRoutes()
s.assertionRoutes()
s.privacyRoutes()
m.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) {
writeError(w, r, http.StatusNotFound, "not_found", "no such endpoint")
+2
View File
@@ -34,6 +34,7 @@ import (
"github.com/andydixon/noema/internal/notifications"
"github.com/andydixon/noema/internal/observability"
"github.com/andydixon/noema/internal/predictions"
"github.com/andydixon/noema/internal/privacy"
"github.com/andydixon/noema/internal/reflection"
"github.com/andydixon/noema/internal/replay"
"github.com/andydixon/noema/internal/security/secrets"
@@ -54,6 +55,7 @@ type Deps struct {
Memory *memory.Store
Beliefs *beliefs.Store
Assertions *epistemics.Store
Privacy *privacy.Service
Conversations *conversation.Store
Goals *goals.Store
Motivation *motivation.Store
+5 -4
View File
@@ -43,6 +43,7 @@ type Belief struct {
ObservedAt *time.Time `json:"observed_at,omitempty"`
ValidFrom *time.Time `json:"valid_from,omitempty"`
ValidUntil *time.Time `json:"valid_until,omitempty"`
RedactedAt *time.Time `json:"redacted_at,omitempty"`
}
// EvidenceRow is stored evidence.
@@ -301,14 +302,14 @@ func nullID(id ids.ID) *string {
return &s
}
const cols = `b.id, b.mind_id, b.proposition, b.subject, b.epistemic, b.status, b.confidence, b.prior, b.version, b.source, b.detail, b.origin_event, b.superseded_by, b.pinned_status, b.first_created, b.last_evaluated, b.evaluations, b.temporal_scope, b.observed_at, b.valid_from, b.valid_until`
const cols = `b.id, b.mind_id, b.proposition, b.subject, b.epistemic, b.status, b.confidence, b.prior, b.version, b.source, b.detail, b.origin_event, b.superseded_by, b.pinned_status, b.first_created, b.last_evaluated, b.evaluations, b.temporal_scope, b.observed_at, b.valid_from, b.valid_until, b.redacted_at`
func scan(row pgx.Row) (Belief, error) {
var b Belief
var epi, status string
var detail []byte
var origin, superseded, pinned *string
if err := row.Scan(&b.ID, &b.MindID, &b.Proposition, &b.Subject, &epi, &status, &b.Confidence, &b.Prior, &b.Version, &b.Source, &detail, &origin, &superseded, &pinned, &b.FirstCreated, &b.LastEvaluated, &b.Evaluations, &b.TemporalScope, &b.ObservedAt, &b.ValidFrom, &b.ValidUntil); err != nil {
if err := row.Scan(&b.ID, &b.MindID, &b.Proposition, &b.Subject, &epi, &status, &b.Confidence, &b.Prior, &b.Version, &b.Source, &detail, &origin, &superseded, &pinned, &b.FirstCreated, &b.LastEvaluated, &b.Evaluations, &b.TemporalScope, &b.ObservedAt, &b.ValidFrom, &b.ValidUntil, &b.RedactedAt); err != nil {
return b, storage.Translate(err)
}
b.Epistemic, b.Status = concepts.Epistemic(epi), Status(status)
@@ -793,7 +794,7 @@ func (s *Store) Links(ctx context.Context, mind, id ids.ID) ([]Link, error) {
var detail []byte
var origin, superseded, pinned *string
if err := rows.Scan(&l.ID, &l.FromID, &l.ToID, &l.Relation, &l.Confidence, &l.CreatedAt,
&other.ID, &other.MindID, &other.Proposition, &other.Subject, &epi, &status, &other.Confidence, &other.Prior, &other.Version, &other.Source, &detail, &origin, &superseded, &pinned, &other.FirstCreated, &other.LastEvaluated, &other.Evaluations, &other.TemporalScope, &other.ObservedAt, &other.ValidFrom, &other.ValidUntil); err != nil {
&other.ID, &other.MindID, &other.Proposition, &other.Subject, &epi, &status, &other.Confidence, &other.Prior, &other.Version, &other.Source, &detail, &origin, &superseded, &pinned, &other.FirstCreated, &other.LastEvaluated, &other.Evaluations, &other.TemporalScope, &other.ObservedAt, &other.ValidFrom, &other.ValidUntil, &other.RedactedAt); err != nil {
return nil, err
}
other.Epistemic, other.Status = concepts.Epistemic(epi), Status(status)
@@ -919,7 +920,7 @@ func (s *Store) RecentChanges(ctx context.Context, mind ids.ID, since time.Time,
var detail []byte
var origin, superseded, pinned *string
b := &c.Belief
if err := rows.Scan(&b.ID, &b.MindID, &b.Proposition, &b.Subject, &epi, &status, &b.Confidence, &b.Prior, &b.Version, &b.Source, &detail, &origin, &superseded, &pinned, &b.FirstCreated, &b.LastEvaluated, &b.Evaluations, &b.TemporalScope, &b.ObservedAt, &b.ValidFrom, &b.ValidUntil, &fromStatus, &c.FromConf, &c.Reason, &c.At); err != nil {
if err := rows.Scan(&b.ID, &b.MindID, &b.Proposition, &b.Subject, &epi, &status, &b.Confidence, &b.Prior, &b.Version, &b.Source, &detail, &origin, &superseded, &pinned, &b.FirstCreated, &b.LastEvaluated, &b.Evaluations, &b.TemporalScope, &b.ObservedAt, &b.ValidFrom, &b.ValidUntil, &b.RedactedAt, &fromStatus, &c.FromConf, &c.Reason, &c.At); err != nil {
return nil, err
}
b.Epistemic, b.Status, c.FromStatus = concepts.Epistemic(epi), Status(status), Status(fromStatus)
+5 -4
View File
@@ -45,6 +45,7 @@ type Message struct {
Intent json.RawMessage `json:"intent,omitempty"`
LLMCallID ids.ID `json:"llm_call_id,omitempty"`
CreatedAt time.Time `json:"created_at"`
RedactedAt *time.Time `json:"redacted_at,omitempty"`
}
// ErrInvalid marks bad input.
@@ -156,7 +157,7 @@ func (s *Store) Messages(ctx context.Context, mind, conv ids.ID, limit int) ([]M
if limit <= 0 || limit > 1000 {
limit = 200
}
rows, err := s.db.Query(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at FROM messages
rows, err := s.db.Query(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at, redacted_at FROM messages
WHERE conversation_id=$1 AND mind_id=$2 ORDER BY created_at DESC LIMIT $3`, conv, mind, limit)
if err != nil {
return nil, err
@@ -180,7 +181,7 @@ func scanMessage(row pgx.Row) (Message, error) {
var m Message
var ev, reply, call *string
var intent *json.RawMessage
if err := row.Scan(&m.ID, &m.ConversationID, &m.MindID, &m.Role, &m.Actor, &m.Content, &ev, &reply, &intent, &call, &m.CreatedAt); err != nil {
if err := row.Scan(&m.ID, &m.ConversationID, &m.MindID, &m.Role, &m.Actor, &m.Content, &ev, &reply, &intent, &call, &m.CreatedAt, &m.RedactedAt); err != nil {
return m, storage.Translate(err)
}
if ev != nil {
@@ -200,7 +201,7 @@ func scanMessage(row pgx.Row) (Message, error) {
// Reply returns the mind's reply to a user message, if any.
func (s *Store) Reply(ctx context.Context, mind, userMessage ids.ID) (Message, error) {
return scanMessage(s.db.QueryRow(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at FROM messages WHERE reply_to=$1 AND mind_id=$2 ORDER BY created_at LIMIT 1`, userMessage, mind))
return scanMessage(s.db.QueryRow(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at, redacted_at FROM messages WHERE reply_to=$1 AND mind_id=$2 ORDER BY created_at LIMIT 1`, userMessage, mind))
}
// Recent returns the last n messages of a conversation for context.
@@ -211,5 +212,5 @@ func (s *Store) Recent(ctx context.Context, mind, conv ids.ID, n int) ([]Message
// GetMessage fetches one message.
func (s *Store) GetMessage(ctx context.Context, mind, id ids.ID) (Message, error) {
return scanMessage(s.db.QueryRow(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at FROM messages WHERE id=$1 AND mind_id=$2`, id, mind))
return scanMessage(s.db.QueryRow(ctx, `SELECT id, conversation_id, mind_id, role, actor, content, event_id, reply_to, intent, llm_call_id, created_at, redacted_at FROM messages WHERE id=$1 AND mind_id=$2`, id, mind))
}
+29 -27
View File
@@ -2,6 +2,7 @@ package epistemics
import (
"context"
"strings"
"time"
"github.com/jackc/pgx/v5"
@@ -13,30 +14,31 @@ import (
// Assertion is the append-only record that an actor took a stance.
type Assertion struct {
ID ids.ID `json:"id"`
MindID ids.ID `json:"mind_id"`
Actor string `json:"actor"`
ActorKind string `json:"actor_kind"`
AttributedTo string `json:"attributed_to"`
Proposition string `json:"proposition"`
Subject string `json:"subject"`
Stance string `json:"stance"`
Certainty string `json:"certainty"`
Modality string `json:"modality"`
Negated bool `json:"negated"`
Reported bool `json:"reported"`
ReportedActor string `json:"reported_actor,omitempty"`
Quoted bool `json:"quoted"`
Hypothetical bool `json:"hypothetical"`
Interpretation float64 `json:"interpretation_confidence"`
Notes []string `json:"notes"`
SourceEvent ids.ID `json:"source_event,omitempty"`
ConversationID ids.ID `json:"conversation_id,omitempty"`
MessageID ids.ID `json:"message_id,omitempty"`
Context string `json:"context,omitempty"`
Trust string `json:"trust"`
BeliefID ids.ID `json:"belief_id,omitempty"`
CreatedAt time.Time `json:"created_at"`
ID ids.ID `json:"id"`
MindID ids.ID `json:"mind_id"`
Actor string `json:"actor"`
ActorKind string `json:"actor_kind"`
AttributedTo string `json:"attributed_to"`
Proposition string `json:"proposition"`
Subject string `json:"subject"`
Stance string `json:"stance"`
Certainty string `json:"certainty"`
Modality string `json:"modality"`
Negated bool `json:"negated"`
Reported bool `json:"reported"`
ReportedActor string `json:"reported_actor,omitempty"`
Quoted bool `json:"quoted"`
Hypothetical bool `json:"hypothetical"`
Interpretation float64 `json:"interpretation_confidence"`
Notes []string `json:"notes"`
SourceEvent ids.ID `json:"source_event,omitempty"`
ConversationID ids.ID `json:"conversation_id,omitempty"`
MessageID ids.ID `json:"message_id,omitempty"`
Context string `json:"context,omitempty"`
Trust string `json:"trust"`
BeliefID ids.ID `json:"belief_id,omitempty"`
CreatedAt time.Time `json:"created_at"`
RedactedAt *time.Time `json:"redacted_at,omitempty"`
}
// New is the input to Record.
@@ -60,7 +62,7 @@ type Store struct{ db *pgxpool.Pool }
// NewStore creates a store.
func NewStore(db *pgxpool.Pool) *Store { return &Store{db: db} }
const cols = `id, mind_id, actor, actor_kind, attributed_to, proposition, subject, stance, certainty, modality, negated, reported, reported_actor, quoted, hypothetical, interpretation_confidence, interpretation_notes, source_event, conversation_id, message_id, context, trust, belief_id, created_at`
const cols = `id, mind_id, actor, actor_kind, attributed_to, proposition, subject, stance, certainty, modality, negated, reported, reported_actor, quoted, hypothetical, interpretation_confidence, interpretation_notes, source_event, conversation_id, message_id, context, trust, belief_id, created_at, redacted_at`
// Record writes an assertion inside q.
func (s *Store) Record(ctx context.Context, q storage.Querier, n New) (Assertion, error) {
@@ -76,7 +78,7 @@ func (s *Store) Record(ctx context.Context, q storage.Querier, n New) (Assertion
// Sarah asserts X" and "Andy doubts X" are both retrievable.
a.Notes = append(a.Notes, "speaker stance: "+in.SpeakerStance)
}
_, err := q.Exec(ctx, `INSERT INTO assertions (`+cols+`) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)`,
_, err := q.Exec(ctx, `INSERT INTO assertions (`+strings.TrimSuffix(cols, ", redacted_at")+`) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22,$23,$24)`,
a.ID, a.MindID, a.Actor, a.ActorKind, a.AttributedTo, a.Proposition, a.Subject, a.Stance, a.Certainty, a.Modality, a.Negated, a.Reported, a.ReportedActor, a.Quoted, a.Hypothetical,
a.Interpretation, a.Notes, nullID(a.SourceEvent), nullID(a.ConversationID), nullID(a.MessageID), a.Context, a.Trust, nullID(a.BeliefID), a.CreatedAt)
return a, err
@@ -142,7 +144,7 @@ func scan(row pgx.Row) (Assertion, error) {
var a Assertion
var src, conv, msg, bel *string
if err := row.Scan(&a.ID, &a.MindID, &a.Actor, &a.ActorKind, &a.AttributedTo, &a.Proposition, &a.Subject, &a.Stance, &a.Certainty, &a.Modality, &a.Negated, &a.Reported, &a.ReportedActor, &a.Quoted, &a.Hypothetical,
&a.Interpretation, &a.Notes, &src, &conv, &msg, &a.Context, &a.Trust, &bel, &a.CreatedAt); err != nil {
&a.Interpretation, &a.Notes, &src, &conv, &msg, &a.Context, &a.Trust, &bel, &a.CreatedAt, &a.RedactedAt); err != nil {
return a, storage.Translate(err)
}
a.SourceEvent, a.ConversationID, a.MessageID, a.BeliefID = idOf(src), idOf(conv), idOf(msg), idOf(bel)
+2 -2
View File
@@ -193,7 +193,7 @@ type Filter struct {
Ascending bool
}
const cols = `id, seq, mind_id, type, category, source, occurred_at, recorded_at, parent_ids, causation_id, correlation_id, salience, confidence, valence, arousal, evidence, summary, metadata, payload`
const cols = `id, seq, mind_id, type, category, source, occurred_at, recorded_at, parent_ids, causation_id, correlation_id, salience, confidence, valence, arousal, evidence, summary, metadata, payload, redacted_at`
// Query returns events matching f ordered by seq (descending unless Ascending).
func (s *Store) Query(ctx context.Context, f Filter) ([]events.Event, error) {
@@ -358,7 +358,7 @@ func scan(row pgx.Row) (events.Event, error) {
var causation, correlation *string
var evidence, metadata []byte
err := row.Scan(&ev.ID, &ev.Seq, &ev.MindID, &typ, &cat, &ev.Source, &ev.OccurredAt, &ev.RecordedAt, &parents, &causation, &correlation,
&ev.Salience, &ev.Confidence, &ev.Valence, &ev.Arousal, &evidence, &ev.Summary, &metadata, &ev.Payload)
&ev.Salience, &ev.Confidence, &ev.Valence, &ev.Arousal, &evidence, &ev.Summary, &metadata, &ev.Payload, &ev.RedactedAt)
if err != nil {
return ev, err
}
+4 -3
View File
@@ -60,6 +60,7 @@ type Memory struct {
LastRetrievedAt *time.Time `json:"last_retrieved_at,omitempty"`
OriginEvent ids.ID `json:"origin_event,omitempty"`
Supersedes ids.ID `json:"supersedes,omitempty"`
RedactedAt *time.Time `json:"redacted_at,omitempty"`
}
// New describes a memory to store.
@@ -237,7 +238,7 @@ func nullID(id ids.ID) *string {
return &s
}
const cols = `m.id, m.mind_id, m.kind, m.epistemic, m.status, m.content, m.detail, m.source, m.trust, m.credibility, m.confidence, m.importance, m.salience, m.activation, m.valence, m.arousal, m.occurred_at, m.created_at, m.updated_at, m.retrieval_count, m.last_retrieved_at, m.origin_event, m.supersedes`
const cols = `m.id, m.mind_id, m.kind, m.epistemic, m.status, m.content, m.detail, m.source, m.trust, m.credibility, m.confidence, m.importance, m.salience, m.activation, m.valence, m.arousal, m.occurred_at, m.created_at, m.updated_at, m.retrieval_count, m.last_retrieved_at, m.origin_event, m.supersedes, m.redacted_at`
func scan(row pgx.Row) (Memory, error) {
var m Memory
@@ -245,7 +246,7 @@ func scan(row pgx.Row) (Memory, error) {
var detail []byte
var origin, supersedes *string
err := row.Scan(&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes)
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt)
if err != nil {
return m, storage.Translate(err)
}
@@ -435,7 +436,7 @@ func (s *Store) Relationships(ctx context.Context, mind, id ids.ID) ([]Related,
m := &r.Other
if err := rows.Scan(&r.ID, &r.FromMemory, &r.ToMemory, &rel, &r.Confidence, &r.Source, &r.CreatedAt, &r.Direction,
&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes); err != nil {
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt); err != nil {
return nil, err
}
r.Relation = concepts.Relation(rel)
+2 -2
View File
@@ -88,7 +88,7 @@ func (s *Store) Retrieve(ctx context.Context, mind ids.ID, q Query) ([]Retrieval
var detail []byte
var origin, supersedes *string
if err := rows.Scan(&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &rank); err != nil {
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt, &rank); err != nil {
rows.Close()
return nil, err
}
@@ -127,7 +127,7 @@ func (s *Store) Retrieve(ctx context.Context, mind ids.ID, q Query) ([]Retrieval
var detail []byte
var origin, supersedes *string
if err := rows.Scan(&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &sim); err != nil {
&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes, &m.RedactedAt, &sim); err != nil {
break
}
m.Kind, m.Epistemic, m.Status, m.Trust = concepts.MemoryKind(kind), concepts.Epistemic(epi), Status(status), events.Trust(trust)
+341
View File
@@ -0,0 +1,341 @@
// Package privacy is the audited path for making sensitive content
// irrecoverable without breaking the history it belongs to (ADR-0009).
//
// A redaction keeps the row, its identifiers, its links, its type and its
// timestamps, and replaces the content columns with a fixed marker. What was
// removed is attested by a SHA-256 of the removed content in the ledger, so
// a later reader can establish that an event existed, when, of what kind,
// what depended on it, that it was redacted, by whom and under which policy,
// while the content itself is gone from the live database and from stored
// snapshot archives. Backups and exports made before the redaction are
// outside this system's reach and are documented as such.
package privacy
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"log/slog"
"strings"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/andydixon/noema/internal/audit"
evstore "github.com/andydixon/noema/internal/events"
"github.com/andydixon/noema/internal/snapshots"
"github.com/andydixon/noema/internal/storage"
"github.com/andydixon/noema/pkg/events"
"github.com/andydixon/noema/pkg/ids"
)
// Kind of row a redaction targets.
type Kind string
// Kinds.
const (
KindEvent Kind = "event"
KindMemory Kind = "memory"
KindMessage Kind = "message"
KindLLMCall Kind = "llm_call"
KindBelief Kind = "belief"
KindAssertion Kind = "assertion"
)
// Marker replaces removed content everywhere.
const Marker = "[redacted]"
// Errors.
var (
ErrInvalid = errors.New("privacy: invalid request")
ErrAlreadyRedacted = errors.New("privacy: already redacted")
)
// Request describes one redaction.
type Request struct {
MindID ids.ID
Kind Kind
ID ids.ID
Reason string
Policy string
Cascade bool // also redact rows derived from the target
By string
}
// Redaction is one ledger row.
type Redaction struct {
ID ids.ID `json:"id"`
MindID ids.ID `json:"mind_id"`
Kind Kind `json:"target_kind"`
TargetID ids.ID `json:"target_id"`
Reason string `json:"reason"`
Policy string `json:"policy"`
ContentHash string `json:"content_hash"`
RequestedBy string `json:"requested_by"`
CascadeOf ids.ID `json:"cascade_of,omitempty"`
CreatedAt time.Time `json:"created_at"`
}
// Service performs redactions.
type Service struct {
DB *pgxpool.Pool
Events *evstore.Store
Audit *audit.Logger
Snapshots *snapshots.Store
Log *slog.Logger
}
// Redact makes the target's content irrecoverable and returns the ledger
// row plus any cascaded rows. Everything runs in one transaction; stored
// snapshot archives are scrubbed afterwards and the outcome is audited.
func (s *Service) Redact(ctx context.Context, req Request) (Redaction, []Redaction, error) {
if req.MindID.IsZero() || req.ID.IsZero() || strings.TrimSpace(req.Reason) == "" || !validKind(req.Kind) {
return Redaction{}, nil, fmt.Errorf("%w: mind, kind, id and reason are required", ErrInvalid)
}
var primary Redaction
var cascaded []Redaction
err := storage.Tx(ctx, s.DB, func(tx pgx.Tx) error {
var err error
primary, err = s.redactOne(ctx, tx, req, ids.Nil)
if err != nil {
return err
}
if !req.Cascade {
return nil
}
deps := s.dependants(ctx, tx, req.MindID, req.Kind, req.ID)
// Events that quote a redacted row (the perception of a message, the
// encoding of an episode, the creation of a belief) are redacted too,
// one level out from the target and from each dependant.
deps = append(deps, s.referencingEvents(ctx, tx, req.MindID, req.Kind, req.ID)...)
seen := map[dep]bool{{req.Kind, req.ID}: true}
for i := 0; i < len(deps); i++ {
d := deps[i]
if seen[d] {
continue
}
seen[d] = true
r, err := s.redactOne(ctx, tx, Request{MindID: req.MindID, Kind: d.kind, ID: d.id, Reason: req.Reason, Policy: req.Policy, By: req.By}, primary.ID)
if errors.Is(err, ErrAlreadyRedacted) {
continue
}
if err != nil {
return err
}
cascaded = append(cascaded, r)
if d.kind != KindEvent {
deps = append(deps, s.referencingEvents(ctx, tx, req.MindID, d.kind, d.id)...)
}
}
return nil
})
if err != nil {
return primary, cascaded, err
}
// Stored archives hold copies of memories, messages and beliefs; scrub them.
if s.Snapshots != nil {
for _, r := range append([]Redaction{primary}, cascaded...) {
if n, err := s.Snapshots.Scrub(ctx, r.MindID, string(r.Kind), r.TargetID); err != nil {
s.Log.Warn("snapshot scrub failed", "redaction", r.ID, "err", err)
} else if n > 0 {
s.Log.Info("snapshot archives scrubbed", "redaction", r.ID, "archives", n)
}
}
}
if s.Audit != nil {
s.Audit.Log(ctx, "privacy.redact", string(req.Kind), req.ID.String(), map[string]any{"redaction_id": primary.ID, "reason": req.Reason, "policy": req.Policy, "cascaded": len(cascaded), "content_hash": primary.ContentHash})
}
return primary, cascaded, nil
}
func (s *Service) redactOne(ctx context.Context, tx pgx.Tx, req Request, cascadeOf ids.ID) (Redaction, error) {
content, mind, err := s.content(ctx, tx, req.Kind, req.ID)
if err != nil {
return Redaction{}, err
}
if mind != req.MindID {
return Redaction{}, fmt.Errorf("%w: %s %s does not belong to this mind", ErrInvalid, req.Kind, req.ID)
}
if content == nil {
return Redaction{}, ErrAlreadyRedacted
}
sum := sha256.Sum256(content)
r := Redaction{ID: ids.New(), MindID: req.MindID, Kind: req.Kind, TargetID: req.ID, Reason: req.Reason, Policy: req.Policy, ContentHash: "sha256:" + hex.EncodeToString(sum[:]), RequestedBy: req.By, CascadeOf: cascadeOf, CreatedAt: time.Now()}
if _, err := tx.Exec(ctx, `INSERT INTO redactions (id, mind_id, target_kind, target_id, reason, policy, content_hash, requested_by, cascade_of, created_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`,
r.ID, r.MindID, string(r.Kind), r.TargetID, r.Reason, r.Policy, r.ContentHash, r.RequestedBy, nullID(cascadeOf), r.CreatedAt); err != nil {
if strings.Contains(err.Error(), "redactions_target_kind_target_id_key") {
return Redaction{}, ErrAlreadyRedacted
}
return Redaction{}, err
}
var n int
if err := tx.QueryRow(ctx, `SELECT noema_redact($1, $2, $3)`, string(req.Kind), req.ID, r.ID).Scan(&n); err != nil {
return Redaction{}, err
}
if n == 0 {
return Redaction{}, ErrAlreadyRedacted
}
// The history records that a redaction happened, never what was removed.
if s.Events != nil {
if _, err := s.Events.AppendIn(ctx, tx, events.New{MindID: req.MindID, Type: events.Redaction, Source: "privacy", Summary: fmt.Sprintf("%s %s redacted under %s", req.Kind, req.ID.String()[len(req.ID.String())-8:], firstNonEmpty(req.Policy, "operator request")),
Salience: 0.2, Confidence: 1, Evidence: []events.EvidenceRef{{Kind: string(req.Kind), ID: string(req.ID), Note: "redacted"}},
Metadata: map[string]any{"redaction_id": r.ID, "target_kind": string(req.Kind), "target_id": req.ID, "policy": req.Policy, "content_hash": r.ContentHash, "cascade_of": cascadeOf},
Payload: map[string]any{"redaction_id": r.ID, "reason": req.Reason}}); err != nil {
return Redaction{}, err
}
}
return r, nil
}
// content returns the JSON of the columns a redaction will remove (nil when
// already redacted) and the row's mind, for the hash and the scoping check.
func (s *Service) content(ctx context.Context, q storage.Querier, kind Kind, id ids.ID) ([]byte, ids.ID, error) {
var mind ids.ID
var already *time.Time
var raw []byte
var err error
switch kind {
case KindEvent:
err = q.QueryRow(ctx, `SELECT mind_id, redacted_at, jsonb_build_object('summary', summary, 'payload', payload, 'metadata', metadata) FROM cognitive_events WHERE id=$1`, id).Scan(&mind, &already, &raw)
case KindMemory:
err = q.QueryRow(ctx, `SELECT mind_id, redacted_at, jsonb_build_object('content', content, 'detail', detail) FROM memories WHERE id=$1`, id).Scan(&mind, &already, &raw)
case KindMessage:
err = q.QueryRow(ctx, `SELECT mind_id, redacted_at, jsonb_build_object('content', content, 'intent', intent) FROM messages WHERE id=$1`, id).Scan(&mind, &already, &raw)
case KindLLMCall:
err = q.QueryRow(ctx, `SELECT coalesce(mind_id, '00000000-0000-0000-0000-000000000000'::uuid), redacted_at, jsonb_build_object('request', request, 'response', response) FROM llm_calls WHERE id=$1`, id).Scan(&mind, &already, &raw)
case KindBelief:
err = q.QueryRow(ctx, `SELECT mind_id, redacted_at, jsonb_build_object('proposition', proposition, 'detail', detail) FROM beliefs WHERE id=$1`, id).Scan(&mind, &already, &raw)
case KindAssertion:
err = q.QueryRow(ctx, `SELECT mind_id, redacted_at, jsonb_build_object('proposition', proposition, 'context', context) FROM assertions WHERE id=$1`, id).Scan(&mind, &already, &raw)
}
if err != nil {
return nil, ids.Nil, storage.Translate(err)
}
if already != nil {
return nil, mind, nil
}
return raw, mind, nil
}
type dep struct {
kind Kind
id ids.ID
}
// dependants lists rows whose content derives from the target: episodes
// encoded from an event, the message it carried, assertions made in it,
// beliefs whose evidence roots at it, and the model calls that worded the
// messages. Cascade is one level from an event; deeper structure keeps its
// links but not the content.
func (s *Service) dependants(ctx context.Context, q storage.Querier, mind ids.ID, kind Kind, id ids.ID) []dep {
var out []dep
collect := func(k Kind, sql string, args ...any) {
rows, err := q.Query(ctx, sql, args...)
if err != nil {
return
}
defer rows.Close()
for rows.Next() {
var x ids.ID
if rows.Scan(&x) == nil && !x.IsZero() {
out = append(out, dep{k, x})
}
}
}
switch kind {
case KindEvent:
collect(KindMemory, `SELECT id FROM memories WHERE mind_id=$1 AND origin_event=$2 AND redacted_at IS NULL`, mind, id)
collect(KindMessage, `SELECT id FROM messages WHERE mind_id=$1 AND event_id=$2 AND redacted_at IS NULL`, mind, id)
collect(KindAssertion, `SELECT id FROM assertions WHERE mind_id=$1 AND source_event=$2 AND redacted_at IS NULL`, mind, id)
collect(KindBelief, `SELECT DISTINCT belief_id FROM belief_evidence WHERE mind_id=$1 AND $2::uuid = ANY(root_ids)`, mind, id)
collect(KindLLMCall, `SELECT llm_call_id FROM messages WHERE mind_id=$1 AND (event_id=$2 OR reply_to IN (SELECT id FROM messages WHERE event_id=$2)) AND llm_call_id IS NOT NULL`, mind, id)
collect(KindMessage, `SELECT id FROM messages WHERE mind_id=$1 AND reply_to IN (SELECT id FROM messages WHERE event_id=$2) AND redacted_at IS NULL`, mind, id)
case KindMessage:
collect(KindLLMCall, `SELECT llm_call_id FROM messages WHERE mind_id=$1 AND id=$2 AND llm_call_id IS NOT NULL`, mind, id)
case KindBelief:
collect(KindAssertion, `SELECT id FROM assertions WHERE mind_id=$1 AND belief_id=$2 AND redacted_at IS NULL`, mind, id)
}
return out
}
// referencingEvents finds events that quote a row: direct children of an
// event, and events whose evidence or metadata point at the row. Redaction
// events are never included, so the record of redaction survives.
func (s *Service) referencingEvents(ctx context.Context, q storage.Querier, mind ids.ID, kind Kind, id ids.ID) []dep {
var out []dep
ref := fmt.Sprintf(`[{"kind":%q,"id":%q}]`, string(kind), id.String())
metaKey := string(kind) + "_id"
rows, err := q.Query(ctx, `SELECT id FROM cognitive_events WHERE mind_id=$1 AND redacted_at IS NULL AND type <> 'redaction'
AND (evidence @> $2::jsonb OR metadata->>$3 = $4 OR ($5 AND (causation_id=$6 OR $6 = ANY(parent_ids)))) LIMIT 500`, mind, ref, metaKey, id.String(), kind == KindEvent, id)
if err != nil {
return nil
}
defer rows.Close()
for rows.Next() {
var x ids.ID
if rows.Scan(&x) == nil && x != id {
out = append(out, dep{KindEvent, x})
}
}
return out
}
// List returns the ledger for a mind, newest first.
func (s *Service) List(ctx context.Context, mind ids.ID, limit int) ([]Redaction, error) {
if limit <= 0 || limit > 500 {
limit = 100
}
rows, err := s.DB.Query(ctx, `SELECT id, mind_id, target_kind, target_id, reason, policy, content_hash, requested_by, cascade_of, created_at FROM redactions WHERE mind_id=$1 ORDER BY created_at DESC LIMIT $2`, mind, limit)
if err != nil {
return nil, err
}
defer rows.Close()
var out []Redaction
for rows.Next() {
var r Redaction
var kind string
var cascade *string
if err := rows.Scan(&r.ID, &r.MindID, &kind, &r.TargetID, &r.Reason, &r.Policy, &r.ContentHash, &r.RequestedBy, &cascade, &r.CreatedAt); err != nil {
return nil, err
}
r.Kind = Kind(kind)
if cascade != nil {
r.CascadeOf = ids.ID(*cascade)
}
out = append(out, r)
}
return out, rows.Err()
}
func validKind(k Kind) bool {
switch k {
case KindEvent, KindMemory, KindMessage, KindLLMCall, KindBelief, KindAssertion:
return true
}
return false
}
func nullID(id ids.ID) *string {
if id.IsZero() {
return nil
}
s := string(id)
return &s
}
func firstNonEmpty(a, b string) string {
if a != "" {
return a
}
return b
}
// Attest recomputes nothing: redacted content is gone. It exposes the hash
// so an operator holding an external copy can prove what was removed.
func Attest(r Redaction) string { return r.ContentHash }
var _ = json.Marshal
+197
View File
@@ -0,0 +1,197 @@
package privacy
import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"log/slog"
"strings"
"testing"
"time"
"github.com/jackc/pgx/v5/pgxpool"
"github.com/andydixon/noema/internal/audit"
"github.com/andydixon/noema/internal/beliefs"
"github.com/andydixon/noema/internal/epistemics"
evstore "github.com/andydixon/noema/internal/events"
"github.com/andydixon/noema/internal/memory"
"github.com/andydixon/noema/internal/minds"
"github.com/andydixon/noema/internal/snapshots"
"github.com/andydixon/noema/internal/storage"
"github.com/andydixon/noema/migrations"
"github.com/andydixon/noema/pkg/concepts"
"github.com/andydixon/noema/pkg/events"
"github.com/andydixon/noema/pkg/ids"
)
// TestRedactionIsIrrecoverableButReferentiallyIntact: content goes, the
// row, its links, its type and time stay; the hash attests what was removed;
// derived rows are cascaded; stored archives are scrubbed; and the general
// append-only guarantee still holds for every other write.
func TestRedactionIsIrrecoverableButReferentiallyIntact(t *testing.T) {
pool := storage.TestPool(t, migrations.FS)
ctx := context.Background()
log := slog.New(slog.DiscardHandler)
ms := minds.NewStore(pool)
m, _ := ms.Create(ctx, minds.CreateParams{Slug: "priv"})
es := evstore.NewStore(pool, evstore.NewBus(8), nil)
mem := memory.NewStore(ctx, pool, nil)
bs := beliefs.NewStore(pool)
as := epistemics.NewStore(pool)
snaps := &snapshots.Store{DB: pool, Events: es, Minds: ms}
secret := "Andy said Steve steals laptops from the Leeds office"
obs, err := es.Append(ctx, events.New{MindID: m.ID, Type: events.Observation, Source: "conversation", Summary: secret, Payload: events.ObservationPayload{Text: secret, Actor: "andy", Trust: events.TrustUser}, Metadata: map[string]any{"conversation_id": "c1", "channel": "conversation"}})
if err != nil {
t.Fatal(err)
}
child, _ := es.Append(ctx, events.New{MindID: m.ID, Type: events.Perception, Source: "brain.perceive", Summary: "perceived: " + secret, CausationID: obs[0].ID, ParentIDs: []ids.ID{obs[0].ID}})
episode, _ := mem.Create(ctx, pool, memory.New{MindID: m.ID, Kind: concepts.Episodic, Epistemic: concepts.Observation, Content: "andy said: " + secret, Source: "conversation", Trust: events.TrustUser, Confidence: 1, Importance: 0.5, OccurredAt: time.Now(), OriginEvent: obs[0].ID})
emit := func(ctx context.Context, q storage.Querier, n events.New) (events.Event, error) {
out, err := es.AppendIn(ctx, q, n)
if err != nil {
return events.Event{}, err
}
return out[0], nil
}
res, _ := bs.Create(ctx, pool, emit, beliefs.New{MindID: m.ID, Proposition: "Steve steals laptops", Subject: "steve", Epistemic: concepts.Claim, Source: "conversation",
Evidence: []beliefs.EvidenceInput{{Direction: "for", Kind: "assertion", Source: "asserted-by:andy", Family: "actor:andy", Derivation: beliefs.DerivationAssertion, Weight: 0.5, Credibility: 0.85, Note: "andy asserted it", RootIDs: []string{string(obs[0].ID)}}}})
assertion, _ := as.Record(ctx, pool, epistemics.New{MindID: m.ID, Actor: "andy", Interpretation: epistemics.Interpret("andy", "Steve steals laptops"), SourceEvent: obs[0].ID, Context: secret, BeliefID: res.Belief.ID})
convID, msgID, replyID, callID := ids.New(), ids.New(), ids.New(), ids.New()
pool.Exec(ctx, `INSERT INTO conversations (id, mind_id, title, channel) VALUES ($1,$2,'t','conversation')`, convID, m.ID) //nolint:errcheck
pool.Exec(ctx, `INSERT INTO llm_calls (id, mind_id, role, provider, model, request_hash, request, response) VALUES ($1,$2,'respond','p','m','h', $3, $4)`, callID, m.ID, `{"messages":[{"content":"`+secret+`"}]}`, `{"text":"noted"}`) //nolint:errcheck
if _, err := pool.Exec(ctx, `INSERT INTO messages (id, conversation_id, mind_id, role, actor, content, event_id) VALUES ($1,$2,$3,'user','andy',$4,$5)`, msgID, convID, m.ID, secret, obs[0].ID); err != nil {
t.Fatal(err)
}
pool.Exec(ctx, `INSERT INTO messages (id, conversation_id, mind_id, role, actor, content, reply_to, llm_call_id) VALUES ($1,$2,$3,'mind','mind','Noted: Steve steals laptops',$4,$5)`, replyID, convID, m.ID, msgID, callID) //nolint:errcheck
sn, err := snaps.Create(ctx, m, "before", "", "test")
if err != nil {
t.Fatal(err)
}
if raw, _ := json.Marshal(mustArchive(t, snaps, sn.ID)); !strings.Contains(string(raw), "Leeds") {
t.Fatal("test setup: archive should contain the secret before redaction")
}
svc := &Service{DB: pool, Events: es, Audit: audit.New(pool, log), Snapshots: snaps, Log: log}
// Refuses bad requests and rows from other minds.
other, _ := ms.Create(ctx, minds.CreateParams{Slug: "other"})
if _, _, err := svc.Redact(ctx, Request{MindID: other.ID, Kind: KindEvent, ID: obs[0].ID, Reason: "x", By: "admin"}); err == nil {
t.Fatal("redacting another mind's row must fail")
}
if _, _, err := svc.Redact(ctx, Request{MindID: m.ID, Kind: KindEvent, ID: obs[0].ID, By: "admin"}); err == nil {
t.Fatal("a reason is required")
}
primary, cascaded, err := svc.Redact(ctx, Request{MindID: m.ID, Kind: KindEvent, ID: obs[0].ID, Reason: "data subject request", Policy: "GDPR art. 17", Cascade: true, By: "admin"})
if err != nil {
t.Fatal(err)
}
// The event exists, keeps type, time, links and ids; content is a marker.
ev, _ := es.Get(ctx, obs[0].ID)
if ev.Summary != Marker || string(ev.Payload) != `{"redacted": true}` || ev.RedactedAt == nil || ev.Type != events.Observation || ev.Seq != obs[0].Seq {
t.Fatalf("event after redaction: %+v payload=%s", ev, ev.Payload)
}
if ev.Metadata["conversation_id"] != "c1" || ev.Metadata["redacted"] != true {
t.Fatalf("metadata should keep only structural keys: %+v", ev.Metadata)
}
if kids, _ := es.Children(ctx, obs[0].ID, 10); len(kids) != 1 || kids[0].ID != child[0].ID {
t.Fatal("links to dependants must survive")
}
// The hash attests the removed content.
want := sha256.Sum256(mustJSONB(t, pool, `SELECT jsonb_build_object('summary', $1::text, 'payload', $2::jsonb, 'metadata', $3::jsonb)`, secret, string(obs[0].Payload), mustJSON(obs[0].Metadata)))
if primary.ContentHash != "sha256:"+hex.EncodeToString(want[:]) {
t.Fatalf("hash: %s", primary.ContentHash)
}
// Cascade reached the episode, the message, the reply, the model call, the assertion and the belief.
kinds := map[Kind]int{}
for _, c := range cascaded {
kinds[c.Kind]++
if c.CascadeOf != primary.ID {
t.Fatalf("cascade lineage missing: %+v", c)
}
}
if kinds[KindMemory] != 1 || kinds[KindMessage] != 2 || kinds[KindLLMCall] != 1 || kinds[KindAssertion] != 1 || kinds[KindBelief] != 1 {
t.Fatalf("cascade: %+v", kinds)
}
if mm, _ := mem.Get(ctx, m.ID, episode.ID); mm.Content != Marker || mm.RedactedAt == nil || mm.OriginEvent != obs[0].ID {
t.Fatalf("memory: %+v", mm)
}
if b, _ := bs.Get(ctx, pool, m.ID, res.Belief.ID); b.Proposition != Marker || b.Confidence != res.Belief.Confidence {
t.Fatalf("belief proposition should be redacted and its score untouched: %+v", b)
}
var vprop, note, msg, reply, req string
var resp *string
pool.QueryRow(ctx, `SELECT proposition FROM belief_versions WHERE belief_id=$1 LIMIT 1`, res.Belief.ID).Scan(&vprop) //nolint:errcheck
pool.QueryRow(ctx, `SELECT note FROM belief_evidence WHERE belief_id=$1 LIMIT 1`, res.Belief.ID).Scan(&note) //nolint:errcheck
pool.QueryRow(ctx, `SELECT content FROM messages WHERE id=$1`, msgID).Scan(&msg) //nolint:errcheck
pool.QueryRow(ctx, `SELECT content FROM messages WHERE id=$1`, replyID).Scan(&reply) //nolint:errcheck
pool.QueryRow(ctx, `SELECT request::text, response::text FROM llm_calls WHERE id=$1`, callID).Scan(&req, &resp) //nolint:errcheck
if vprop != Marker || note != Marker || msg != Marker || reply != Marker || strings.Contains(req, "Leeds") || resp != nil {
t.Fatalf("cascade content: version=%q note=%q msg=%q reply=%q req=%q", vprop, note, msg, reply, req)
}
if a, _ := as.List(ctx, m.ID, epistemics.Filter{Limit: 5}); len(a) != 1 || a[0].Proposition != Marker || a[0].ID != assertion.ID {
t.Fatalf("assertion: %+v", a)
}
// Nothing in the live database still holds the secret.
var leaks int
pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM cognitive_events WHERE mind_id=$1 AND (summary ILIKE '%leeds%' OR payload::text ILIKE '%leeds%' OR metadata::text ILIKE '%leeds%'))
+ (SELECT count(*) FROM memories WHERE mind_id=$1 AND content ILIKE '%leeds%') + (SELECT count(*) FROM messages WHERE mind_id=$1 AND content ILIKE '%leeds%')
+ (SELECT count(*) FROM llm_calls WHERE mind_id=$1 AND request::text ILIKE '%leeds%') + (SELECT count(*) FROM assertions WHERE mind_id=$1 AND (proposition ILIKE '%leeds%' OR context ILIKE '%leeds%'))`, m.ID).Scan(&leaks) //nolint:errcheck
if leaks != 0 {
t.Fatalf("secret still present in %d rows", leaks)
}
// Stored archives were scrubbed and re-signed; they still validate.
a := mustArchive(t, snaps, sn.ID)
if raw, _ := json.Marshal(a); strings.Contains(string(raw), "Leeds") || strings.Contains(string(raw), "steals") {
t.Fatal("archive still holds redacted content")
}
if len(a.Redacted) < 3 || snapshots.Validate(a) != nil {
t.Fatalf("archive scrub record or checksum: %v %v", a.Redacted, snapshots.Validate(a))
}
// The ledger and the history record what happened, not what was removed.
ledger, _ := svc.List(ctx, m.ID, 20)
if len(ledger) != 1+len(cascaded) || ledger[0].Reason != "data subject request" {
t.Fatalf("ledger: %d", len(ledger))
}
reds, _ := es.Query(ctx, evstore.Filter{MindID: m.ID, Types: []events.Type{events.Redaction}, Limit: 20})
if len(reds) != 1+len(cascaded) || strings.Contains(reds[0].Summary, "Leeds") {
t.Fatalf("redaction events: %d", len(reds))
}
// Idempotent and still append-only for everything else.
if _, _, err := svc.Redact(ctx, Request{MindID: m.ID, Kind: KindEvent, ID: obs[0].ID, Reason: "again", By: "admin"}); err == nil {
t.Fatal("second redaction should report already redacted")
}
if _, err := pool.Exec(ctx, `UPDATE cognitive_events SET summary='tampered' WHERE id=$1`, child[0].ID); err == nil {
t.Fatal("plain updates must still be forbidden")
}
if _, err := pool.Exec(ctx, `DELETE FROM cognitive_events WHERE id=$1`, obs[0].ID); err == nil {
t.Fatal("deletes must still be forbidden, even of redacted rows")
}
if _, err := pool.Exec(ctx, `UPDATE redactions SET reason='x' WHERE id=$1`, primary.ID); err == nil {
t.Fatal("the ledger is append-only")
}
}
func mustArchive(t *testing.T, s *snapshots.Store, id ids.ID) snapshots.Archive {
t.Helper()
a, err := s.Archive(context.Background(), id)
if err != nil {
t.Fatal(err)
}
return a
}
func mustJSON(v any) string {
b, _ := json.Marshal(v)
return string(b)
}
func mustJSONB(t *testing.T, pool *pgxpool.Pool, sql string, args ...any) []byte {
t.Helper()
var raw []byte
if err := pool.QueryRow(context.Background(), sql, args...).Scan(&raw); err != nil {
t.Fatal(err)
}
return raw
}
+8
View File
@@ -69,6 +69,7 @@ type Summary struct {
LLMLive int `json:"llm_live"`
LLMReplayed int `json:"llm_replayed"`
Cycles int `json:"cycles"`
Redacted int `json:"redacted"` // stimuli whose content was redacted and could not be replayed
}
// Step is the outcome of replaying one stimulus.
@@ -218,6 +219,13 @@ func (s *Service) Execute(ctx context.Context, r Replay) (Replay, error) {
}
orig, _ := s.Events.Query(ctx, evstore.Filter{MindID: src.ID, AfterSeq: stim.Seq, BeforeSeq: upper, Ascending: true, Limit: 2000})
st.Original = Digest(withoutPerception(orig))
if stim.RedactedAt != nil {
// The content is gone; replaying a marker would fabricate cognition.
st.Label, st.Note = "redacted", "stimulus content was redacted under a privacy policy and was not replayed"
r.Summary.Redacted++
steps = append(steps, st)
continue
}
startedAt := time.Now()
inj, err := drv.Inject(ctx, stim, map[string]any{"replay_of": stim.ID, "replay_id": r.ID})
if err != nil {
+12
View File
@@ -87,6 +87,18 @@ func TestReplayIsExactWithoutModel(t *testing.T) {
if after[0].Seq != last[0].Seq {
t.Fatal("replay wrote into the source mind")
}
// A redacted stimulus is not replayed and is labelled as such.
if _, err := h.Pool.Exec(ctx, `INSERT INTO redactions (id, mind_id, target_kind, target_id, reason, content_hash, requested_by) VALUES ($1,$2,'event',$3,'test','sha256:0','test')`, ids.New(), src.ID, steps[0].StimulusID); err != nil {
t.Fatal(err)
}
var n int
if err := h.Pool.QueryRow(ctx, `SELECT noema_redact('event', $1, $2)`, steps[0].StimulusID, ids.New()).Scan(&n); err != nil || n != 1 {
t.Fatalf("redact: %v %d", err, n)
}
red, err := svc.Run(ctx, src, sn.ID, sn.EventSeq, last[0].Seq, "test")
if err != nil || red.Summary.Redacted != 1 || red.Summary.Exact != 0 || StepsOf(red)[0].Label != "redacted" {
t.Fatalf("redacted stimulus should be skipped and labelled: %v %+v", err, red.Summary)
}
// Digest normalisation ignores ids and numbers.
if Normalise("selected 3 thoughts for 01a0a148-9034-7685-9794-cf5ec0c25fb7") != "selected <n> thoughts for <id>" {
t.Fatal(Normalise("selected 3 thoughts for 01a0a148-9034-7685-9794-cf5ec0c25fb7"))
+79
View File
@@ -68,6 +68,10 @@ type Archive struct {
// Tables maps table name → rows (column → value, as JSON).
Tables map[string][]map[string]any `json:"tables"`
Checksum string `json:"checksum"`
// Redacted lists row ids whose content was removed from this archive
// after capture under a privacy redaction; the checksum covers the
// scrubbed content.
Redacted []string `json:"redacted,omitempty"`
}
// Source records where an archive came from.
@@ -544,3 +548,78 @@ func SortedTables(a Archive) []string {
sort.Strings(out)
return out
}
// Scrub removes the content of a redacted row from every stored archive of
// the mind, recording the id in the archive and re-signing it. Returns the
// number of archives changed. Events and model calls are never in archives.
func (s *Store) Scrub(ctx context.Context, mind ids.ID, kind string, id ids.ID) (int, error) {
rows, err := s.DB.Query(ctx, `SELECT id, archive FROM snapshots WHERE mind_id=$1`, mind)
if err != nil {
return 0, err
}
type snap struct {
id ids.ID
raw []byte
}
var snaps []snap
for rows.Next() {
var sn snap
if err := rows.Scan(&sn.id, &sn.raw); err != nil {
rows.Close()
return 0, err
}
snaps = append(snaps, sn)
}
rows.Close()
changed := 0
target := id.String()
for _, sn := range snaps {
var a Archive
if err := json.Unmarshal(sn.raw, &a); err != nil {
continue
}
if !scrubArchive(&a, kind, target) {
continue
}
a.Redacted = append(a.Redacted, target)
a.Checksum = Sum(a)
raw, _ := json.Marshal(a)
counts, _ := json.Marshal(a.Counts())
if _, err := s.DB.Exec(ctx, `UPDATE snapshots SET archive=$2, checksum=$3, size_bytes=$4, counts=$5 WHERE id=$1`, sn.id, raw, a.Checksum, len(raw), counts); err != nil {
return changed, err
}
changed++
}
return changed, nil
}
// scrubArchive replaces the content fields of the matching rows in place.
func scrubArchive(a *Archive, kind, id string) bool {
changed := false
replace := func(table string, match func(r map[string]any) bool, fields map[string]any) {
for _, r := range a.Tables[table] {
if match(r) {
for k, v := range fields {
if _, ok := r[k]; ok {
r[k] = v
}
}
changed = true
}
}
}
byID := func(r map[string]any) bool { return r["id"] == id }
switch kind {
case "memory":
replace("memories", byID, map[string]any{"content": "[redacted]", "detail": map[string]any{"redacted": true}})
case "message":
replace("messages", byID, map[string]any{"content": "[redacted]", "intent": nil})
case "belief":
replace("beliefs", byID, map[string]any{"proposition": "[redacted]", "detail": map[string]any{"redacted": true}})
byBelief := func(r map[string]any) bool { return r["belief_id"] == id }
replace("belief_versions", byBelief, map[string]any{"proposition": "[redacted]", "reason": "[redacted]"})
replace("belief_evidence", byBelief, map[string]any{"note": "[redacted]"})
replace("belief_alternatives", byBelief, map[string]any{"proposition": "[redacted]", "note": "[redacted]"})
}
return changed
}
+43
View File
@@ -0,0 +1,43 @@
package ui
import (
"net/http"
"strconv"
"github.com/andydixon/noema/internal/api"
"github.com/andydixon/noema/internal/auth"
"github.com/andydixon/noema/pkg/ids"
)
func (u *UI) privacyRoutes(m *http.ServeMux) {
m.Handle("GET /minds/{mind}/redactions", u.require(auth.RoleOperator, u.redactionsPage))
m.Handle("POST /minds/{mind}/redactions", u.require(auth.RoleAdministrator, u.redact))
}
func (u *UI) redactionsPage(w http.ResponseWriter, r *http.Request) {
m, ok := u.mind(w, r)
if !ok {
return
}
list, _ := u.deps.Privacy.List(r.Context(), m.ID, 200)
u.render(w, r, "redactions", m.DisplayName+" · Redactions", map[string]any{"Mind": m, "Redactions": list, "Kind": r.URL.Query().Get("kind"), "ID": r.URL.Query().Get("id")})
}
func (u *UI) redact(w http.ResponseWriter, r *http.Request) {
m, ok := u.mind(w, r)
if !ok {
return
}
id, err := ids.Parse(r.PostFormValue("id"))
if err != nil {
u.flash(w, r, "error", "Bad id.", "/minds/"+m.Slug+"/redactions")
return
}
cascade, _ := strconv.ParseBool(r.PostFormValue("cascade"))
p, c, err := u.studyF().Redact(r, m.ID, api.RedactionInput{Kind: r.PostFormValue("kind"), ID: id, Reason: r.PostFormValue("reason"), Policy: r.PostFormValue("policy"), Cascade: cascade}, username(r))
if err != nil {
u.flash(w, r, "error", err.Error(), "/minds/"+m.Slug+"/redactions")
return
}
u.flash(w, r, "ok", "Redacted "+string(p.Kind)+" "+p.TargetID.String()[len(p.TargetID.String())-8:]+" and "+itoa(len(c))+" dependant row(s). The content is irrecoverable; hash "+p.ContentHash[:23]+"… attests what was removed.", "/minds/"+m.Slug+"/redactions")
}
+2
View File
@@ -133,6 +133,7 @@ func (u *UI) parse(fsys fs.FS) error {
"duration": func(s int64) string { return (time.Duration(s) * time.Second).Truncate(time.Second).String() },
"bytes": humanBytes,
"pct": func(f float64) string { return fmt.Sprintf("%.0f%%", f*100) },
"list": func(xs ...string) []string { return xs },
"conf": func(v any) string {
switch x := v.(type) {
case float64:
@@ -388,6 +389,7 @@ func (u *UI) Register(m *http.ServeMux) {
u.integrationRoutes(m)
u.dreamRoutes(m)
u.studyRoutes(m)
u.privacyRoutes(m)
for _, p := range []struct{ path, title, note string }{} {
m.Handle("GET "+p.path, u.require(auth.RoleObserver, func(w http.ResponseWriter, r *http.Request) { u.render(w, r, "placeholder", p.title, p.note) }))