noema/internal/memory/memory.go

// Package memory implements long-term memory: episodic, semantic, procedural
// and autobiographical records with explicit epistemic status, credibility,
// importance and decaying activation, plus deterministic fused retrieval
// (ADR-0003). Nothing here is deleted by decay.
package memory

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math"
	"strings"
	"time"

	"github.com/jackc/pgx/v5"
	"github.com/jackc/pgx/v5/pgxpool"

	"github.com/andydixon/noema/internal/embeddings"
	"github.com/andydixon/noema/internal/storage"
	"github.com/andydixon/noema/pkg/concepts"
	"github.com/andydixon/noema/pkg/events"
	"github.com/andydixon/noema/pkg/ids"
)

// Status of a memory record. Retracted and superseded memories stay
// retrievable for provenance but are excluded from normal recall.
type Status string

// Statuses.
const (
	StatusActive     Status = "active"
	StatusRetracted  Status = "retracted"
	StatusSuperseded Status = "superseded"
	StatusArchived   Status = "archived"
)

// Memory is one long-term record.
type Memory struct {
	ID              ids.ID              `json:"id"`
	MindID          ids.ID              `json:"mind_id"`
	Kind            concepts.MemoryKind `json:"kind"`
	Epistemic       concepts.Epistemic  `json:"epistemic"`
	Status          Status              `json:"status"`
	Content         string              `json:"content"`
	Detail          map[string]any      `json:"detail"`
	Source          string              `json:"source"`
	Trust           events.Trust        `json:"trust"`
	Credibility     float64             `json:"credibility"`
	Confidence      float64             `json:"confidence"`
	Importance      float64             `json:"importance"`
	Salience        float64             `json:"salience"`
	Activation      float64             `json:"activation"`
	Valence         *float64            `json:"valence,omitempty"`
	Arousal         *float64            `json:"arousal,omitempty"`
	OccurredAt      time.Time           `json:"occurred_at"`
	CreatedAt       time.Time           `json:"created_at"`
	UpdatedAt       time.Time           `json:"updated_at"`
	RetrievalCount  int                 `json:"retrieval_count"`
	LastRetrievedAt *time.Time          `json:"last_retrieved_at,omitempty"`
	OriginEvent     ids.ID              `json:"origin_event,omitempty"`
	Supersedes      ids.ID              `json:"supersedes,omitempty"`
}

// New describes a memory to store.
type New struct {
	MindID      ids.ID
	Kind        concepts.MemoryKind
	Epistemic   concepts.Epistemic
	Content     string
	Detail      map[string]any
	Source      string
	Trust       events.Trust
	Confidence  float64
	Importance  float64
	Salience    float64
	Valence     *float64
	Arousal     *float64
	OccurredAt  time.Time
	OriginEvent ids.ID
	Supersedes  ids.ID
}

// Relationship is an edge between two memories.
type Relationship struct {
	ID         ids.ID            `json:"id"`
	FromMemory ids.ID            `json:"from_memory"`
	ToMemory   ids.ID            `json:"to_memory"`
	Relation   concepts.Relation `json:"relation"`
	Confidence float64           `json:"confidence"`
	Source     string            `json:"source"`
	CreatedAt  time.Time         `json:"created_at"`
}

// ErrInvalid marks bad input.
var ErrInvalid = errors.New("invalid memory")

// Store persists memories.
type Store struct {
	db       *pgxpool.Pool
	embedder embeddings.Embedder
	vectors  bool // memory_embeddings exists (pgvector available)
}

// NewStore creates a store and detects vector support.
func NewStore(ctx context.Context, db *pgxpool.Pool, embedder embeddings.Embedder) *Store {
	s := &Store{db: db, embedder: embedder}
	if embedder == nil {
		s.embedder = embeddings.None{}
	}
	db.QueryRow(ctx, `SELECT EXISTS (SELECT 1 FROM information_schema.tables WHERE table_name = 'memory_embeddings')`).Scan(&s.vectors) //nolint:errcheck
	return s
}

// VectorsEnabled reports whether pgvector storage is present.
func (s *Store) VectorsEnabled() bool { return s.vectors }

// SemanticEnabled reports whether semantic (embedding) retrieval can run.
func (s *Store) SemanticEnabled() bool { return s.vectors && s.embedder.Model() != "" }

// SetEmbedder swaps the embedder (providers are configured at runtime).
func (s *Store) SetEmbedder(e embeddings.Embedder) {
	if e == nil {
		e = embeddings.None{}
	}
	s.embedder = e
}

func (n *New) validate() error {
	if n.MindID.IsZero() {
		return fmt.Errorf("%w: mind id required", ErrInvalid)
	}
	if !n.Kind.Valid() {
		return fmt.Errorf("%w: unknown kind %q", ErrInvalid, n.Kind)
	}
	if !n.Epistemic.Valid() {
		return fmt.Errorf("%w: unknown epistemic kind %q", ErrInvalid, n.Epistemic)
	}
	n.Content = strings.TrimSpace(n.Content)
	if n.Content == "" || len(n.Content) > 20000 {
		return fmt.Errorf("%w: content required (max 20000 chars)", ErrInvalid)
	}
	if !n.Trust.Valid() {
		n.Trust = events.TrustLow
	}
	n.Confidence, n.Importance, n.Salience = clamp(n.Confidence), clamp(n.Importance), clamp(n.Salience)
	if n.OccurredAt.IsZero() {
		n.OccurredAt = time.Now()
	}
	if n.Detail == nil {
		n.Detail = map[string]any{}
	}
	return nil
}

func clamp(v float64) float64 {
	if math.IsNaN(v) || v < 0 {
		return 0
	}
	if v > 1 {
		return 1
	}
	return v
}

// Create stores a memory using q (a pool or transaction). Embedding happens
// best-effort after the row exists so a slow provider never blocks encoding.
func (s *Store) Create(ctx context.Context, q storage.Querier, n New) (Memory, error) {
	if err := n.validate(); err != nil {
		return Memory{}, err
	}
	m := Memory{ID: ids.New(), MindID: n.MindID, Kind: n.Kind, Epistemic: n.Epistemic, Status: StatusActive, Content: n.Content, Detail: n.Detail, Source: n.Source,
		Trust: n.Trust, Credibility: n.Trust.Credibility(), Confidence: n.Confidence, Importance: n.Importance, Salience: n.Salience, Activation: 1,
		Valence: n.Valence, Arousal: n.Arousal, OccurredAt: n.OccurredAt, CreatedAt: time.Now(), UpdatedAt: time.Now(), OriginEvent: n.OriginEvent, Supersedes: n.Supersedes}
	detail, _ := json.Marshal(n.Detail)
	_, err := q.Exec(ctx, `INSERT INTO memories (id, mind_id, kind, epistemic, content, detail, source, trust, credibility, confidence, importance, salience, valence, arousal, occurred_at, origin_event, supersedes)
		VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17)`,
		m.ID, m.MindID, string(m.Kind), string(m.Epistemic), m.Content, detail, m.Source, string(m.Trust), m.Credibility, m.Confidence, m.Importance, m.Salience, m.Valence, m.Arousal, m.OccurredAt, nullID(m.OriginEvent), nullID(m.Supersedes))
	if err != nil {
		return Memory{}, storage.Translate(err)
	}
	if !n.Supersedes.IsZero() {
		q.Exec(ctx, `UPDATE memories SET status='superseded', updated_at=now() WHERE id=$1 AND mind_id=$2`, n.Supersedes, n.MindID) //nolint:errcheck
	}
	return m, nil
}

// Embed computes and stores the embedding for a memory if a provider exists.
// Safe to call from a background task; errors are returned for logging only.
func (s *Store) Embed(ctx context.Context, m Memory) error {
	if !s.SemanticEnabled() {
		return nil
	}
	vecs, err := s.embedder.Embed(ctx, []string{m.Content})
	if err != nil || len(vecs) != 1 {
		return fmt.Errorf("embed: %w", err)
	}
	_, err = s.db.Exec(ctx, `INSERT INTO memory_embeddings (memory_id, model, dims, embedding) VALUES ($1,$2,$3,$4::vector)
		ON CONFLICT (memory_id, model) DO UPDATE SET embedding=excluded.embedding, dims=excluded.dims, created_at=now()`, m.ID, s.embedder.Model(), len(vecs[0]), embeddings.Format(vecs[0]))
	return err
}

// EmbedMissing embeds up to limit memories lacking a vector for the current model.
func (s *Store) EmbedMissing(ctx context.Context, limit int) (int, error) {
	if !s.SemanticEnabled() {
		return 0, nil
	}
	rows, err := s.db.Query(ctx, `SELECT `+cols+` FROM memories m WHERE status='active' AND NOT EXISTS (SELECT 1 FROM memory_embeddings e WHERE e.memory_id=m.id AND e.model=$1) ORDER BY created_at DESC LIMIT $2`, s.embedder.Model(), limit)
	if err != nil {
		return 0, err
	}
	var list []Memory
	for rows.Next() {
		m, err := scan(rows)
		if err != nil {
			rows.Close()
			return 0, err
		}
		list = append(list, m)
	}
	rows.Close()
	n := 0
	for _, m := range list {
		if err := s.Embed(ctx, m); err != nil {
			return n, err
		}
		n++
	}
	return n, nil
}

func nullID(id ids.ID) *string {
	if id.IsZero() {
		return nil
	}
	s := string(id)
	return &s
}

const cols = `m.id, m.mind_id, m.kind, m.epistemic, m.status, m.content, m.detail, m.source, m.trust, m.credibility, m.confidence, m.importance, m.salience, m.activation, m.valence, m.arousal, m.occurred_at, m.created_at, m.updated_at, m.retrieval_count, m.last_retrieved_at, m.origin_event, m.supersedes`

func scan(row pgx.Row) (Memory, error) {
	var m Memory
	var kind, epi, status, trust string
	var detail []byte
	var origin, supersedes *string
	err := row.Scan(&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
		&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes)
	if err != nil {
		return m, storage.Translate(err)
	}
	m.Kind, m.Epistemic, m.Status, m.Trust = concepts.MemoryKind(kind), concepts.Epistemic(epi), Status(status), events.Trust(trust)
	json.Unmarshal(detail, &m.Detail) //nolint:errcheck
	if m.Detail == nil {
		m.Detail = map[string]any{}
	}
	if origin != nil {
		m.OriginEvent = ids.ID(*origin)
	}
	if supersedes != nil {
		m.Supersedes = ids.ID(*supersedes)
	}
	return m, nil
}

// Get fetches a memory scoped to a mind (cross-mind reads are impossible by construction).
func (s *Store) Get(ctx context.Context, mind, id ids.ID) (Memory, error) {
	return scan(s.db.QueryRow(ctx, `SELECT `+cols+` FROM memories m WHERE m.id=$1 AND m.mind_id=$2`, id, mind))
}

// Filter selects memories for listing.
type Filter struct {
	Kinds         []concepts.MemoryKind
	Epistemic     []concepts.Epistemic
	Statuses      []Status // default: active
	MinConfidence float64
	MinImportance float64
	Source        string
	Since, Until  time.Time
	Search        string // plain text search (websearch syntax)
	Before        time.Time
	Limit         int
}

// List returns memories newest first (by occurred_at) with keyset paging on Before.
func (s *Store) List(ctx context.Context, mind ids.ID, f Filter) ([]Memory, error) {
	where := []string{"m.mind_id = $1"}
	args := []any{mind}
	arg := func(v any) string { args = append(args, v); return fmt.Sprintf("$%d", len(args)) }
	if len(f.Kinds) > 0 {
		where = append(where, "m.kind = ANY("+arg(strs(f.Kinds))+")")
	}
	if len(f.Epistemic) > 0 {
		where = append(where, "m.epistemic = ANY("+arg(strs(f.Epistemic))+")")
	}
	if len(f.Statuses) > 0 {
		where = append(where, "m.status = ANY("+arg(strs(f.Statuses))+")")
	} else {
		where = append(where, "m.status = 'active'")
	}
	if f.MinConfidence > 0 {
		where = append(where, "m.confidence >= "+arg(f.MinConfidence))
	}
	if f.MinImportance > 0 {
		where = append(where, "m.importance >= "+arg(f.MinImportance))
	}
	if f.Source != "" {
		where = append(where, "m.source = "+arg(f.Source))
	}
	if !f.Since.IsZero() {
		where = append(where, "m.occurred_at >= "+arg(f.Since))
	}
	if !f.Until.IsZero() {
		where = append(where, "m.occurred_at <= "+arg(f.Until))
	}
	if !f.Before.IsZero() {
		where = append(where, "m.occurred_at < "+arg(f.Before))
	}
	if q := strings.TrimSpace(f.Search); q != "" {
		where = append(where, "m.search @@ websearch_to_tsquery('english', "+arg(q)+")")
	}
	limit := f.Limit
	if limit <= 0 || limit > 500 {
		limit = 100
	}
	rows, err := s.db.Query(ctx, `SELECT `+cols+` FROM memories m WHERE `+strings.Join(where, " AND ")+` ORDER BY m.occurred_at DESC, m.id DESC LIMIT `+arg(limit), args...)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Memory
	for rows.Next() {
		m, err := scan(rows)
		if err != nil {
			return nil, err
		}
		out = append(out, m)
	}
	return out, rows.Err()
}

func strs[T ~string](xs []T) []string {
	out := make([]string, len(xs))
	for i, x := range xs {
		out[i] = string(x)
	}
	return out
}

// SetStatus changes status (retract, archive). It never deletes.
func (s *Store) SetStatus(ctx context.Context, mind, id ids.ID, st Status) error {
	tag, err := s.db.Exec(ctx, `UPDATE memories SET status=$3, updated_at=now() WHERE id=$1 AND mind_id=$2`, id, mind, string(st))
	if err == nil && tag.RowsAffected() == 0 {
		return storage.ErrNotFound
	}
	return err
}

// UpdateConfidence sets a new confidence (used by consolidation and corrections).
func (s *Store) UpdateConfidence(ctx context.Context, q storage.Querier, mind, id ids.ID, confidence float64) error {
	tag, err := q.Exec(ctx, `UPDATE memories SET confidence=$3, updated_at=now() WHERE id=$1 AND mind_id=$2`, id, mind, clamp(confidence))
	if err == nil && tag.RowsAffected() == 0 {
		return storage.ErrNotFound
	}
	return err
}

// Reinforce records a retrieval: count, timestamp and activation towards 1.
func (s *Store) Reinforce(ctx context.Context, q storage.Querier, mind, id ids.ID) error {
	_, err := q.Exec(ctx, `UPDATE memories SET retrieval_count = retrieval_count + 1, last_retrieved_at = now(), activation = LEAST(1, activation + (1 - activation) * 0.5), updated_at = now() WHERE id=$1 AND mind_id=$2`, id, mind)
	return err
}

// Decay lowers activation of memories not retrieved recently. Importance slows
// decay; nothing is deleted. Returns the number of rows touched.
func (s *Store) Decay(ctx context.Context, mind ids.ID, halfLife time.Duration) (int64, error) {
	if halfLife <= 0 {
		halfLife = 14 * 24 * time.Hour
	}
	// activation *= 0.5 ^ (age / (halfLife * (1 + importance)))
	tag, err := s.db.Exec(ctx, `UPDATE memories SET activation = GREATEST(0.01, activation * power(0.5, EXTRACT(EPOCH FROM (now() - coalesce(last_retrieved_at, created_at))) / ($2 * (1 + importance)))), updated_at = now()
		WHERE mind_id=$1 AND status='active' AND activation > 0.01 AND coalesce(last_retrieved_at, created_at) < now() - interval '1 hour'`, mind, halfLife.Seconds())
	if err != nil {
		return 0, err
	}
	return tag.RowsAffected(), nil
}

// Relate adds an edge between two memories of the same mind.
func (s *Store) Relate(ctx context.Context, q storage.Querier, mind, from, to ids.ID, rel concepts.Relation, confidence float64, source string) (Relationship, error) {
	if !rel.Valid() {
		return Relationship{}, fmt.Errorf("%w: unknown relation %q", ErrInvalid, rel)
	}
	if from == to {
		return Relationship{}, fmt.Errorf("%w: self relation", ErrInvalid)
	}
	r := Relationship{ID: ids.New(), FromMemory: from, ToMemory: to, Relation: rel, Confidence: clamp(confidence), Source: source, CreatedAt: time.Now()}
	// Both endpoints must belong to the mind; the subquery enforces it.
	tag, err := q.Exec(ctx, `INSERT INTO memory_relationships (id, mind_id, from_memory, to_memory, relation, confidence, source)
		SELECT $1, $2, $3, $4, $5, $6, $7 WHERE (SELECT count(*) FROM memories WHERE id IN ($3,$4) AND mind_id=$2) = 2
		ON CONFLICT (from_memory, to_memory, relation) DO UPDATE SET confidence = GREATEST(memory_relationships.confidence, excluded.confidence)`,
		r.ID, mind, from, to, string(rel), r.Confidence, source)
	if err != nil {
		return Relationship{}, err
	}
	if tag.RowsAffected() == 0 {
		return Relationship{}, fmt.Errorf("%w: memories not found in mind", ErrInvalid)
	}
	return r, nil
}

// Related lists edges touching a memory, with the other endpoint loaded.
type Related struct {
	Relationship
	Other     Memory `json:"other"`
	Direction string `json:"direction"` // out | in
}

// Relationships returns edges in both directions.
func (s *Store) Relationships(ctx context.Context, mind, id ids.ID) ([]Related, error) {
	rows, err := s.db.Query(ctx, `SELECT r.id, r.from_memory, r.to_memory, r.relation, r.confidence, r.source, r.created_at, CASE WHEN r.from_memory=$1 THEN 'out' ELSE 'in' END, `+cols+`
		FROM memory_relationships r JOIN memories m ON m.id = CASE WHEN r.from_memory=$1 THEN r.to_memory ELSE r.from_memory END
		WHERE r.mind_id=$2 AND (r.from_memory=$1 OR r.to_memory=$1) ORDER BY r.created_at`, id, mind)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	var out []Related
	for rows.Next() {
		var r Related
		var rel string
		var kind, epi, status, trust string
		var detail []byte
		var origin, supersedes *string
		m := &r.Other
		if err := rows.Scan(&r.ID, &r.FromMemory, &r.ToMemory, &rel, &r.Confidence, &r.Source, &r.CreatedAt, &r.Direction,
			&m.ID, &m.MindID, &kind, &epi, &status, &m.Content, &detail, &m.Source, &trust, &m.Credibility, &m.Confidence, &m.Importance, &m.Salience, &m.Activation,
			&m.Valence, &m.Arousal, &m.OccurredAt, &m.CreatedAt, &m.UpdatedAt, &m.RetrievalCount, &m.LastRetrievedAt, &origin, &supersedes); err != nil {
			return nil, err
		}
		r.Relation = concepts.Relation(rel)
		m.Kind, m.Epistemic, m.Status, m.Trust = concepts.MemoryKind(kind), concepts.Epistemic(epi), Status(status), events.Trust(trust)
		json.Unmarshal(detail, &m.Detail) //nolint:errcheck
		if origin != nil {
			m.OriginEvent = ids.ID(*origin)
		}
		out = append(out, r)
	}
	return out, rows.Err()
}

// Stats summarises a mind's memory for overview pages.
type Stats struct {
	Total  int64            `json:"total"`
	ByKind map[string]int64 `json:"by_kind"`
}

// Stats counts active memories by kind.
func (s *Store) Stats(ctx context.Context, mind ids.ID) (Stats, error) {
	st := Stats{ByKind: map[string]int64{}}
	rows, err := s.db.Query(ctx, `SELECT kind, count(*) FROM memories WHERE mind_id=$1 AND status='active' GROUP BY kind`, mind)
	if err != nil {
		return st, err
	}
	defer rows.Close()
	for rows.Next() {
		var k string
		var n int64
		if err := rows.Scan(&k, &n); err != nil {
			return st, err
		}
		st.ByKind[k] = n
		st.Total += n
	}
	return st, rows.Err()
}