feat: add configuration, logging, metrics, scheduler and storage foundation
- config: defaults -> JSON file -> NOEMA_* env with validation and redaction - observability: slog with secret-redacting handler; Prometheus registry with the metric set from the plan (events, workspace, memory, LLM, tools, queues) - scheduler: supervised Group and bounded Pool so no goroutine is unowned - storage: pgx pool with statement timeout, embedded migration runner (advisory lock, checksums, up/down/status), per-test schema helper - migrations/0001: append-only guard trigger, users, sessions, api tokens, recovery codes, audit log, settings - pkg/ids: UUIDv7 with monotonic counter
This commit is contained in:
Andy Dixon
@@ -0,0 +1,89 @@
|
||||
// Package storage opens the PostgreSQL pool, applies embedded migrations and
|
||||
// provides small helpers shared by repositories. All SQL lives in the domain
|
||||
// packages; this package knows nothing about minds or events.
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgconn"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// Open connects to PostgreSQL and verifies the connection.
|
||||
func Open(ctx context.Context, url string) (*pgxpool.Pool, error) {
|
||||
cfg, err := pgxpool.ParseConfig(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: parse url: %w", err)
|
||||
}
|
||||
if cfg.MaxConns < 8 {
|
||||
cfg.MaxConns = 8
|
||||
}
|
||||
cfg.MaxConnLifetime = time.Hour
|
||||
cfg.HealthCheckPeriod = 30 * time.Second
|
||||
if cfg.ConnConfig.RuntimeParams == nil {
|
||||
cfg.ConnConfig.RuntimeParams = map[string]string{}
|
||||
}
|
||||
cfg.ConnConfig.RuntimeParams["application_name"] = "noema"
|
||||
// A statement timeout bounds runaway queries from any subsystem (T18).
|
||||
if _, set := cfg.ConnConfig.RuntimeParams["statement_timeout"]; !set {
|
||||
cfg.ConnConfig.RuntimeParams["statement_timeout"] = "30000"
|
||||
}
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("storage: connect: %w", err)
|
||||
}
|
||||
pingCtx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
if err := pool.Ping(pingCtx); err != nil {
|
||||
pool.Close()
|
||||
return nil, fmt.Errorf("storage: ping: %w", err)
|
||||
}
|
||||
return pool, nil
|
||||
}
|
||||
|
||||
// ErrNotFound is returned by repositories when a row does not exist.
|
||||
var ErrNotFound = errors.New("not found")
|
||||
|
||||
// ErrConflict is returned when a uniqueness constraint is violated.
|
||||
var ErrConflict = errors.New("conflict")
|
||||
|
||||
// Translate maps driver errors to storage sentinels so callers can use errors.Is.
|
||||
func Translate(err error) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return ErrNotFound
|
||||
}
|
||||
var pgErr *pgconn.PgError
|
||||
if errors.As(err, &pgErr) && pgErr.Code == "23505" {
|
||||
return fmt.Errorf("%w: %s", ErrConflict, pgErr.ConstraintName)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// Tx runs fn in a transaction, committing on nil error and rolling back otherwise.
|
||||
func Tx(ctx context.Context, pool *pgxpool.Pool, fn func(tx pgx.Tx) error) error {
|
||||
tx, err := pool.Begin(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("begin: %w", err)
|
||||
}
|
||||
defer tx.Rollback(ctx) //nolint:errcheck // rollback after commit is a no-op
|
||||
if err := fn(tx); err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Commit(ctx)
|
||||
}
|
||||
|
||||
// Querier is implemented by both *pgxpool.Pool and pgx.Tx so repositories can
|
||||
// run inside or outside a transaction.
|
||||
type Querier interface {
|
||||
Exec(ctx context.Context, sql string, args ...any) (pgconn.CommandTag, error)
|
||||
Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error)
|
||||
QueryRow(ctx context.Context, sql string, args ...any) pgx.Row
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// migrationLock is an arbitrary advisory lock key so concurrent daemons do not
|
||||
// race to apply the same migration.
|
||||
const migrationLock = 0x6e6f656d61 // "noema"
|
||||
|
||||
// Migration is one numbered schema step.
|
||||
type Migration struct {
|
||||
Version int
|
||||
Name string
|
||||
Up, Down string
|
||||
Checksum string
|
||||
}
|
||||
|
||||
// MigrationStatus describes one migration and whether it has been applied.
|
||||
type MigrationStatus struct {
|
||||
Migration
|
||||
Applied bool
|
||||
AppliedAt string
|
||||
Drift bool // applied checksum differs from the embedded file
|
||||
}
|
||||
|
||||
// LoadMigrations parses NNNN_name.up.sql / .down.sql pairs from fsys.
|
||||
func LoadMigrations(fsys fs.FS) ([]Migration, error) {
|
||||
entries, err := fs.ReadDir(fsys, ".")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byVersion := map[int]*Migration{}
|
||||
for _, e := range entries {
|
||||
name := e.Name()
|
||||
if !strings.HasSuffix(name, ".sql") {
|
||||
continue
|
||||
}
|
||||
var version int
|
||||
var rest string
|
||||
if _, err := fmt.Sscanf(name, "%d_%s", &version, &rest); err != nil {
|
||||
return nil, fmt.Errorf("migration %s: bad name", name)
|
||||
}
|
||||
body, err := fs.ReadFile(fsys, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m := byVersion[version]
|
||||
if m == nil {
|
||||
m = &Migration{Version: version}
|
||||
byVersion[version] = m
|
||||
}
|
||||
switch {
|
||||
case strings.HasSuffix(name, ".up.sql"):
|
||||
m.Name = strings.TrimSuffix(rest, ".up.sql")
|
||||
m.Up = string(body)
|
||||
sum := sha256.Sum256(body)
|
||||
m.Checksum = hex.EncodeToString(sum[:])
|
||||
case strings.HasSuffix(name, ".down.sql"):
|
||||
m.Down = string(body)
|
||||
default:
|
||||
return nil, fmt.Errorf("migration %s: must end in .up.sql or .down.sql", name)
|
||||
}
|
||||
}
|
||||
out := make([]Migration, 0, len(byVersion))
|
||||
for _, m := range byVersion {
|
||||
if m.Up == "" {
|
||||
return nil, fmt.Errorf("migration %d has no up file", m.Version)
|
||||
}
|
||||
out = append(out, *m)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Version < out[j].Version })
|
||||
for i, m := range out {
|
||||
if m.Version != i+1 {
|
||||
return nil, fmt.Errorf("migrations must be contiguous from 1; got %d at position %d", m.Version, i+1)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func ensureTable(ctx context.Context, tx pgx.Tx) error {
|
||||
_, err := tx.Exec(ctx, `CREATE TABLE IF NOT EXISTS schema_migrations (
|
||||
version integer PRIMARY KEY, name text NOT NULL, checksum text NOT NULL, applied_at timestamptz NOT NULL DEFAULT now())`)
|
||||
return err
|
||||
}
|
||||
|
||||
// MigrateUp applies all pending migrations and returns the names applied.
|
||||
func MigrateUp(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS) ([]string, error) {
|
||||
ms, err := LoadMigrations(fsys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var applied []string
|
||||
err = withLock(ctx, pool, func(tx pgx.Tx) error {
|
||||
if err := ensureTable(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
done, err := appliedVersions(ctx, tx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range ms {
|
||||
if sum, ok := done[m.Version]; ok {
|
||||
if sum != m.Checksum {
|
||||
return fmt.Errorf("migration %04d_%s was modified after being applied", m.Version, m.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if _, err := tx.Exec(ctx, m.Up); err != nil {
|
||||
return fmt.Errorf("migration %04d_%s: %w", m.Version, m.Name, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `INSERT INTO schema_migrations (version, name, checksum) VALUES ($1,$2,$3)`, m.Version, m.Name, m.Checksum); err != nil {
|
||||
return err
|
||||
}
|
||||
applied = append(applied, fmt.Sprintf("%04d_%s", m.Version, m.Name))
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return applied, err
|
||||
}
|
||||
|
||||
// MigrateDown rolls back the most recently applied migration.
|
||||
func MigrateDown(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS) (string, error) {
|
||||
ms, err := LoadMigrations(fsys)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
var rolled string
|
||||
err = withLock(ctx, pool, func(tx pgx.Tx) error {
|
||||
if err := ensureTable(ctx, tx); err != nil {
|
||||
return err
|
||||
}
|
||||
var version int
|
||||
err := tx.QueryRow(ctx, `SELECT coalesce(max(version),0) FROM schema_migrations`).Scan(&version)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if version == 0 {
|
||||
return nil
|
||||
}
|
||||
m := ms[version-1]
|
||||
if m.Down == "" {
|
||||
return fmt.Errorf("migration %04d_%s has no down file", m.Version, m.Name)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, m.Down); err != nil {
|
||||
return fmt.Errorf("rollback %04d_%s: %w", m.Version, m.Name, err)
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM schema_migrations WHERE version=$1`, version); err != nil {
|
||||
return err
|
||||
}
|
||||
rolled = fmt.Sprintf("%04d_%s", m.Version, m.Name)
|
||||
return nil
|
||||
})
|
||||
return rolled, err
|
||||
}
|
||||
|
||||
// Status reports every known migration and its applied state.
|
||||
func Status(ctx context.Context, pool *pgxpool.Pool, fsys fs.FS) ([]MigrationStatus, error) {
|
||||
ms, err := LoadMigrations(fsys)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type row struct{ checksum, at string }
|
||||
done := map[int]row{}
|
||||
rows, err := pool.Query(ctx, `SELECT version, checksum, applied_at::text FROM schema_migrations`)
|
||||
if err == nil {
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var v int
|
||||
var r row
|
||||
if err := rows.Scan(&v, &r.checksum, &r.at); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
done[v] = r
|
||||
}
|
||||
} // a missing table simply means nothing is applied
|
||||
out := make([]MigrationStatus, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
s := MigrationStatus{Migration: m}
|
||||
if r, ok := done[m.Version]; ok {
|
||||
s.Applied, s.AppliedAt, s.Drift = true, r.at, r.checksum != m.Checksum
|
||||
}
|
||||
out = append(out, s)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func appliedVersions(ctx context.Context, tx pgx.Tx) (map[int]string, error) {
|
||||
rows, err := tx.Query(ctx, `SELECT version, checksum FROM schema_migrations`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[int]string{}
|
||||
for rows.Next() {
|
||||
var v int
|
||||
var c string
|
||||
if err := rows.Scan(&v, &c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[v] = c
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func withLock(ctx context.Context, pool *pgxpool.Pool, fn func(tx pgx.Tx) error) error {
|
||||
return Tx(ctx, pool, func(tx pgx.Tx) error {
|
||||
if _, err := tx.Exec(ctx, `SELECT pg_advisory_xact_lock($1)`, int64(migrationLock)); err != nil {
|
||||
return fmt.Errorf("migration lock: %w", err)
|
||||
}
|
||||
return fn(tx)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
|
||||
"github.com/andydixon/noema/migrations"
|
||||
)
|
||||
|
||||
func TestLoadMigrationsValidation(t *testing.T) {
|
||||
if _, err := LoadMigrations(migrations.FS); err != nil {
|
||||
t.Fatalf("embedded migrations invalid: %v", err)
|
||||
}
|
||||
bad := fstest.MapFS{"0002_x.up.sql": {Data: []byte("select 1")}}
|
||||
if _, err := LoadMigrations(bad); err == nil {
|
||||
t.Fatal("expected contiguity error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMigrateUpDownAndAppendOnly(t *testing.T) {
|
||||
pool := TestPool(t, migrations.FS)
|
||||
ctx := context.Background()
|
||||
st, err := Status(ctx, pool, migrations.FS)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, s := range st {
|
||||
if !s.Applied || s.Drift {
|
||||
t.Fatalf("migration %d not cleanly applied: %+v", s.Version, s)
|
||||
}
|
||||
}
|
||||
// Append-only trigger on audit_log.
|
||||
if _, err := pool.Exec(ctx, `INSERT INTO audit_log (id, actor_kind, action) VALUES (gen_random_uuid(),'system','test')`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `UPDATE audit_log SET action='x'`); err == nil {
|
||||
t.Fatal("update on audit_log should be rejected")
|
||||
}
|
||||
if _, err := pool.Exec(ctx, `DELETE FROM audit_log`); err == nil {
|
||||
t.Fatal("delete on audit_log should be rejected")
|
||||
}
|
||||
// Roll everything back and forward again.
|
||||
for i := len(st); i > 0; i-- {
|
||||
if _, err := MigrateDown(ctx, pool, migrations.FS); err != nil {
|
||||
t.Fatalf("down %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
applied, err := MigrateUp(ctx, pool, migrations.FS)
|
||||
if err != nil || len(applied) != len(st) {
|
||||
t.Fatalf("re-apply: %v (%d)", err, len(applied))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"io/fs"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
// TestPool returns a pool bound to a fresh schema with all migrations applied.
|
||||
// It skips the test when NOEMA_TEST_DATABASE_URL is unset. Each test gets its
|
||||
// own schema so tests may run in parallel and leave nothing behind.
|
||||
func TestPool(t testing.TB, migrations fs.FS) *pgxpool.Pool {
|
||||
t.Helper()
|
||||
url := os.Getenv("NOEMA_TEST_DATABASE_URL")
|
||||
if url == "" {
|
||||
t.Skip("NOEMA_TEST_DATABASE_URL not set")
|
||||
}
|
||||
ctx := context.Background()
|
||||
var b [6]byte
|
||||
rand.Read(b[:])
|
||||
schema := "t_" + hex.EncodeToString(b[:])
|
||||
|
||||
admin, err := Open(ctx, url)
|
||||
if err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
if _, err := admin.Exec(ctx, "CREATE SCHEMA "+schema); err != nil {
|
||||
t.Fatalf("create schema: %v", err)
|
||||
}
|
||||
cfg, _ := pgxpool.ParseConfig(url)
|
||||
cfg.ConnConfig.RuntimeParams = map[string]string{"search_path": schema + ",public"}
|
||||
pool, err := pgxpool.NewWithConfig(ctx, cfg)
|
||||
if err != nil {
|
||||
t.Fatalf("connect schema: %v", err)
|
||||
}
|
||||
if _, err := MigrateUp(ctx, pool, migrations); err != nil {
|
||||
t.Fatalf("migrate: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
pool.Close()
|
||||
admin.Exec(ctx, "DROP SCHEMA "+schema+" CASCADE") //nolint:errcheck
|
||||
admin.Close()
|
||||
})
|
||||
return pool
|
||||
}
|
||||
Reference in New Issue
Block a user