Code sync, security fixes

2026-08-11 10:40:10 +01:00
parent 4e00deedf8
commit 53dbc28383
141 changed files with 36194 additions and 0 deletions
@@ -0,0 +1,88 @@
+// Package observability provides structured logging, request/job correlation
+// IDs, and Prometheus-compatible application metrics.
+//
+// Logging policy: never log passwords, session tokens, API credentials, whole
+// uploaded documents or embedding vectors. Content fields are recorded as
+// lengths and hashes, not values.
+package observability
+
+import (
+	"context"
+	"log/slog"
+	"os"
+	"strings"
+)
+
+type ctxKey int
+
+const (
+	ctxKeyRequestID ctxKey = iota
+	ctxKeyJobID
+	ctxKeyLogger
+)
+
+// NewLogger builds the application logger.
+func NewLogger(level string, asJSON bool) *slog.Logger {
+	var lv slog.Level
+	switch strings.ToLower(level) {
+	case "debug":
+		lv = slog.LevelDebug
+	case "warn":
+		lv = slog.LevelWarn
+	case "error":
+		lv = slog.LevelError
+	default:
+		lv = slog.LevelInfo
+	}
+	opts := &slog.HandlerOptions{Level: lv}
+	var h slog.Handler
+	if asJSON {
+		h = slog.NewJSONHandler(os.Stdout, opts)
+	} else {
+		h = slog.NewTextHandler(os.Stdout, opts)
+	}
+	return slog.New(h)
+}
+
+// WithRequestID attaches a request correlation ID to ctx.
+func WithRequestID(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, ctxKeyRequestID, id)
+}
+
+// RequestID returns the correlation ID attached to ctx, if any.
+func RequestID(ctx context.Context) string {
+	v, _ := ctx.Value(ctxKeyRequestID).(string)
+	return v
+}
+
+// WithJobID attaches a job correlation ID to ctx.
+func WithJobID(ctx context.Context, id string) context.Context {
+	return context.WithValue(ctx, ctxKeyJobID, id)
+}
+
+// JobID returns the job ID attached to ctx, if any.
+func JobID(ctx context.Context) string {
+	v, _ := ctx.Value(ctxKeyJobID).(string)
+	return v
+}
+
+// WithLogger attaches a logger to ctx.
+func WithLogger(ctx context.Context, l *slog.Logger) context.Context {
+	return context.WithValue(ctx, ctxKeyLogger, l)
+}
+
+// LoggerFrom returns the context logger, enriched with any correlation IDs.
+// It never returns nil.
+func LoggerFrom(ctx context.Context) *slog.Logger {
+	l, _ := ctx.Value(ctxKeyLogger).(*slog.Logger)
+	if l == nil {
+		l = slog.Default()
+	}
+	if id := RequestID(ctx); id != "" {
+		l = l.With("request_id", id)
+	}
+	if id := JobID(ctx); id != "" {
+		l = l.With("job_id", id)
+	}
+	return l
+}
@@ -0,0 +1,144 @@
+package observability
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/collectors"
+)
+
+// Metrics holds Relata's application metrics.
+//
+// These describe the health of the *service* — request rates, job outcomes,
+// provider latency. They deliberately contain no stored-knowledge cardinality:
+// no item IDs, no entity names, no query strings ever become label values.
+type Metrics struct {
+	Registry *prometheus.Registry
+
+	HTTPRequests *prometheus.CounterVec
+	HTTPDuration *prometheus.HistogramVec
+	HTTPInFlight prometheus.Gauge
+
+	JobsProcessed *prometheus.CounterVec
+	JobDuration   *prometheus.HistogramVec
+	JobsQueued    *prometheus.GaugeVec
+
+	AIRequests *prometheus.CounterVec
+	AIDuration *prometheus.HistogramVec
+
+	StorageOps      *prometheus.CounterVec
+	StorageDuration *prometheus.HistogramVec
+
+	IngestItems   *prometheus.CounterVec
+	ModelInvalid  *prometheus.CounterVec
+	AuthAttempts  *prometheus.CounterVec
+	SearchOps     *prometheus.CounterVec
+	SearchLatency prometheus.Histogram
+}
+
+var (
+	metricsOnce sync.Once
+	metricsInst *Metrics
+)
+
+// NewMetrics returns the process-wide metrics registry, creating it once.
+func NewMetrics() *Metrics {
+	metricsOnce.Do(func() {
+		reg := prometheus.NewRegistry()
+		reg.MustRegister(collectors.NewGoCollector())
+		reg.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
+
+		m := &Metrics{Registry: reg}
+
+		m.HTTPRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_http_requests_total",
+			Help: "HTTP requests by route pattern, method and status class.",
+		}, []string{"route", "method", "status"})
+
+		m.HTTPDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "relata_http_request_duration_seconds",
+			Help:    "HTTP request duration in seconds.",
+			Buckets: []float64{0.005, 0.025, 0.1, 0.25, 1, 2.5, 10, 30},
+		}, []string{"route", "method"})
+
+		m.HTTPInFlight = prometheus.NewGauge(prometheus.GaugeOpts{
+			Name: "relata_http_in_flight_requests",
+			Help: "HTTP requests currently being served.",
+		})
+
+		m.JobsProcessed = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_jobs_processed_total",
+			Help: "Background jobs finished, by type and outcome.",
+		}, []string{"type", "outcome"})
+
+		m.JobDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "relata_job_duration_seconds",
+			Help:    "Background job execution duration in seconds.",
+			Buckets: []float64{0.05, 0.25, 1, 5, 15, 60, 180, 600},
+		}, []string{"type"})
+
+		m.JobsQueued = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "relata_jobs_queued",
+			Help: "Jobs currently in each status.",
+		}, []string{"status"})
+
+		m.AIRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_ai_requests_total",
+			Help: "AI provider calls by provider, operation and outcome.",
+		}, []string{"provider", "operation", "outcome"})
+
+		m.AIDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "relata_ai_request_duration_seconds",
+			Help:    "AI provider call duration in seconds.",
+			Buckets: []float64{0.1, 0.5, 1, 2.5, 5, 15, 60, 180},
+		}, []string{"provider", "operation"})
+
+		m.StorageOps = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_storage_operations_total",
+			Help: "Storage operations by store, operation and outcome.",
+		}, []string{"store", "operation", "outcome"})
+
+		m.StorageDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:    "relata_storage_operation_duration_seconds",
+			Help:    "Storage operation duration in seconds.",
+			Buckets: []float64{0.001, 0.005, 0.025, 0.1, 0.5, 2, 10},
+		}, []string{"store", "operation"})
+
+		m.IngestItems = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_ingest_items_total",
+			Help: "Items accepted for ingestion by source type and outcome.",
+		}, []string{"source_type", "outcome"})
+
+		m.ModelInvalid = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_model_invalid_output_total",
+			Help: "Model responses rejected by schema validation, by task and reason.",
+		}, []string{"task", "reason"})
+
+		m.AuthAttempts = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_auth_attempts_total",
+			Help: "Authentication attempts by outcome.",
+		}, []string{"outcome"})
+
+		m.SearchOps = prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "relata_search_operations_total",
+			Help: "Search operations by mode and outcome.",
+		}, []string{"mode", "outcome"})
+
+		m.SearchLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
+			Name:    "relata_search_duration_seconds",
+			Help:    "End-to-end search duration in seconds.",
+			Buckets: []float64{0.01, 0.05, 0.15, 0.5, 1.5, 5, 20},
+		})
+
+		reg.MustRegister(
+			m.HTTPRequests, m.HTTPDuration, m.HTTPInFlight,
+			m.JobsProcessed, m.JobDuration, m.JobsQueued,
+			m.AIRequests, m.AIDuration,
+			m.StorageOps, m.StorageDuration,
+			m.IngestItems, m.ModelInvalid, m.AuthAttempts,
+			m.SearchOps, m.SearchLatency,
+		)
+		metricsInst = m
+	})
+	return metricsInst
+}