feat: add configuration, logging, metrics, scheduler and storage foundation
- config: defaults -> JSON file -> NOEMA_* env with validation and redaction - observability: slog with secret-redacting handler; Prometheus registry with the metric set from the plan (events, workspace, memory, LLM, tools, queues) - scheduler: supervised Group and bounded Pool so no goroutine is unowned - storage: pgx pool with statement timeout, embedded migration runner (advisory lock, checksums, up/down/status), per-test schema helper - migrations/0001: append-only guard trigger, users, sessions, api tokens, recovery codes, audit log, settings - pkg/ids: UUIDv7 with monotonic counter
This commit is contained in:
Andy Dixon
@@ -0,0 +1,62 @@
|
||||
// Package observability configures structured logging and Prometheus metrics.
|
||||
package observability
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"log/slog"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// secretKeys are attribute names whose values are never written to logs,
|
||||
// whatever the caller passed. The list is deliberately broad.
|
||||
var secretKeys = []string{"password", "passwd", "secret", "token", "api_key", "apikey", "authorization", "cookie", "master_key", "recovery_code"}
|
||||
|
||||
// NewLogger builds a slog.Logger at the given level and format ("text" or
|
||||
// "json") that redacts any attribute whose key looks like a secret.
|
||||
func NewLogger(w io.Writer, level, format string) *slog.Logger {
|
||||
var lvl slog.Level
|
||||
switch level {
|
||||
case "debug":
|
||||
lvl = slog.LevelDebug
|
||||
case "warn":
|
||||
lvl = slog.LevelWarn
|
||||
case "error":
|
||||
lvl = slog.LevelError
|
||||
default:
|
||||
lvl = slog.LevelInfo
|
||||
}
|
||||
opts := &slog.HandlerOptions{Level: lvl, ReplaceAttr: redact}
|
||||
var h slog.Handler
|
||||
if format == "json" {
|
||||
h = slog.NewJSONHandler(w, opts)
|
||||
} else {
|
||||
h = slog.NewTextHandler(w, opts)
|
||||
}
|
||||
return slog.New(h)
|
||||
}
|
||||
|
||||
func redact(_ []string, a slog.Attr) slog.Attr {
|
||||
k := strings.ToLower(a.Key)
|
||||
for _, s := range secretKeys {
|
||||
if strings.Contains(k, s) {
|
||||
return slog.String(a.Key, "[redacted]")
|
||||
}
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
type ctxKey struct{}
|
||||
|
||||
// WithLogger stores a request-scoped logger in ctx.
|
||||
func WithLogger(ctx context.Context, l *slog.Logger) context.Context {
|
||||
return context.WithValue(ctx, ctxKey{}, l)
|
||||
}
|
||||
|
||||
// Logger returns the request-scoped logger or the default logger.
|
||||
func Logger(ctx context.Context) *slog.Logger {
|
||||
if l, ok := ctx.Value(ctxKey{}).(*slog.Logger); ok {
|
||||
return l
|
||||
}
|
||||
return slog.Default()
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestRedaction(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
l := NewLogger(&buf, "info", "json")
|
||||
l.Info("x", "api_key", "sk-123", "Password", "hunter2", "user", "andy")
|
||||
out := buf.String()
|
||||
if strings.Contains(out, "sk-123") || strings.Contains(out, "hunter2") || !strings.Contains(out, "andy") {
|
||||
t.Fatalf("redaction wrong: %s", out)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package observability
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/collectors"
|
||||
)
|
||||
|
||||
// Metrics holds every Prometheus instrument Noema exposes. Instruments are
|
||||
// created once here so that names are visible in one place and no package
|
||||
// registers globals.
|
||||
type Metrics struct {
|
||||
Registry *prometheus.Registry
|
||||
|
||||
HTTPRequests *prometheus.CounterVec
|
||||
HTTPDuration *prometheus.HistogramVec
|
||||
AuthFailures prometheus.Counter
|
||||
Events *prometheus.CounterVec
|
||||
Candidates prometheus.Counter
|
||||
Admissions prometheus.Counter
|
||||
Retrievals prometheus.Counter
|
||||
Beliefs prometheus.Gauge
|
||||
Predictions prometheus.Counter
|
||||
PredictionErr prometheus.Counter
|
||||
LLMRequests *prometheus.CounterVec
|
||||
LLMTokens *prometheus.CounterVec
|
||||
LLMFailures *prometheus.CounterVec
|
||||
ToolRequests *prometheus.CounterVec
|
||||
ToolDenied *prometheus.CounterVec
|
||||
QueueDepth *prometheus.GaugeVec
|
||||
Workers *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
// NewMetrics creates and registers all instruments on a fresh registry.
|
||||
func NewMetrics() *Metrics {
|
||||
reg := prometheus.NewRegistry()
|
||||
m := &Metrics{
|
||||
Registry: reg,
|
||||
HTTPRequests: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_http_requests_total", Help: "HTTP requests by route and status."}, []string{"route", "status"}),
|
||||
HTTPDuration: prometheus.NewHistogramVec(prometheus.HistogramOpts{Name: "noema_http_request_duration_seconds", Help: "HTTP request latency.", Buckets: prometheus.DefBuckets}, []string{"route"}),
|
||||
AuthFailures: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_auth_failures_total", Help: "Failed authentication attempts."}),
|
||||
Events: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_events_total", Help: "Cognitive events appended by type."}, []string{"type"}),
|
||||
Candidates: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_workspace_candidates_total", Help: "Thought candidates scored."}),
|
||||
Admissions: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_workspace_admissions_total", Help: "Thoughts admitted to the workspace."}),
|
||||
Retrievals: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_memory_retrievals_total", Help: "Memory retrievals."}),
|
||||
Beliefs: prometheus.NewGauge(prometheus.GaugeOpts{Name: "noema_beliefs_total", Help: "Beliefs currently held across minds."}),
|
||||
Predictions: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_predictions_total", Help: "Predictions created."}),
|
||||
PredictionErr: prometheus.NewCounter(prometheus.CounterOpts{Name: "noema_prediction_errors_total", Help: "Prediction error events."}),
|
||||
LLMRequests: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_llm_requests_total", Help: "LLM requests by provider and role."}, []string{"provider", "role"}),
|
||||
LLMTokens: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_llm_tokens_total", Help: "LLM tokens by provider and direction."}, []string{"provider", "direction"}),
|
||||
LLMFailures: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_llm_failures_total", Help: "LLM failures by provider."}, []string{"provider"}),
|
||||
ToolRequests: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_tool_requests_total", Help: "Capability invocations by capability."}, []string{"capability"}),
|
||||
ToolDenied: prometheus.NewCounterVec(prometheus.CounterOpts{Name: "noema_tool_denied_total", Help: "Denied capability invocations."}, []string{"capability"}),
|
||||
QueueDepth: prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "noema_cognitive_queue_depth", Help: "Pending work per queue."}, []string{"queue"}),
|
||||
Workers: prometheus.NewGaugeVec(prometheus.GaugeOpts{Name: "noema_worker_count", Help: "Busy workers per pool."}, []string{"pool"}),
|
||||
}
|
||||
reg.MustRegister(
|
||||
collectors.NewGoCollector(), collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}),
|
||||
m.HTTPRequests, m.HTTPDuration, m.AuthFailures, m.Events, m.Candidates, m.Admissions, m.Retrievals,
|
||||
m.Beliefs, m.Predictions, m.PredictionErr, m.LLMRequests, m.LLMTokens, m.LLMFailures, m.ToolRequests,
|
||||
m.ToolDenied, m.QueueDepth, m.Workers,
|
||||
)
|
||||
return m
|
||||
}
|
||||
Reference in New Issue
Block a user