Code sync, security fixes

2026-08-11 10:40:10 +01:00
parent 4e00deedf8
commit 53dbc28383
141 changed files with 36194 additions and 0 deletions
@@ -0,0 +1,165 @@
+package main
+
+import (
+	"bufio"
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log/slog"
+	"os"
+	"strings"
+	"time"
+
+	"golang.org/x/term"
+
+	"github.com/andydixon/relata/internal/auth"
+	"github.com/andydixon/relata/internal/config"
+	"github.com/andydixon/relata/internal/domain"
+	"github.com/andydixon/relata/internal/ids"
+	"github.com/andydixon/relata/internal/observability"
+	"github.com/andydixon/relata/internal/storage/mongo"
+)
+
+// loggerType is the logger the commands share.
+type loggerType = *slog.Logger
+
+// newID returns a fresh identifier.
+func newID() string { return ids.New() }
+
+// cmdCreateAdmin creates the first administrator account.
+//
+// The password is read from the terminal without echo, or from an environment
+// variable for automated setup. It is never accepted as a command-line flag,
+// because arguments are visible in the process list and shell history.
+func cmdCreateAdmin(args []string) error {
+	fs := flag.NewFlagSet("createadmin", flag.ExitOnError)
+	configFile := fs.String("config", ".env", "path to a key=value configuration file (optional)")
+	username := fs.String("username", "", "username for the new administrator (required)")
+	displayName := fs.String("display-name", "", "display name (optional)")
+	fs.Usage = func() {
+		fmt.Fprint(os.Stderr, `Create the initial administrator account.
+
+Usage:
+  relata createadmin --username <name>
+
+The password is read from the terminal without echo. For unattended setup,
+set RELATA_ADMIN_PASSWORD in the environment instead.
+
+Flags:
+`)
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	if strings.TrimSpace(*username) == "" {
+		fs.Usage()
+		return errors.New("--username is required")
+	}
+	name, err := auth.ValidateUsername(*username)
+	if err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(*configFile)
+	if err != nil {
+		return err
+	}
+	logger := observability.NewLogger(cfg.LogLevel, cfg.LogJSON)
+
+	password, err := readPassword()
+	if err != nil {
+		return err
+	}
+	if err := auth.ValidatePassword(password); err != nil {
+		return err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
+	defer cancel()
+
+	store, err := mongo.Connect(ctx, cfg.Mongo, nil)
+	if err != nil {
+		return fmt.Errorf("mongodb: %w", err)
+	}
+	defer func() { _ = store.Close(context.Background()) }()
+
+	if err := store.EnsureIndexes(ctx); err != nil {
+		return fmt.Errorf("create mongodb indexes: %w", err)
+	}
+
+	ws, err := ensureWorkspace(ctx, store)
+	if err != nil {
+		return fmt.Errorf("workspace: %w", err)
+	}
+
+	hash, err := auth.HashPassword(password)
+	if err != nil {
+		return err
+	}
+	// The plain-text password is not referenced again from here on.
+
+	now := time.Now().UTC()
+	user := &domain.User{
+		ID:           newID(),
+		Username:     name,
+		PasswordHash: hash,
+		DisplayName:  strings.TrimSpace(*displayName),
+		Role:         domain.RoleAdmin,
+		WorkspaceID:  ws.ID,
+		CreatedAt:    now,
+		UpdatedAt:    now,
+	}
+
+	if err := store.Users.Create(ctx, user); err != nil {
+		if errors.Is(err, domain.ErrDuplicate) {
+			return fmt.Errorf("a user named %q already exists", name)
+		}
+		return err
+	}
+
+	logger.Info("administrator created", "username", name, "workspace", ws.Slug)
+	fmt.Printf("Administrator %q created in workspace %q.\nSign in at %s\n",
+		name, ws.Slug, cfg.HTTP.BaseURL)
+	return nil
+}
+
+// readPassword obtains the password without echoing it and without ever
+// placing it on the command line.
+func readPassword() (string, error) {
+	if env := os.Getenv("RELATA_ADMIN_PASSWORD"); env != "" {
+		return env, nil
+	}
+
+	fd := int(os.Stdin.Fd())
+	if !term.IsTerminal(fd) {
+		// Piped input: read a single line. Useful in scripts, and no less safe
+		// than an environment variable.
+		sc := bufio.NewScanner(os.Stdin)
+		if !sc.Scan() {
+			return "", errors.New("no password supplied on standard input")
+		}
+		return strings.TrimRight(sc.Text(), "\r\n"), nil
+	}
+
+	fmt.Print("Password (at least 12 characters): ")
+	first, err := term.ReadPassword(fd)
+	fmt.Println()
+	if err != nil {
+		return "", fmt.Errorf("read password: %w", err)
+	}
+
+	fmt.Print("Confirm password: ")
+	second, err := term.ReadPassword(fd)
+	fmt.Println()
+	if err != nil {
+		return "", fmt.Errorf("read password: %w", err)
+	}
+
+	if string(first) != string(second) {
+		return "", errors.New("the passwords do not match")
+	}
+	return string(first), nil
+}
@@ -0,0 +1,384 @@
+// Command relata runs the Relata application and its administrative commands.
+//
+// Subcommands:
+//
+//	serve         run the HTTP server and background workers (default)
+//	createadmin   create the initial administrator account
+//	seed          load the demonstration dataset
+//	version       print version information
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/andydixon/relata/internal/analysis"
+	"github.com/andydixon/relata/internal/api"
+	"github.com/andydixon/relata/internal/audit"
+	"github.com/andydixon/relata/internal/auth"
+	"github.com/andydixon/relata/internal/config"
+	"github.com/andydixon/relata/internal/domain"
+	"github.com/andydixon/relata/internal/embeddings"
+	"github.com/andydixon/relata/internal/entities"
+	"github.com/andydixon/relata/internal/ingest"
+	"github.com/andydixon/relata/internal/jobs"
+	"github.com/andydixon/relata/internal/observability"
+	"github.com/andydixon/relata/internal/relationships"
+	"github.com/andydixon/relata/internal/search"
+	"github.com/andydixon/relata/internal/storage/mongo"
+	"github.com/andydixon/relata/internal/storage/qdrant"
+	"github.com/andydixon/relata/internal/web"
+)
+
+// version is overridden at build time with -ldflags "-X main.version=…".
+var version = "dev"
+
+func main() {
+	if err := run(); err != nil {
+		fmt.Fprintf(os.Stderr, "relata: %v\n", err)
+		os.Exit(1)
+	}
+}
+
+func run() error {
+	cmd := "serve"
+	args := os.Args[1:]
+	if len(args) > 0 && !isFlag(args[0]) {
+		cmd = args[0]
+		args = args[1:]
+	}
+
+	switch cmd {
+	case "serve":
+		return cmdServe(args)
+	case "createadmin":
+		return cmdCreateAdmin(args)
+	case "seed":
+		return cmdSeed(args)
+	case "version":
+		fmt.Printf("relata %s\n", version)
+		return nil
+	case "help", "-h", "--help":
+		usage()
+		return nil
+	default:
+		usage()
+		return fmt.Errorf("unknown command %q", cmd)
+	}
+}
+
+func isFlag(s string) bool { return len(s) > 0 && s[0] == '-' }
+
+func usage() {
+	fmt.Fprint(os.Stderr, `Relata — a self-organising semantic document database
+
+Usage:
+  relata [command] [flags]
+
+Commands:
+  serve         Run the HTTP server and background workers (default)
+  createadmin   Create the initial administrator account
+  seed          Load the demonstration dataset
+  version       Print the version
+
+Run "relata <command> -h" for the flags of a specific command.
+Configuration is read from the environment; see .env.example.
+`)
+}
+
+func cmdServe(args []string) error {
+	fs := flag.NewFlagSet("serve", flag.ExitOnError)
+	configFile := fs.String("config", ".env", "path to a key=value configuration file (optional)")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(*configFile)
+	if err != nil {
+		return err
+	}
+
+	logger := observability.NewLogger(cfg.LogLevel, cfg.LogJSON)
+	metrics := observability.NewMetrics()
+
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	ctx = observability.WithLogger(ctx, logger)
+
+	logger.Info("starting relata", "version", version, "config", cfg.Redacted())
+
+	// --- Storage ------------------------------------------------------------
+	store, err := mongo.Connect(ctx, cfg.Mongo, metrics)
+	if err != nil {
+		return fmt.Errorf("mongodb: %w", err)
+	}
+	defer func() {
+		shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+		defer cancel()
+		_ = store.Close(shutdownCtx)
+	}()
+
+	if err := store.EnsureIndexes(ctx); err != nil {
+		return fmt.Errorf("create mongodb indexes: %w", err)
+	}
+	logger.Info("mongodb ready", "database", cfg.Mongo.Database)
+
+	vectors := qdrant.New(cfg.Qdrant, metrics)
+	if err := vectors.Health(ctx); err != nil {
+		return fmt.Errorf("qdrant at %s is unreachable: %w", cfg.Qdrant.URL, err)
+	}
+
+	// --- AI providers -------------------------------------------------------
+	retry := embeddings.RetryPolicy{
+		MaxAttempts: cfg.Ollama.MaxRetries,
+		BaseDelay:   cfg.Ollama.RetryBaseDelay,
+		MaxDelay:    cfg.Ollama.RetryMaxDelay,
+	}
+	embedder := embeddings.NewOllama(embeddings.OllamaOptions{
+		BaseURL:      cfg.Ollama.BaseURL,
+		Model:        cfg.Ollama.EmbeddingModel,
+		Timeout:      cfg.Ollama.EmbedTimeout,
+		BatchSize:    cfg.Ollama.EmbedBatchSize,
+		Retry:        retry,
+		ExpectedDims: cfg.Ollama.EmbeddingDims,
+		Metrics:      metrics,
+	})
+	analyser := analysis.NewOllamaAnalyser(analysis.OllamaOptions{
+		BaseURL:   cfg.Ollama.BaseURL,
+		Model:     cfg.Ollama.AnalysisModel,
+		Timeout:   cfg.Ollama.AnalysisTimeout,
+		Retry:     retry,
+		MaxTokens: cfg.Ollama.MaxOutputTokens,
+		Metrics:   metrics,
+	})
+
+	if err := vectors.EnsureCollection(ctx, embedder.Dimensions()); err != nil {
+		return fmt.Errorf("qdrant collection: %w", err)
+	}
+	logger.Info("qdrant ready", "collection", cfg.Qdrant.Collection, "dimensions", embedder.Dimensions())
+
+	// The AI providers are checked but not required: Relata's deterministic
+	// pipeline, search and browsing all work without them, and refusing to
+	// start would make a model restart an outage.
+	if h := embedder.Health(ctx); !h.Available {
+		logger.Warn("embedding model unavailable at startup; items will not be embedded until it returns",
+			"model", h.Model, "detail", h.Detail)
+	}
+	if cfg.Ollama.AnalysisEnabled {
+		if h := analyser.Health(ctx); !h.Available {
+			logger.Warn("analysis model unavailable at startup; extraction falls back to deterministic rules",
+				"model", h.Model, "detail", h.Detail)
+		}
+	}
+
+	// --- Services -----------------------------------------------------------
+	recorder := audit.NewRecorder(store.Audit)
+	analysisSvc := analysis.NewService(analyser, metrics, cfg.Ollama.AnalysisEnabled)
+	entitySvc := entities.NewService(store.Entities, store.Aliases, store.Claims, analysisSvc, cfg)
+	discoverer := relationships.NewDiscoverer(store.Items, store.Chunks, store.Entities,
+		store.Aliases, store.Claims, store.Relationships, vectors, embedder, analysisSvc, cfg)
+	ingestSvc := ingest.NewService(store.Items, store.Jobs, cfg, recorder, metrics)
+	searchSvc := search.NewService(store.Items, store.Chunks, store.Entities,
+		store.Relationships, vectors, embedder, cfg, metrics)
+	answerSvc := search.NewAnswerer(searchSvc, analysisSvc, store.Claims, cfg.Search.AnswerEnabled)
+	reviewer := relationships.NewReviewer(store.Relationships, store.Items, recorder)
+	authSvc := auth.NewService(store.Users, store.Sessions, cfg.Session, cfg.HTTP.SecureCookies, metrics)
+
+	tokenSvc := auth.NewTokenService(store.APITokens, store.Users, auth.TokenConfig{
+		DefaultTTL: cfg.Auth.OAuth.TokenDefaultTTL,
+		MaxTTL:     cfg.Auth.OAuth.TokenMaxTTL,
+	})
+
+	// OAuth is optional. When it is not configured these stay nil and the
+	// handlers report the feature as unavailable, rather than the process
+	// refusing to start — the break-glass local login lives here too.
+	var oauthSvc *auth.OAuthService
+	var provisioner *auth.Provisioner
+	if cfg.Auth.OAuth.Enabled {
+		oauthSvc = auth.NewOAuthService(auth.OAuthConfig{
+			Enabled:             true,
+			ProviderName:        cfg.Auth.OAuth.ProviderName,
+			Issuer:              cfg.Auth.OAuth.Issuer,
+			AuthorizeURL:        cfg.Auth.OAuth.AuthorizeURL,
+			TokenURL:            cfg.Auth.OAuth.TokenURL,
+			UserInfoURL:         cfg.Auth.OAuth.UserInfoURL,
+			JWKSURL:             cfg.Auth.OAuth.JWKSURL,
+			ClientID:            cfg.Auth.OAuth.ClientID,
+			ClientSecret:        cfg.Auth.OAuth.ClientSecret,
+			RedirectURI:         cfg.RedirectURI(),
+			Scopes:              cfg.Auth.OAuth.Scopes,
+			AllowedEmailDomains: cfg.Auth.OAuth.AllowedEmailDomains,
+			Timeout:             cfg.Auth.OAuth.Timeout,
+		})
+		provisioner = auth.NewProvisioner(authSvc, store.Users, store.Workspaces)
+		logger.Info("federated sign-in enabled",
+			"provider", cfg.Auth.OAuth.ProviderName,
+			"redirect_uri", cfg.RedirectURI(),
+			"local_login_enabled", cfg.Auth.LocalLoginEnabled)
+	} else {
+		logger.Warn("federated sign-in is not configured; local accounts are the only way in",
+			"local_login_enabled", cfg.Auth.LocalLoginEnabled)
+	}
+
+	pool := jobs.NewPool(store.Jobs, cfg.Workers, metrics)
+	pipeline := ingest.NewPipeline(ingest.Repos{
+		Items: store.Items, Chunks: store.Chunks, Entities: store.Entities,
+		Aliases: store.Aliases, Claims: store.Claims, Rels: store.Relationships,
+		Jobs: store.Jobs, Runs: store.Runs, Models: store.ModelVersions,
+	}, vectors, embedder, analysisSvc, entitySvc, discoverer, ingestSvc, cfg, pool, recorder)
+	pipeline.Register(pool)
+
+	// --- HTTP ---------------------------------------------------------------
+	renderer, err := web.NewRenderer()
+	if err != nil {
+		return fmt.Errorf("templates: %w", err)
+	}
+
+	webHandlers := web.NewHandlers(web.Deps{
+		Cfg: cfg, Renderer: renderer, Auth: authSvc, Version: version,
+		OAuth: oauthSvc, Provisioner: provisioner, Tokens: tokenSvc,
+		Items: store.Items, Chunks: store.Chunks, Entities: store.Entities,
+		Aliases: store.Aliases, Claims: store.Claims, Rels: store.Relationships,
+		Jobs: store.Jobs, Runs: store.Runs, Models: store.ModelVersions,
+		Users: store.Users, Vectors: vectors,
+		Ingest: ingestSvc, Pipeline: pipeline, Search: searchSvc, Answer: answerSvc,
+		Reviewer: reviewer, Analysis: analysisSvc, Audit: recorder,
+		Embedder: embedder, EmbedHealth: embedder, AnalyseHealth: analyser,
+	})
+
+	apiHandlers := api.NewHandlers(api.Deps{
+		Cfg:   cfg,
+		Items: store.Items, Chunks: store.Chunks, Entities: store.Entities,
+		Aliases: store.Aliases, Claims: store.Claims, Rels: store.Relationships,
+		Jobs: store.Jobs, Runs: store.Runs, Models: store.ModelVersions, Vectors: vectors,
+		Ingest: ingestSvc, Pipeline: pipeline, Search: searchSvc, Answer: answerSvc,
+		Reviewer: reviewer, Analysis: analysisSvc, Audit: recorder, Embedder: embedder,
+		Tokens: tokenSvc,
+	})
+
+	health := &api.HealthChecker{
+		Mongo: store.Ping, Vectors: vectors,
+		Embedder: embedder, Analyser: analyser,
+		Version: version, Started: time.Now(),
+	}
+
+	router := api.NewRouter(api.RouterConfig{
+		Cfg: cfg, Auth: authSvc, Tokens: tokenSvc, API: apiHandlers, Web: webHandlers,
+		Health: health, Metrics: metrics,
+		Static: http.FileServerFS(web.StaticFS()),
+	})
+
+	srv := &http.Server{
+		Addr:              cfg.HTTP.Addr,
+		Handler:           withLogger(router, logger),
+		ReadHeaderTimeout: 15 * time.Second,
+		ReadTimeout:       cfg.HTTP.ReadTimeout,
+		WriteTimeout:      cfg.HTTP.WriteTimeout,
+		IdleTimeout:       cfg.HTTP.IdleTimeout,
+	}
+
+	// --- Start --------------------------------------------------------------
+	workerCtx, stopWorkers := context.WithCancel(ctx)
+	defer stopWorkers()
+	pool.Start(workerCtx)
+
+	// Sweep expired sessions periodically, so revoked and stale sessions do not
+	// accumulate even if the TTL index is unavailable.
+	go sweepSessions(workerCtx, authSvc, logger)
+
+	serverErr := make(chan error, 1)
+	go func() {
+		logger.Info("http server listening",
+			"addr", cfg.HTTP.Addr, "base_url", cfg.HTTP.BaseURL,
+			"external_bind_allowed", cfg.HTTP.AllowExternalBind)
+		if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) {
+			serverErr <- err
+		}
+	}()
+
+	select {
+	case err := <-serverErr:
+		return fmt.Errorf("http server: %w", err)
+	case <-ctx.Done():
+		logger.Info("shutdown signal received; draining")
+	}
+
+	// --- Shut down ----------------------------------------------------------
+	//
+	// The HTTP server drains first so no new work is accepted, then the workers
+	// are given time to finish what they hold. A job that does not finish keeps
+	// its lease, which expires and returns it to the queue.
+	shutdownCtx, cancel := context.WithTimeout(context.Background(), cfg.HTTP.ShutdownTimeout)
+	defer cancel()
+
+	if err := srv.Shutdown(shutdownCtx); err != nil {
+		logger.Warn("http server did not shut down cleanly", "error", err.Error())
+	}
+	stopWorkers()
+	pool.Stop(cfg.HTTP.ShutdownTimeout)
+
+	logger.Info("relata stopped")
+	return nil
+}
+
+// sweepSessions periodically deletes expired sessions.
+func sweepSessions(ctx context.Context, a *auth.Service, logger loggerType) {
+	t := time.NewTicker(1 * time.Hour)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			if n, err := a.SweepExpired(ctx); err != nil {
+				logger.Debug("session sweep failed", "error", err.Error())
+			} else if n > 0 {
+				logger.Info("removed expired sessions", "count", n)
+			}
+		}
+	}
+}
+
+// withLogger attaches the application logger to every request context.
+func withLogger(next http.Handler, logger loggerType) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		next.ServeHTTP(w, r.WithContext(observability.WithLogger(r.Context(), logger)))
+	})
+}
+
+// ensureWorkspace returns the default workspace, creating it if absent.
+//
+// Relata is single-workspace in practice today, but every record carries a
+// workspace ID and every query filters on it, so adding more later is a matter
+// of creating them rather than migrating the schema.
+func ensureWorkspace(ctx context.Context, store *mongo.Store) (*domain.Workspace, error) {
+	const defaultSlug = "default"
+
+	if ws, err := store.Workspaces.GetBySlug(ctx, defaultSlug); err == nil {
+		return ws, nil
+	} else if !errors.Is(err, domain.ErrNotFound) {
+		return nil, err
+	}
+
+	ws := &domain.Workspace{
+		ID:        newID(),
+		Name:      "Default workspace",
+		Slug:      defaultSlug,
+		CreatedAt: time.Now().UTC(),
+	}
+	if err := store.Workspaces.Create(ctx, ws); err != nil {
+		// A concurrent process may have created it first.
+		if existing, gerr := store.Workspaces.GetBySlug(ctx, defaultSlug); gerr == nil {
+			return existing, nil
+		}
+		return nil, err
+	}
+	return ws, nil
+}
@@ -0,0 +1,496 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/andydixon/relata/internal/audit"
+	"github.com/andydixon/relata/internal/config"
+	"github.com/andydixon/relata/internal/domain"
+	"github.com/andydixon/relata/internal/ingest"
+	"github.com/andydixon/relata/internal/observability"
+	"github.com/andydixon/relata/internal/storage/mongo"
+)
+
+// seedDoc is one document in the demonstration dataset.
+type seedDoc struct {
+	Title      string
+	Body       string
+	SourceType domain.SourceType
+	MIME       string
+	Filename   string
+	Metadata   map[string]string
+	DaysAgo    int
+	// Role explains what this document demonstrates, printed after seeding.
+	Role string
+}
+
+// demoDate is the reference point the dataset is built around.
+func demoDate(daysAgo int) time.Time {
+	return time.Now().UTC().AddDate(0, 0, -daysAgo).Truncate(24 * time.Hour)
+}
+
+// seedDocuments is a small, varied corpus that exercises every part of the
+// relationship model — including the cases where similarity must NOT be
+// mistaken for a factual connection.
+func seedDocuments() []seedDoc {
+	return []seedDoc{
+		// --- Two clearly related documents ---------------------------------
+		{
+			Title:      "Project Atlas — delivery schedule (original)",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/markdown",
+			DaysAgo:    30,
+			Role:       "Clearly related to the revision below, and superseded by it.",
+			Metadata:   map[string]string{"project": "atlas", "department": "operations"},
+			Body: `# Project Atlas — delivery schedule
+
+Prepared by Dr Alice Chen for Example Ltd on 2025-02-10.
+
+## Summary
+
+The initial delivery date for the Atlas units is 3 March 2025.
+Example Ltd will ship 240 units to the Northfield depot.
+
+## Contact
+
+Operational questions should go to ops@example.com.
+The total project cost is £42,000 excluding VAT.
+`,
+		},
+		{
+			Title:      "Project Atlas — revised delivery schedule",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/markdown",
+			DaysAgo:    12,
+			Role:       "Updates and supersedes the original schedule; shares entities and subject.",
+			Metadata:   map[string]string{"project": "atlas", "department": "operations"},
+			Body: `# Project Atlas — revised delivery schedule
+
+Prepared by Dr Alice Chen for Example Ltd on 2025-02-28.
+This revision replaces the schedule issued on 2025-02-10.
+
+## Summary
+
+The revised delivery date is 14 March 2025.
+Example Ltd will ship 240 units to the Northfield depot as before.
+
+The delay follows a shortage of housings from the subcontractor.
+
+## Contact
+
+Operational questions should go to ops@example.com.
+`,
+		},
+
+		// --- One contradiction ---------------------------------------------
+		{
+			Title:      "Atlas invoice reconciliation",
+			SourceType: domain.SourceTypeNote,
+			MIME:       "text/plain",
+			DaysAgo:    9,
+			Role:       "Contradicts the original schedule about the project cost.",
+			Metadata:   map[string]string{"project": "atlas", "department": "finance"},
+			Body: `Finance note, prepared 2025-03-01.
+
+Reconciling the Atlas paperwork against what Example Ltd has actually invoiced.
+
+The total project cost is £51,500 excluding VAT. This does not match the figure in the
+delivery schedule and needs resolving before the next payment run.
+
+Raised with Dr Alice Chen; awaiting a response.
+`,
+		},
+
+		// --- One update that supersedes older information -------------------
+		{
+			Title:      "Returns policy v3 (current)",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/markdown",
+			DaysAgo:    5,
+			Role:       "Supersedes the older returns policy below.",
+			Metadata:   map[string]string{"policy": "returns", "version": "3"},
+			Body: `# Returns policy, version 3
+
+Effective 2025-03-05. This policy replaces version 2, issued 2024-11-01.
+
+## Terms
+
+The returns window is 60 days from delivery.
+Refunds are issued to the original payment method within 14 days.
+
+Damaged goods must be reported to Example Ltd within 48 hours of delivery.
+`,
+		},
+		{
+			Title:      "Returns policy v2 (withdrawn)",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/markdown",
+			DaysAgo:    120,
+			Role:       "The older policy, superseded by version 3.",
+			Metadata:   map[string]string{"policy": "returns", "version": "2"},
+			Body: `# Returns policy, version 2
+
+Effective 2024-11-01.
+
+## Terms
+
+The returns window is 30 days from delivery.
+Refunds are issued to the original payment method within 21 days.
+
+Damaged goods must be reported to Example Ltd within 24 hours of delivery.
+`,
+		},
+
+		// --- Two superficially similar but meaningfully different docs -------
+		{
+			Title:      "Northfield depot fire drill report",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/plain",
+			DaysAgo:    20,
+			Role:       "Superficially similar to the safety inspection: same place, same vocabulary, different event.",
+			Metadata:   map[string]string{"site": "northfield"},
+			Body: `Fire drill report — Northfield depot
+
+Date of exercise: 2025-02-18. Conducted by the site safety officer.
+
+The depot was evacuated in 4 minutes. All 38 staff were accounted for at the assembly point.
+Two fire doors on the east side were found to be obstructed by pallets and were cleared.
+
+This was a scheduled drill. No actual incident occurred.
+`,
+		},
+		{
+			Title:      "Northfield depot safety inspection",
+			SourceType: domain.SourceTypeDocument,
+			MIME:       "text/plain",
+			DaysAgo:    18,
+			Role:       "Similar wording to the fire drill report, but a different event entirely.",
+			Metadata:   map[string]string{"site": "northfield"},
+			Body: `Safety inspection — Northfield depot
+
+Date of inspection: 2025-02-20. Conducted by an external assessor.
+
+The depot was inspected against the site safety schedule. Racking, lighting and fire doors
+were examined. Two fire doors on the east side were noted as previously obstructed and
+now clear.
+
+This was a routine annual inspection, not a drill and not an incident.
+`,
+		},
+
+		// --- One document sharing an entity but not a subject ---------------
+		{
+			Title:      "Example Ltd sponsorship announcement",
+			SourceType: domain.SourceTypeNote,
+			MIME:       "text/plain",
+			DaysAgo:    15,
+			Role:       "Shares the entity Example Ltd with the Atlas documents, but is about something else entirely.",
+			Metadata:   map[string]string{"department": "marketing"},
+			Body: `Announcement, 2025-02-22.
+
+Example Ltd has agreed to sponsor the regional youth athletics championship for the
+2025 season. The sponsorship covers kit and travel for four clubs.
+
+Dr Alice Chen will present the awards at the closing ceremony.
+
+This has no bearing on any current delivery or procurement work.
+`,
+		},
+
+		// --- One likely duplicate -------------------------------------------
+		{
+			Title:      "Atlas revised delivery — forwarded copy",
+			SourceType: domain.SourceTypeEmail,
+			MIME:       "text/plain",
+			DaysAgo:    11,
+			Role:       "A near-duplicate of the revised schedule. Relata flags it; it never merges it.",
+			Metadata:   map[string]string{"project": "atlas"},
+			Body: `From: alice.chen@example.com
+To: depot@example.com
+Subject: FW: Project Atlas revised delivery schedule
+Date: Sat, 1 Mar 2025 08:15:00 +0000
+
+Forwarding for the depot's records.
+
+The revised delivery date is 14 March 2025.
+Example Ltd will ship 240 units to the Northfield depot as before.
+The delay follows a shortage of housings from the subcontractor.
+
+Operational questions should go to ops@example.com.
+`,
+		},
+
+		// --- One unrelated document ------------------------------------------
+		{
+			Title:      "Office plant care rota",
+			SourceType: domain.SourceTypeNote,
+			MIME:       "text/plain",
+			DaysAgo:    25,
+			Role:       "Unrelated to everything else. Should not acquire relationships.",
+			Body: `Plant care rota, agreed 2025-02-14.
+
+Watering is on Mondays and Thursdays. The ficus by the window needs less water in winter
+than the ferns. Feed everything monthly between March and September.
+
+Whoever is last out on Friday should close the blinds on the south side.
+`,
+		},
+
+		// --- A JSON document ------------------------------------------------
+		{
+			Title:      "Atlas shipment record (JSON)",
+			SourceType: domain.SourceTypeJSON,
+			MIME:       "application/json",
+			Filename:   "atlas-shipment.json",
+			DaysAgo:    10,
+			Role:       "Structured data sharing entities and figures with the Atlas documents.",
+			Metadata:   map[string]string{"project": "atlas"},
+			Body: `{
+  "shipment": {
+    "project": "Atlas",
+    "supplier": "Example Ltd",
+    "destination": "Northfield depot",
+    "units": 240,
+    "scheduled_date": "2025-03-14",
+    "contact": "ops@example.com"
+  },
+  "notes": [
+    "Revised from the original date of 2025-03-03.",
+    "Housing shortage reported by the subcontractor."
+  ]
+}`,
+		},
+	}
+}
+
+// demoSearches are the queries the dataset is built to answer.
+var demoSearches = []struct{ Query, Shows string }{
+	{"What information relates to the revised delivery date?",
+		"The revised schedule, the original it replaces, the forwarded copy and the JSON shipment record."},
+	{"Which sources disagree about the project cost?",
+		"The original schedule (£42,000) and the finance note (£51,500), linked by a contradicts relationship."},
+	{"Show information associated with Example Ltd.",
+		"Every item naming the company, including the sponsorship note, which shares the entity but not the subject."},
+	{"What replaced the original policy?",
+		"Returns policy v3, related to the withdrawn v2."},
+	{"Find items similar in meaning but not actually about the same event.",
+		"The fire drill report and the safety inspection: similar wording, different events."},
+}
+
+// cmdSeed loads the demonstration dataset.
+func cmdSeed(args []string) error {
+	fs := flag.NewFlagSet("seed", flag.ExitOnError)
+	configFile := fs.String("config", ".env", "path to a key=value configuration file (optional)")
+	username := fs.String("username", "", "username to attribute the seeded items to (defaults to the first admin)")
+	wait := fs.Bool("wait", false, "wait for processing to finish before returning")
+	timeout := fs.Duration("timeout", 10*time.Minute, "how long to wait when --wait is set")
+	fs.Usage = func() {
+		fmt.Fprint(os.Stderr, `Load Relata's demonstration dataset.
+
+The dataset contains two clearly related documents, two superficially similar but
+different ones, a contradiction, an update that supersedes older information, a
+document sharing an entity but not a subject, a likely duplicate, and an unrelated
+document — so that the difference between similarity and evidence is visible.
+
+Usage:
+  relata seed [--wait]
+
+Flags:
+`)
+		fs.PrintDefaults()
+	}
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+
+	cfg, err := config.Load(*configFile)
+	if err != nil {
+		return err
+	}
+	logger := observability.NewLogger(cfg.LogLevel, cfg.LogJSON)
+
+	ctx, cancel := context.WithTimeout(context.Background(), *timeout+time.Minute)
+	defer cancel()
+	ctx = observability.WithLogger(ctx, logger)
+
+	store, err := mongo.Connect(ctx, cfg.Mongo, nil)
+	if err != nil {
+		return fmt.Errorf("mongodb: %w", err)
+	}
+	defer func() { _ = store.Close(context.Background()) }()
+
+	if err := store.EnsureIndexes(ctx); err != nil {
+		return fmt.Errorf("create mongodb indexes: %w", err)
+	}
+
+	// Seeding is attributed to a real account, so the audit trail records who
+	// the items belong to rather than inventing a system actor.
+	user, err := seedUser(ctx, store, *username)
+	if err != nil {
+		return err
+	}
+	principal := &domain.Principal{
+		UserID: user.ID, Username: user.Username, Role: user.Role, WorkspaceID: user.WorkspaceID,
+	}
+
+	recorder := audit.NewRecorder(store.Audit)
+	ingestSvc := ingest.NewService(store.Items, store.Jobs, cfg, recorder, nil)
+
+	docs := seedDocuments()
+	fmt.Printf("Loading %d demonstration documents into workspace %s…\n\n", len(docs), user.WorkspaceID)
+
+	var created []string
+	for _, d := range docs {
+		created2, err := seedOne(ctx, ingestSvc, principal, d)
+		if err != nil {
+			return err
+		}
+		created = append(created, created2)
+	}
+
+	fmt.Printf("\n%d documents queued for processing.\n", len(created))
+
+	if *wait {
+		fmt.Println("\nWaiting for background processing to finish…")
+		if err := waitForProcessing(ctx, store, principal.WorkspaceID, len(created), *timeout); err != nil {
+			return err
+		}
+		fmt.Println("Processing complete.")
+	} else {
+		fmt.Println("\nProcessing continues in the background. Watch it at /jobs.")
+	}
+
+	fmt.Print("\nWhat this dataset demonstrates:\n\n")
+	for _, d := range docs {
+		if d.Role != "" {
+			fmt.Printf("  • %-46s %s\n", trimTo(d.Title, 46), d.Role)
+		}
+	}
+
+	fmt.Print("\nDemonstration searches:\n\n")
+	for _, s := range demoSearches {
+		fmt.Printf("  %s\n      → %s\n\n", s.Query, s.Shows)
+	}
+
+	fmt.Printf("Open %s/search to try them.\n", cfg.HTTP.BaseURL)
+	fmt.Println("\nNote: relationship strengths shown in the interface are heuristic scores")
+	fmt.Println("computed from explicit signals. They are not calibrated probabilities, and an")
+	fmt.Println("inferred relationship is a proposal until a person confirms it.")
+	return nil
+}
+
+func seedOne(ctx context.Context, svc *ingest.Service, p *domain.Principal, d seedDoc) (string, error) {
+	created := demoDate(d.DaysAgo)
+	req := ingest.Request{
+		Title:        d.Title,
+		Content:      []byte(d.Body),
+		DeclaredMIME: d.MIME,
+		Filename:     d.Filename,
+		SourceType:   d.SourceType,
+		Metadata:     d.Metadata,
+		CreatedAt:    &created,
+		// The dataset deliberately contains a near-duplicate. It is a distinct
+		// document, not byte-identical, so it is accepted normally — which is
+		// the point: Relata flags near-duplicates rather than refusing them.
+		AllowDuplicate: false,
+	}
+
+	res, err := svc.Ingest(ctx, p, req)
+	if err != nil {
+		var dup *ingest.DuplicateError
+		if errors.As(err, &dup) {
+			fmt.Printf("  – %-50s already present (identical content)\n", trimTo(d.Title, 50))
+			return dup.ExistingID, nil
+		}
+		return "", fmt.Errorf("seed %q: %w", d.Title, err)
+	}
+	fmt.Printf("  ✓ %-50s %s\n", trimTo(d.Title, 50), res.Item.ID)
+	return res.Item.ID, nil
+}
+
+// seedUser finds the account to attribute seeded items to.
+func seedUser(ctx context.Context, store *mongo.Store, username string) (*domain.User, error) {
+	if username != "" {
+		u, err := store.Users.GetByUsername(ctx, username)
+		if err != nil {
+			return nil, fmt.Errorf("no such user %q — create one with: relata createadmin --username %s", username, username)
+		}
+		return u, nil
+	}
+
+	n, err := store.Users.Count(ctx)
+	if err != nil {
+		return nil, err
+	}
+	if n == 0 {
+		return nil, errors.New("no accounts exist yet — create one first with: relata createadmin --username admin")
+	}
+
+	// Fall back to the default workspace's first administrator.
+	ws, err := store.Workspaces.GetBySlug(ctx, "default")
+	if err != nil {
+		return nil, errors.New("no default workspace — create an administrator first with: relata createadmin --username admin")
+	}
+	for _, candidate := range []string{"admin", "administrator", "relata"} {
+		if u, err := store.Users.GetByUsername(ctx, candidate); err == nil && u.WorkspaceID == ws.ID {
+			return u, nil
+		}
+	}
+	return nil, errors.New("could not determine which account to use — pass --username")
+}
+
+// waitForProcessing blocks until the queue drains or the timeout elapses.
+func waitForProcessing(ctx context.Context, store *mongo.Store, workspaceID string, expected int, timeout time.Duration) error {
+	deadline := time.Now().Add(timeout)
+	ticker := time.NewTicker(2 * time.Second)
+	defer ticker.Stop()
+
+	lastReport := time.Now()
+	for {
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-ticker.C:
+		}
+
+		counts, err := store.Jobs.CountByStatus(ctx, workspaceID)
+		if err != nil {
+			return err
+		}
+		pending := counts[domain.JobPending] + counts[domain.JobRunning]
+
+		if pending == 0 {
+			ready, _ := store.Items.CountByStatus(ctx, workspaceID)
+			done := ready[domain.StatusReady] + ready[domain.StatusPartial]
+			if done >= int64(expected) || counts[domain.JobFailed] > 0 {
+				if counts[domain.JobFailed] > 0 {
+					fmt.Printf("  %d job(s) failed — see /jobs for details.\n", counts[domain.JobFailed])
+				}
+				return nil
+			}
+			// No queued work and not everything is ready: the workers are
+			// probably not running.
+			return errors.New("no jobs are queued but items are unprocessed — is `relata serve` running?")
+		}
+
+		if time.Since(lastReport) > 10*time.Second {
+			fmt.Printf("  %d job(s) still running…\n", pending)
+			lastReport = time.Now()
+		}
+		if time.Now().After(deadline) {
+			return fmt.Errorf("timed out after %s with %d job(s) outstanding", timeout, pending)
+		}
+	}
+}
+
+func trimTo(s string, n int) string {
+	if len(s) <= n {
+		return s
+	}
+	return s[:n-1] + "…"
+}