Code sync, security fixes
This commit is contained in:
Andy Dixon
@@ -0,0 +1,296 @@
|
||||
# AI provider interface
|
||||
|
||||
Relata's AI providers sit behind two interfaces declared in `internal/domain`. Ollama is the
|
||||
default; nothing in the pipeline knows that.
|
||||
|
||||
---
|
||||
|
||||
## The two ports
|
||||
|
||||
```go
|
||||
// internal/domain/ports.go
|
||||
|
||||
type Embedder interface {
|
||||
// Embed returns one vector per input text, in the same order.
|
||||
// Implementations must honour ctx cancellation and must not retry indefinitely.
|
||||
Embed(ctx context.Context, texts []string) ([][]float32, error)
|
||||
ModelID() string // stamped onto every derived record
|
||||
Dimensions() int // vector width
|
||||
}
|
||||
|
||||
type Analyser interface {
|
||||
Complete(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
|
||||
ModelID() string
|
||||
}
|
||||
```
|
||||
|
||||
`CompletionRequest` carries messages, an optional JSON schema, a token ceiling and a
|
||||
temperature. `Complete` returns the **raw content** — it does not parse it. Parsing belongs to
|
||||
`internal/analysis`, so a validation failure is attributed to the task rather than the
|
||||
transport.
|
||||
|
||||
Optionally, a provider may implement `HealthChecker` to appear on the settings page:
|
||||
|
||||
```go
|
||||
type HealthChecker interface {
|
||||
Health(ctx context.Context) ProviderHealth
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Contract
|
||||
|
||||
Anything implementing these interfaces must:
|
||||
|
||||
| Requirement | Why |
|
||||
|---|---|
|
||||
| Honour `ctx` cancellation | A worker that loses its lease cancels the context; a provider that ignores it wastes the slot |
|
||||
| Bound retries | A permanently failing provider must surface as a job failure, not an endless loop |
|
||||
| Retry only transient errors | A 400 or an unknown model will not fix itself |
|
||||
| Return `Permanent(err)` for unrecoverable failures | The worker uses this to decide not to retry |
|
||||
| Return vectors in input order | The caller pairs them with chunks positionally |
|
||||
| Return exactly one vector per input | Anything else is a permanent error |
|
||||
| Keep vector width stable | A change corrupts the Qdrant collection |
|
||||
| Bound response size | A misbehaving provider must not exhaust memory |
|
||||
| Never log document content | Content is the user's data |
|
||||
| Apply a timeout | Every outbound call is bounded |
|
||||
| Refuse redirects | A compromised provider must not bounce requests elsewhere |
|
||||
|
||||
The helpers in `internal/embeddings/retry.go` — `Do`, `Permanent`, `Retryable`, `StatusError`
|
||||
— are provider-agnostic and are the expected way to satisfy the retry requirements.
|
||||
|
||||
---
|
||||
|
||||
## Adding an OpenAI-compatible provider
|
||||
|
||||
Two files. Nothing else changes.
|
||||
|
||||
### 1. The embedder
|
||||
|
||||
```go
|
||||
package embeddings
|
||||
|
||||
type OpenAICompatible struct {
|
||||
base, model, apiKey string
|
||||
dims int
|
||||
client *http.Client
|
||||
retry RetryPolicy
|
||||
}
|
||||
|
||||
var _ domain.Embedder = (*OpenAICompatible)(nil)
|
||||
|
||||
func (o *OpenAICompatible) ModelID() string { return o.model }
|
||||
func (o *OpenAICompatible) Dimensions() int { return o.dims }
|
||||
|
||||
func (o *OpenAICompatible) Embed(ctx context.Context, texts []string) ([][]float32, error) {
|
||||
var out [][]float32
|
||||
|
||||
for start := 0; start < len(texts); start += o.batch {
|
||||
end := min(start+o.batch, len(texts))
|
||||
body, _ := json.Marshal(map[string]any{
|
||||
"model": o.model,
|
||||
"input": texts[start:end],
|
||||
})
|
||||
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
Index int `json:"index"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"data"`
|
||||
}
|
||||
|
||||
err := Do(ctx, o.retry, func(ctx context.Context, attempt int) error {
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost,
|
||||
o.base+"/v1/embeddings", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return Permanent(err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Authorization", "Bearer "+o.apiKey)
|
||||
|
||||
httpResp, err := o.client.Do(req)
|
||||
if err != nil {
|
||||
return err // transport failures are retried
|
||||
}
|
||||
defer httpResp.Body.Close()
|
||||
|
||||
if httpResp.StatusCode != http.StatusOK {
|
||||
msg, _ := io.ReadAll(io.LimitReader(httpResp.Body, 2048))
|
||||
se := &StatusError{Status: httpResp.StatusCode, Body: string(msg), Op: "embeddings"}
|
||||
if httpResp.StatusCode == 400 || httpResp.StatusCode == 404 {
|
||||
return Permanent(se) // never retried
|
||||
}
|
||||
return se // Retryable() decides from the status
|
||||
}
|
||||
return json.NewDecoder(io.LimitReader(httpResp.Body, 64<<20)).Decode(&resp)
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// The API may return results out of order.
|
||||
sort.Slice(resp.Data, func(i, j int) bool { return resp.Data[i].Index < resp.Data[j].Index })
|
||||
for _, d := range resp.Data {
|
||||
out = append(out, d.Embedding)
|
||||
}
|
||||
}
|
||||
|
||||
if len(out) != len(texts) {
|
||||
return nil, Permanent(fmt.Errorf("provider returned %d vectors for %d inputs", len(out), len(texts)))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
```
|
||||
|
||||
### 2. The analyser
|
||||
|
||||
```go
|
||||
package analysis
|
||||
|
||||
type OpenAICompatibleAnalyser struct { /* … */ }
|
||||
|
||||
var _ domain.Analyser = (*OpenAICompatibleAnalyser)(nil)
|
||||
|
||||
func (a *OpenAICompatibleAnalyser) Complete(ctx context.Context,
|
||||
req domain.CompletionRequest) (*domain.CompletionResponse, error) {
|
||||
|
||||
payload := map[string]any{
|
||||
"model": a.model,
|
||||
"messages": toOpenAIMessages(req.Messages),
|
||||
"max_tokens": req.MaxTokens,
|
||||
"temperature": req.Temperature,
|
||||
}
|
||||
// Structured output, where the provider supports it.
|
||||
if len(req.JSONSchema) > 0 {
|
||||
payload["response_format"] = map[string]any{
|
||||
"type": "json_schema",
|
||||
"json_schema": map[string]any{
|
||||
"name": "relata_task",
|
||||
"strict": true,
|
||||
"schema": json.RawMessage(req.JSONSchema),
|
||||
},
|
||||
}
|
||||
}
|
||||
// … POST /v1/chat/completions, return the raw content unparsed.
|
||||
}
|
||||
```
|
||||
|
||||
### 3. Wire it in
|
||||
|
||||
In `cmd/relata/main.go`, choose the implementation from configuration:
|
||||
|
||||
```go
|
||||
var embedder domain.Embedder
|
||||
switch cfg.Ollama.Provider { // e.g. RELATA_AI_PROVIDER
|
||||
case "openai":
|
||||
embedder = embeddings.NewOpenAICompatible(...)
|
||||
default:
|
||||
embedder = embeddings.NewOllama(...)
|
||||
}
|
||||
```
|
||||
|
||||
Every service takes `domain.Embedder` and `domain.Analyser`, so nothing downstream changes.
|
||||
|
||||
---
|
||||
|
||||
## Structured output without native schema support
|
||||
|
||||
`analysis.DecodeStrict` does not require the provider to enforce the schema. It:
|
||||
|
||||
- rejects unknown fields and trailing content;
|
||||
- bounds the payload before parsing;
|
||||
- unwraps a complete markdown fence (a structural operation on a delimiter — if what remains
|
||||
is not valid JSON it is still rejected);
|
||||
- never uses a regular expression to find or repair JSON.
|
||||
|
||||
A provider without structured-output support therefore still works: the schema is included in
|
||||
the prompt as guidance, and validation catches what the model gets wrong. Expect a higher
|
||||
rejection rate, visible in `relata_model_invalid_output_total`.
|
||||
|
||||
---
|
||||
|
||||
## Task-level validation
|
||||
|
||||
Above the transport, each task validates independently of the provider:
|
||||
|
||||
| Task | Checks |
|
||||
|---|---|
|
||||
| `extract_entities` | Entity type enum per row; a bad row is skipped, not fatal. An evidence quotation absent from the source is dropped. |
|
||||
| `extract_claims` | Nature enum; **quotation must be locatable in the source** (whitespace reflow tolerated), or the claim is discarded. |
|
||||
| `classify_relationship` | Relationship type enum; confidence in 0–1. |
|
||||
| `generate_answer` | Citations resolved against supplied refs; fabricated references dropped and counted. |
|
||||
|
||||
### Substance, not only shape
|
||||
|
||||
A JSON schema constrains a response's *shape*, not its *substance*. A small model asked for
|
||||
`{"answer": ..., "sufficient": ...}` will sometimes return the schema back:
|
||||
|
||||
```json
|
||||
{"answer": "sufficient", "sufficient": true, "citations": []}
|
||||
```
|
||||
|
||||
That is schema-valid and completely empty. `llama3.2:3b` produced exactly this, and an earlier
|
||||
build displayed the word "sufficient" to the user as an answer.
|
||||
|
||||
Free-text fields are therefore also checked for degeneracy (`analysis.Degenerate`), which
|
||||
rejects a value that is a bare field name or enum member from the schema itself, a placeholder
|
||||
(`"string"`, `"n/a"`, `"TODO"`, `"..."`), or too short to carry meaning:
|
||||
|
||||
| Field | Minimum | Rejected as |
|
||||
|---|---:|---|
|
||||
| `generate_answer` → answer | 15 chars | `degenerate_content` |
|
||||
| `extract_entities` → name | 2 chars | `degenerate_content` |
|
||||
| `extract_claims` → statement | 10 chars | `degenerate_content` |
|
||||
| proposed alias → name | 2 chars | `degenerate_content` |
|
||||
|
||||
A rejected answer is reported as unavailable rather than shown, and the request still returns
|
||||
its retrieved sources and any contradictions Relata recorded between them — those are
|
||||
deterministic and do not depend on the model. Only a transport failure fails the request.
|
||||
|
||||
The same word list applies to every field, which costs a little precision on names: a person
|
||||
genuinely surnamed "Na", or an organisation called exactly "Example", would be rejected as a
|
||||
placeholder. The cost is bounded — a rejected row is skipped, the rest of the extraction
|
||||
proceeds, and deterministic extraction is unaffected — and a model returning a bare
|
||||
placeholder word as an entity name is overwhelmingly more often noise than a real name.
|
||||
|
||||
This is a floor on obvious emptiness, not a quality judgement. It cannot tell a poor answer
|
||||
from a good one, only a non-answer from an answer.
|
||||
|
||||
Rejections are recorded on `relata_model_invalid_output_total{task,reason}` with a bounded
|
||||
reason set, and are **not retried** — a permanently invalid response will not become valid by
|
||||
repetition.
|
||||
|
||||
---
|
||||
|
||||
## Running without a language model
|
||||
|
||||
Set `RELATA_ANALYSIS_ENABLED=false`. `analysis.Service` reports itself disabled and every task
|
||||
returns `ErrDisabled`, which the pipeline treats as degraded quality rather than failure.
|
||||
|
||||
Still working: ingestion, parsing, chunking, embedding, deterministic entity extraction,
|
||||
deterministic claim extraction, relationship discovery and scoring, hybrid search, and the
|
||||
whole interface.
|
||||
|
||||
Not working: model-assisted entity and claim extraction, relationship type classification by
|
||||
model, and generated answers.
|
||||
|
||||
An embedder is still required for meaning-based search. Without one, search degrades to
|
||||
keyword, entity and relationship matching, and says so in the response notes.
|
||||
|
||||
---
|
||||
|
||||
## Model versions and stored data
|
||||
|
||||
Every derived record carries the model that produced it, and `model_versions` records what has
|
||||
been seen per workspace. The settings page shows this, and warns when the stored vector count
|
||||
does not match the chunk count.
|
||||
|
||||
**Changing the embedding model changes the vector geometry.** Relata detects a dimension
|
||||
mismatch and refuses to write mixed-width vectors — a permanent error naming the fix. After
|
||||
such a change: set `RELATA_EMBEDDING_DIMENSIONS`, then use **Settings → Re-embed workspace**,
|
||||
which processes in bounded batches and re-enqueues itself rather than holding one lease for
|
||||
hours.
|
||||
|
||||
**Changing the analysis model** does not invalidate anything; reprocess items to refresh
|
||||
model-derived entities, claims and relationship classifications.
|
||||
@@ -0,0 +1,304 @@
|
||||
# Architecture
|
||||
|
||||
Relata is a **modular monolith with in-process background workers**: one binary, one process,
|
||||
three external data stores. That choice is deliberate and is documented in
|
||||
[decisions/0001-modular-monolith.md](decisions/0001-modular-monolith.md).
|
||||
|
||||
---
|
||||
|
||||
## Package layout
|
||||
|
||||
```
|
||||
cmd/relata/ serve · createadmin · seed · version
|
||||
|
||||
internal/
|
||||
domain/ types + port interfaces. Standard library only.
|
||||
config/ environment configuration, validated at startup
|
||||
ids/ UUIDs, session tokens, deterministic dedupe keys
|
||||
observability/ structured logging, correlation IDs, metrics
|
||||
|
||||
parsing/ text · markdown · json · pdf · docx → Document
|
||||
chunking/ structure-aware chunking
|
||||
entities/ normalisation, deterministic extraction, extraction service
|
||||
analysis/ Analyser port, Ollama adapter, strict model-output validation
|
||||
embeddings/ Embedder port, Ollama adapter, bounded retry
|
||||
relationships/ candidate discovery, versioned scoring, human review
|
||||
search/ hybrid retrieval, answer generation
|
||||
ranking/ rank fusion and result explanation
|
||||
ingest/ ingestion service and pipeline stages
|
||||
jobs/ leased job queue worker pool
|
||||
|
||||
storage/mongo/ repositories implementing the domain ports
|
||||
storage/qdrant/ vector store implementing the domain port
|
||||
|
||||
auth/ Argon2id, sessions, CSRF, login rate limiting
|
||||
audit/ append-only audit recording
|
||||
api/ /api/v1 REST, middleware, router
|
||||
web/ server-rendered HTML, templates, static assets
|
||||
|
||||
tests/ integration and security tests (build tag: integration)
|
||||
migrations/ note on why there are none; indexes are applied at startup
|
||||
docs/ this documentation, plus openapi.yaml
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## The dependency rule
|
||||
|
||||
**`internal/domain` imports only the standard library.** No MongoDB driver, no HTTP types, no
|
||||
Qdrant client. It declares:
|
||||
|
||||
- the domain types (`Item`, `Chunk`, `Entity`, `Claim`, `Relationship`, `Job`, …);
|
||||
- the **port interfaces** the service layer depends on (`ItemRepository`, `VectorStore`,
|
||||
`Embedder`, `Analyser`, …).
|
||||
|
||||
Adapters implement those ports:
|
||||
|
||||
```
|
||||
┌───────────────┐
|
||||
│ domain │ types + ports, stdlib only
|
||||
└───────▲───────┘
|
||||
│ implements │ depends on
|
||||
┌──────────┴──────────┐ ┌───────────▼──────────┐
|
||||
│ storage/mongo │ │ ingest · search │
|
||||
│ storage/qdrant │ │ relationships │
|
||||
│ embeddings (Ollama) │ │ entities · analysis │
|
||||
│ analysis (Ollama) │ └───────────┬──────────┘
|
||||
└─────────────────────┘ │
|
||||
┌──────────▼──────────┐
|
||||
│ api · web │
|
||||
└─────────────────────┘
|
||||
```
|
||||
|
||||
Practical consequences:
|
||||
|
||||
- Swapping Ollama for an OpenAI-compatible API means adding one file in `embeddings/` and one
|
||||
in `analysis/`. Nothing else changes. See [AI_PROVIDER_INTERFACE.md](AI_PROVIDER_INTERFACE.md).
|
||||
- The scoring model can be unit-tested without a database — `scoring_test.go` runs in
|
||||
milliseconds and needs nothing running.
|
||||
- `api` and `web` depend on services, never on storage directly.
|
||||
|
||||
`api` and `web` have a one-way relationship: `web` imports `api` for the principal/session
|
||||
context helpers, and the router accepts the web handlers through the `api.WebRoutes`
|
||||
interface, so `api` never imports `web`.
|
||||
|
||||
---
|
||||
|
||||
## Data model
|
||||
|
||||
MongoDB is the **authoritative store**. Qdrant holds vectors and the minimum needed to
|
||||
resolve a hit back to MongoDB.
|
||||
|
||||
### Collections
|
||||
|
||||
| Collection | Contents | Key indexes |
|
||||
|---|---|---|
|
||||
| `workspaces` | The data boundary | `slug` (unique) |
|
||||
| `users` | Local accounts, Argon2id hashes | `username` (unique) |
|
||||
| `sessions` | Server-side sessions | `token_hash` (unique), TTL on `absolute_expiry` |
|
||||
| `items` | Source items and their originals | `(workspace, ingested_at)`, `(workspace, content_hash)`, text index |
|
||||
| `chunks` | Retrievable sections | `(workspace, item, sequence)`, `(workspace, embedded, model)`, text index |
|
||||
| `entities` | Extracted subjects | **`(workspace, normalised_key)` unique**, `(workspace, item_ids)` |
|
||||
| `entity_aliases` | Candidate name links | `(workspace, entity, normalised)` unique |
|
||||
| `claims` | Statements items make | `(workspace, subject)`, `(workspace, dedupe_key)` unique |
|
||||
| `relationships` | Scored, evidenced links | **`(workspace, dedupe_key)` unique**, `(workspace, source)`, `(workspace, review_status, score)` |
|
||||
| `jobs` | The persistent queue | `(status, run_after, priority)`, `(status, lease_expires)`, partial unique on `idempotency_key` |
|
||||
| `processing_runs` | Per-item processing history | `(workspace, item, started_at)` |
|
||||
| `model_versions` | Models that produced stored data | `(workspace, kind, name)` unique |
|
||||
| `audit_events` | Append-only action log | `(workspace, at)`, `(workspace, action, at)` |
|
||||
| `api_tokens` | Bearer credentials; hashes only, never secrets | `token_hash` unique, `(workspace, user, created_at)`, TTL on `expires_at` |
|
||||
|
||||
### Identifiers
|
||||
|
||||
Every record uses a **string UUID** `_id`. Chunk IDs are used verbatim as Qdrant point IDs,
|
||||
which Qdrant accepts in UUID form. That removes an entire class of ID-mapping bugs and makes
|
||||
"delete this item's vectors" a filter rather than a join.
|
||||
|
||||
Chunk IDs are **deterministic** — derived from `(workspace, item, sequence, content_hash)` —
|
||||
so re-chunking an unchanged item reuses the same point IDs and overwrites the vectors rather
|
||||
than orphaning them.
|
||||
|
||||
### The three unique indexes that carry the weight
|
||||
|
||||
These are not merely constraints; they are what make the pipeline idempotent under
|
||||
concurrency.
|
||||
|
||||
- **`entities (workspace, normalised_key)`** — one entity per normalised name and type.
|
||||
Concurrent workers extracting the same company from different documents converge on one
|
||||
record instead of racing to create two. Similar-but-different names never collide here;
|
||||
they produce a candidate alias for a person to adjudicate, and even a confirmed alias
|
||||
leaves both records in place.
|
||||
- **`relationships (workspace, dedupe_key)`** — the key is derived from the endpoint pair, so
|
||||
reprocessing updates an existing relationship rather than adding a parallel one. Crucially,
|
||||
the upsert **refuses to overwrite a human review decision**: if someone has confirmed,
|
||||
rejected or edited it, only the evidence is refreshed.
|
||||
- **`jobs (idempotency_key)` where `status = "pending"`** — a partial index, so at most one
|
||||
*pending* job may exist per key while completed jobs remain as history. Re-enqueuing after
|
||||
a crash is a no-op rather than duplicate work.
|
||||
|
||||
### Workspace enforcement
|
||||
|
||||
Every user-content record carries `workspace_id`, and every repository method takes one. The
|
||||
helper `wsFilter` is the only sanctioned way to begin a filter and **returns an error on an
|
||||
empty workspace**, so a missing principal cannot silently widen a query.
|
||||
|
||||
There is deliberately no `GetByID(id)` that skips the workspace. Cross-workspace access
|
||||
returns `not_found`, never `forbidden`, so probing cannot confirm that an ID exists elsewhere.
|
||||
|
||||
### Qdrant payloads
|
||||
|
||||
Only what retrieval needs:
|
||||
|
||||
```
|
||||
workspace_id · item_id · chunk_id · source_type · created_at · embedding_model
|
||||
```
|
||||
|
||||
No text, no titles, no relationship data. Keeping the payload thin is what stops Qdrant
|
||||
drifting into a second, unversioned copy of the corpus. Every payload field is indexed, so a
|
||||
workspace-filtered search stays a filtered search rather than degrading into a scan.
|
||||
|
||||
---
|
||||
|
||||
## Request flow
|
||||
|
||||
### A search
|
||||
|
||||
```
|
||||
GET /api/v1/search?q=…
|
||||
│
|
||||
├─ RequestID · Recover · Logging · SecurityHeaders · RequireAuth
|
||||
│
|
||||
├─ searchQueryFromRequest — typed values only; unknown enums rejected
|
||||
│
|
||||
└─ search.Service.Search
|
||||
├─ items.TextSearch (MongoDB $text, workspace-filtered)
|
||||
├─ chunks.TextSearch (MongoDB $text, for excerpts)
|
||||
├─ embedder.Embed(query) → vectors.Search (Qdrant, workspace-filtered)
|
||||
├─ entities.FindByNormalised (query n-grams → entity keys)
|
||||
├─ items.GetMany (re-checks workspace; drops anything not returned)
|
||||
├─ rels.ListForItems (expansion; rejected links excluded)
|
||||
└─ ranking.Rank (reciprocal-rank fusion + explanations)
|
||||
```
|
||||
|
||||
Note the second workspace check after `GetMany`: candidates whose items are not returned are
|
||||
dropped. Qdrant's filter is already workspace-scoped, but the authoritative check is the
|
||||
MongoDB read.
|
||||
|
||||
### An ingestion
|
||||
|
||||
```
|
||||
POST /api/v1/items
|
||||
│
|
||||
└─ ingest.Service.Ingest
|
||||
├─ admission semaphore (bounded concurrent ingestion)
|
||||
├─ parsing.Sniff — type from CONTENT, not the filename or header
|
||||
├─ validate title, metadata, size, creation date
|
||||
├─ SHA-256 → FindByContentHash → exact-duplicate check
|
||||
├─ store original bytes (binary) or inline text
|
||||
├─ items.Create
|
||||
└─ jobs.Enqueue(parse_item)
|
||||
│
|
||||
└─ parse → chunk → embed → entities → claims → relationships
|
||||
```
|
||||
|
||||
Each stage is a separate job that enqueues the next. See
|
||||
[INGESTION_PIPELINE.md](INGESTION_PIPELINE.md).
|
||||
|
||||
---
|
||||
|
||||
## Background workers
|
||||
|
||||
A bounded pool draining a Mongo-backed queue. The design goals are recoverability and
|
||||
boundedness.
|
||||
|
||||
**Leases, not locks.** A claimed job records its owner and an expiry. A worker renews the
|
||||
lease while it runs. A worker that dies stops renewing; the lease expires; `ReleaseExpired`
|
||||
returns the job to the pending pool. Nothing external is needed to detect the crash.
|
||||
|
||||
**Losing a lease cancels the work.** If a heartbeat finds the lease gone — another worker has
|
||||
taken over — the job's context is cancelled so the original worker abandons its attempt
|
||||
instead of writing a stale result.
|
||||
|
||||
**Two independent limits.** Worker count bounds concurrent jobs; a separate semaphore bounds
|
||||
concurrent AI calls. A small local model is not swamped just because the worker count was
|
||||
raised.
|
||||
|
||||
**Only transient failures retry.** Invalid model output, an unparseable document, a missing
|
||||
handler and a deleted item are permanent — repeating them wastes the queue. Transient
|
||||
failures retry with bounded exponential backoff and full jitter, within an attempt budget.
|
||||
|
||||
**A panicking handler does not take the worker down.** Handlers run guarded; a panic becomes
|
||||
an ordinary job failure.
|
||||
|
||||
**Shutdown drains.** The HTTP server stops accepting first, then workers are given until the
|
||||
shutdown deadline. A job that does not finish keeps its lease, which expires and returns it
|
||||
to the queue.
|
||||
|
||||
---
|
||||
|
||||
## Idempotency
|
||||
|
||||
Reprocessing must not multiply data. Each derived record type has its own mechanism:
|
||||
|
||||
| Records | Mechanism |
|
||||
|---|---|
|
||||
| Chunks | `ReplaceForItem` — delete then insert, so one generation exists |
|
||||
| Vectors | Deterministic chunk IDs; `DeleteByItem` before re-chunking |
|
||||
| Claims | `ReplaceForItem`, plus a unique `dedupe_key` |
|
||||
| Entities | `UpsertByKey` on `(workspace, normalised_key)`; observations accumulate |
|
||||
| Relationships | `UpsertByDedupeKey`; **human decisions are never overwritten** |
|
||||
| Jobs | Partial unique index on `idempotency_key` for pending jobs |
|
||||
| Items | Partial unique index on `(workspace_id, content_hash)` for live, non-deliberate items — duplicate detection is a read then a write, so the index is what actually enforces it ([ADR 6](decisions/0006-duplicate-detection-enforced-by-index.md)) |
|
||||
| Users | Partial unique index on `(oauth_provider, oauth_subject)` — first sign-in is a read then a write, and two concurrent ones must not provision two workspaces for one person ([ADR 8](decisions/0008-federated-identity-private-workspaces.md)) |
|
||||
| API tokens | Unique index on `token_hash`; authentication is a lookup on it and nothing else |
|
||||
|
||||
Processing runs are the exception: each reprocess **adds** a run, because the point is to
|
||||
retain what previous parser and model versions produced.
|
||||
|
||||
---
|
||||
|
||||
## Security architecture
|
||||
|
||||
Detailed in [THREAT_MODEL.md](THREAT_MODEL.md). The structural points:
|
||||
|
||||
- **Content-based type detection.** A browser's `Content-Type` and the filename are hints; the
|
||||
magic bytes decide. A zip renamed `.docx` is rejected unless it genuinely contains
|
||||
`word/document.xml`.
|
||||
- **Stored paths never derive from user input.** An uploaded file is stored under the item's
|
||||
own UUID with an extension from a fixed allow-list. The user's filename is retained for
|
||||
display only.
|
||||
- **Escaping by construction.** All HTML goes through `html/template`'s contextual escaping,
|
||||
and the `web` package deliberately contains **no** helper that turns a stored string into
|
||||
`template.HTML`.
|
||||
- **A CSP that makes XSS inert.** `default-src 'none'`, `script-src 'self'`, no
|
||||
`unsafe-inline`, no remote origins. Relata ships its own CSS and JavaScript as files, so
|
||||
nothing inline is needed.
|
||||
- **Typed queries only.** Filters are assembled in the storage layer from Go values. A client
|
||||
string becomes an *operand*, never a key or an operator.
|
||||
- **Model output is untrusted input.** Strict schema decoding with unknown fields rejected,
|
||||
enum validation, reference checking against what was actually supplied, and no regex-based
|
||||
repair of malformed JSON.
|
||||
- **Uploaded content is data, never instruction.** Retrieved text reaches a model only inside
|
||||
delimited blocks whose terminator the content cannot forge.
|
||||
|
||||
---
|
||||
|
||||
## Testing strategy
|
||||
|
||||
| Level | Location | Needs |
|
||||
|---|---|---|
|
||||
| Unit | `internal/*/[!i]*_test.go` | nothing |
|
||||
| Fuzz | parsers, model-output decoding, fences, hashes | nothing |
|
||||
| HTTP | `internal/api/http_test.go` | nothing |
|
||||
| Integration | `tests/` (tag `integration`) | Docker: MongoDB + Qdrant |
|
||||
| Security regression | across packages | nothing |
|
||||
|
||||
Unit tests assert **behaviour and design invariants**, not coverage. Several encode the
|
||||
central claims of the design and will fail if a change breaks them:
|
||||
|
||||
- similarity alone cannot produce a strong relationship score;
|
||||
- a maximally confident model opinion cannot reach the storage threshold;
|
||||
- observed contradiction outranks a model's opinion when inferring a type;
|
||||
- chronology alone never yields a causal or updating relationship;
|
||||
- a cross-workspace access is indistinguishable from a missing record;
|
||||
- a fabricated citation cannot resolve to a real item.
|
||||
@@ -0,0 +1,176 @@
|
||||
# Backup and restore
|
||||
|
||||
## What must be backed up
|
||||
|
||||
| Store | Contains | Recoverable without a backup? |
|
||||
|---|---|---|
|
||||
| **MongoDB** | Items, extracted text, chunks, entities, claims, relationships, review history, users, audit | **No.** This is the irreplaceable part. |
|
||||
| **Upload directory** | Original PDF and DOCX bytes | **No.** Only the originals a user supplied. |
|
||||
| **Qdrant** | Embedding vectors | **Yes** — rebuildable from MongoDB by re-embedding. |
|
||||
| `.env` | Configuration | Yes, but keep it: it records which models produced your data. |
|
||||
|
||||
MongoDB and the upload directory must be backed up **together and consistently**: an item row
|
||||
referring to a file that is not in the backup is a broken record.
|
||||
|
||||
Qdrant is derived data. Backing it up saves re-embedding time; losing it costs only the time
|
||||
to rebuild.
|
||||
|
||||
---
|
||||
|
||||
## Backing up
|
||||
|
||||
### Quick local snapshot
|
||||
|
||||
```bash
|
||||
./scripts/backup.sh # writes to backups/<timestamp>/
|
||||
./scripts/backup.sh /mnt/backups # or somewhere else
|
||||
```
|
||||
|
||||
The script captures MongoDB and the upload directory together, requests a Qdrant snapshot if
|
||||
one is available, copies the non-secret configuration for reference, and writes a `RESTORE.md`
|
||||
alongside the archives.
|
||||
|
||||
It honours `RELATA_UPLOAD_DIR`, `RELATA_MONGO_DATABASE`, `RELATA_QDRANT_URL` and
|
||||
`RELATA_QDRANT_COLLECTION`, so a non-default deployment needs no edits.
|
||||
|
||||
### Consistency
|
||||
|
||||
Relata does not use MongoDB transactions across collections, so a dump taken while the workers
|
||||
are running may catch an item mid-pipeline — for example chunks written but embeddings not yet
|
||||
stored. That is **recoverable but untidy**: on restore, reprocess any item whose status is not
|
||||
`ready` or `partial`.
|
||||
|
||||
For a clean backup, stop the application first. The workers shut down gracefully: in-flight
|
||||
jobs are drained, and any that do not finish keep their leases and return to the queue on
|
||||
restart.
|
||||
|
||||
For a hot backup, prefer a filesystem or volume snapshot that captures MongoDB's data
|
||||
directory and the upload directory at the same instant.
|
||||
|
||||
---
|
||||
|
||||
## Restoring
|
||||
|
||||
```bash
|
||||
# 1. Stop Relata.
|
||||
docker compose stop app
|
||||
|
||||
# 2. MongoDB
|
||||
docker compose exec -T mongodb mongorestore \
|
||||
--archive --gzip --drop < backups/TIMESTAMP/relata-mongo.archive.gz
|
||||
|
||||
# 3. Original files
|
||||
rm -rf data/uploads
|
||||
tar -xzf backups/TIMESTAMP/uploads.tar.gz -C data
|
||||
|
||||
# 4. Start Relata. Indexes are created automatically at startup.
|
||||
docker compose start app # or: make dev
|
||||
```
|
||||
|
||||
### After restoring
|
||||
|
||||
Check the counts on the settings page. If **vectors stored** does not match **chunks in
|
||||
MongoDB**, Qdrant is out of step. Rebuild:
|
||||
|
||||
**Settings → Re-embed workspace**, or:
|
||||
|
||||
```bash
|
||||
curl -s -b cj.txt -X POST http://127.0.0.1:8080/settings/reembed \
|
||||
-H "X-CSRF-Token: $CSRF"
|
||||
```
|
||||
|
||||
Re-embedding processes in bounded batches, re-enqueuing itself as it goes, so a large
|
||||
workspace does not hold one lease for hours. Progress is visible on the queue page.
|
||||
|
||||
### Restoring without Qdrant data
|
||||
|
||||
Entirely supported — vectors are derived.
|
||||
|
||||
```bash
|
||||
# Recreate the collection and rebuild every vector.
|
||||
curl -X DELETE http://127.0.0.1:6333/collections/relata_chunks
|
||||
# Relata recreates it at startup, then:
|
||||
# Settings → Re-embed workspace
|
||||
```
|
||||
|
||||
Between deletion and completion, meaning-based search returns nothing and search responses say
|
||||
so in their notes. Keyword, entity and relationship search continue to work.
|
||||
|
||||
---
|
||||
|
||||
## Verifying a restore
|
||||
|
||||
```bash
|
||||
# Everything reports healthy
|
||||
curl -s http://127.0.0.1:8080/health/ready | jq '.status, .checks'
|
||||
|
||||
# Counts match the source system
|
||||
docker compose exec -T mongodb mongosh relata --quiet --eval '
|
||||
["items","chunks","entities","claims","relationships","users"].forEach(
|
||||
c => print(c.padEnd(16) + db[c].countDocuments({})))'
|
||||
|
||||
# Vector count matches the chunk count
|
||||
curl -s -X POST http://127.0.0.1:6333/collections/relata_chunks/points/count \
|
||||
-H 'Content-Type: application/json' -d '{"exact":true}' | jq .result.count
|
||||
|
||||
# Any item left mid-pipeline
|
||||
docker compose exec -T mongodb mongosh relata --quiet --eval '
|
||||
db.items.find({status: {$nin: ["ready","partial","tombstoned"]}}, {title:1, status:1}).forEach(printjson)'
|
||||
```
|
||||
|
||||
Reprocess anything the last query returns, from the item page or:
|
||||
|
||||
```bash
|
||||
curl -s -b cj.txt -X POST "http://127.0.0.1:8080/api/v1/items/$ID/reprocess" \
|
||||
-H 'Content-Type: application/json' -H "X-CSRF-Token: $CSRF" \
|
||||
-d '{"reason":"restored from backup"}'
|
||||
```
|
||||
|
||||
Reprocessing is idempotent: chunks and claims are replaced, entities merge on their normalised
|
||||
key, and relationships upsert on a dedupe key **without overwriting human review decisions**.
|
||||
|
||||
---
|
||||
|
||||
## What a backup preserves
|
||||
|
||||
Worth knowing, because it is the point of the audit design:
|
||||
|
||||
- **Original content**, exactly as supplied.
|
||||
- **Review history** — every confirmation, rejection and type change, with actor and timestamp,
|
||||
append-only.
|
||||
- **Original model proposals**, retained even where a person overruled them.
|
||||
- **Processing runs**, recording which parser, embedding model, analysis model and scoring
|
||||
version produced each generation of derived data.
|
||||
- **Audit events** for logins, ingestion, deletion, reprocessing and relationship decisions.
|
||||
|
||||
Restoring therefore recovers not just the data but the record of how it came to be as it is.
|
||||
|
||||
---
|
||||
|
||||
## Retention and disposal
|
||||
|
||||
Deleting an item removes its chunks, vectors, claims, orphaned entities, relationships and the
|
||||
stored original. Audit events and processing runs are **deliberately retained**: they record
|
||||
that something existed and was removed, without retaining its content.
|
||||
|
||||
Backups are not covered by that deletion. If a user asks for their data to be removed, older
|
||||
backups will still contain it until they age out — plan a retention window accordingly.
|
||||
|
||||
---
|
||||
|
||||
## Disaster recovery drill
|
||||
|
||||
Worth doing once, before you need it:
|
||||
|
||||
```bash
|
||||
# 1. Back up.
|
||||
./scripts/backup.sh
|
||||
|
||||
# 2. Destroy everything.
|
||||
make reset
|
||||
|
||||
# 3. Restore, then create an account if the users collection was not restored.
|
||||
# (mongorestore --drop restores users too, so a restore normally includes them.)
|
||||
|
||||
# 4. Verify with the checks above, then run a demonstration search.
|
||||
```
|
||||
@@ -0,0 +1,214 @@
|
||||
# Relata — Implementation Plan
|
||||
|
||||
Status: living document. Written before implementation, updated as decisions were made.
|
||||
|
||||
## Goal
|
||||
|
||||
A self-organising semantic document database: ingest arbitrary information, discover
|
||||
**evidence-backed** relationships between stored items, and retrieve related knowledge with
|
||||
an explanation of *why* each result was returned.
|
||||
|
||||
## Non-negotiable invariants
|
||||
|
||||
These shape every design choice below.
|
||||
|
||||
1. **Originals are never rewritten.** The raw bytes/text a user supplied are stored verbatim.
|
||||
Everything else (extracted text, chunks, entities, claims) is derived and reproducible.
|
||||
2. **Three different numbers, never conflated:**
|
||||
- *semantic similarity* — cosine distance between embeddings,
|
||||
- *relationship strength* — a versioned weighted sum of explicit signals,
|
||||
- *query relevance* — a fused retrieval score for one specific query.
|
||||
Each is stored in its own field and rendered with its own label.
|
||||
3. **Provenance for every inference.** Each relationship records how it came to exist
|
||||
(`explicit` / `deterministic` / `model_inferred` / `user_confirmed` / `user_rejected`)
|
||||
and carries the signal list that produced its score.
|
||||
4. **Model output is untrusted input.** It is parsed with `json.Decoder` against a strict
|
||||
schema, enum-validated, and referenced IDs are checked against what was actually sent.
|
||||
Invalid output is recorded and discarded, never repaired by regex.
|
||||
5. **Stored document content is data, never instruction.** Retrieved text is delimited and
|
||||
the LLM is told to treat it as untrusted quoted material.
|
||||
|
||||
## Architecture
|
||||
|
||||
Modular monolith + in-process background workers. One binary, several subcommands.
|
||||
|
||||
```
|
||||
cmd/relata serve | createadmin | seed | version
|
||||
internal/domain pure types + port interfaces. No I/O imports.
|
||||
internal/config env + file config, validated at startup
|
||||
internal/parsing text, markdown, JSON, PDF, DOCX -> ExtractedDoc
|
||||
internal/chunking structure-aware chunker
|
||||
internal/embeddings Embedder port + Ollama adapter + batching/retry
|
||||
internal/analysis LLM port + Ollama adapter + strict JSON schema validation
|
||||
internal/entities normalisation, deterministic + model-assisted extraction
|
||||
internal/relationships candidate discovery, versioned scoring, review
|
||||
internal/search hybrid retrieval (lexical + vector + entity + relationship expansion)
|
||||
internal/ranking score fusion and explanation building
|
||||
internal/ingest pipeline orchestration
|
||||
internal/jobs Mongo-backed leased job queue + bounded worker pool
|
||||
internal/storage/mongo repositories implementing domain ports
|
||||
internal/storage/qdrant vector store implementing domain ports
|
||||
internal/auth argon2id, sessions, CSRF, login rate limiting,
|
||||
OAuth 2.0 + OIDC sign-in, per-user workspace provisioning,
|
||||
hashed and scoped API tokens
|
||||
internal/api /api/v1 REST
|
||||
internal/web server-rendered HTML + HTMX
|
||||
internal/audit audit event recording
|
||||
internal/observability structured logs, request IDs, Prometheus metrics
|
||||
```
|
||||
|
||||
**Dependency rule:** `internal/domain` imports only the standard library. Storage adapters
|
||||
import domain, never the reverse. `api` and `web` depend on services, not on storage.
|
||||
|
||||
### Why ports live in `domain`
|
||||
|
||||
Repository and provider interfaces are declared in `internal/domain` next to the types they
|
||||
carry, so the service packages compile against domain alone. Adapters
|
||||
(`storage/mongo`, `storage/qdrant`, `embeddings`, `analysis`) implement them.
|
||||
|
||||
## Data model (MongoDB)
|
||||
|
||||
`users, sessions, workspaces, items, chunks, entities, entity_aliases, claims,
|
||||
relationships, jobs, processing_runs, model_versions, audit_events, api_tokens`
|
||||
|
||||
- String UUID `_id` everywhere. Chunk IDs double as Qdrant point IDs (Qdrant accepts UUIDs),
|
||||
which removes an entire class of ID-mapping bugs.
|
||||
- **Every** user-content collection carries `workspace_id`, and every repository method takes
|
||||
a workspace and includes it in the filter. There is no repository method that can read
|
||||
user content without a workspace.
|
||||
- Qdrant payloads hold only `workspace_id, item_id, chunk_id, source_type, created_at,
|
||||
embedding_model` — retrieval metadata, not a second document store.
|
||||
|
||||
## Pipeline
|
||||
|
||||
`ingest -> parse -> chunk -> embed -> extract entities -> extract claims -> discover relationships`
|
||||
|
||||
Each stage is a persistent job with a lease. Stages are idempotent: they key derived records
|
||||
off deterministic hashes so a retry replaces rather than duplicates.
|
||||
|
||||
Candidate relationship discovery never does an all-pairs comparison. It gathers a bounded
|
||||
candidate set from (a) vector neighbours of the new item's chunks, (b) items sharing a
|
||||
normalised entity, (c) items in a temporal window, then scores that set.
|
||||
|
||||
## Scoring
|
||||
|
||||
`relationship-score-v1`: weighted sum of independent signals (semantic similarity, exact
|
||||
entity overlap, alias overlap, shared concepts, temporal proximity, shared metadata, claim
|
||||
support/contradiction, model classification, user confirmation/rejection), each stored with
|
||||
its own value and weight so the UI can show the arithmetic. The LLM contributes **one
|
||||
signal** — never the score.
|
||||
|
||||
## Build order
|
||||
|
||||
1. domain, config, ids, observability
|
||||
2. storage/mongo, storage/qdrant
|
||||
3. parsing, chunking
|
||||
4. embeddings, analysis (ports + Ollama adapters)
|
||||
5. jobs queue + workers
|
||||
6. entities, claims, relationships, ingest
|
||||
7. search, ranking, answer generation
|
||||
8. auth, api, web
|
||||
9. seed, docs, tests, Docker/Makefile
|
||||
|
||||
## Deliberate scope limits (MVP)
|
||||
|
||||
- No OCR. Scanned PDFs are reported as low-extractable-text, original preserved.
|
||||
- No archive formats (zip/tar/gz) — rejected at upload.
|
||||
- Single workspace in practice, but the boundary is enforced in code and storage from day one.
|
||||
- Entity resolution proposes aliases; it never auto-merges on name similarity alone.
|
||||
- Relationship scores are heuristic strengths, not calibrated probabilities. Stated in the UI.
|
||||
|
||||
---
|
||||
|
||||
## Outcome
|
||||
|
||||
Built as planned. The notes below record what changed during implementation and what running
|
||||
the system revealed, because a plan that claims to have gone exactly to plan is not a useful
|
||||
document.
|
||||
|
||||
### Bugs found by running it, not by reading it
|
||||
|
||||
Six defects survived code review and were caught by operating the system, testing it under
|
||||
concurrency, or tightening validation elsewhere. Each now has a regression test.
|
||||
|
||||
1. **Worker deadlock.** `execute` waited for the heartbeat goroutine to exit, while that
|
||||
goroutine's only exit was the job context being cancelled — which happened via `defer`
|
||||
*after* `execute` returned. Every worker wedged after one job; three workers meant three
|
||||
stuck jobs and a queue that never drained. Found by watching the queue page stop moving.
|
||||
`TestWorkerCompletesJobsAndDoesNotDeadlock` queues more jobs than there are workers.
|
||||
|
||||
2. **Dates leaking into figures.** The number pattern could begin at the whitespace *before* a
|
||||
date and take `202` out of `2025-02-10`. The overlap check tested only the match's start
|
||||
offset, so the fragment survived and was then compared against other documents' quantities —
|
||||
manufacturing contradictions between documents that agreed. Found by reading the evidence
|
||||
behind a suspicious `contradicts`. Fixed by testing the whole span.
|
||||
|
||||
3. **Mirror-image relationships.** The dedupe key included endpoint order, so a link found
|
||||
while processing A and rediscovered while processing B became two records. Every pair in
|
||||
the demonstration set was duplicated. `PairKey` now normalises the order.
|
||||
|
||||
4. **Incomparable ranking curves.** Scaled contributions (entity match, relationship
|
||||
expansion) used `weight × strength / 15` while ranked contributions used
|
||||
`weight / (60 + rank)`. A full-strength expansion therefore outranked a direct keyword hit —
|
||||
"related to a match" beating "is a match". Both now sit on the same reciprocal-rank curve.
|
||||
|
||||
5. **Duplicate detection that a race could defeat.** Exact-duplicate detection read the
|
||||
content hash and then inserted, with only a non-unique index underneath. Eight concurrent
|
||||
uploads of identical content stored four items each claiming to be the only one — so
|
||||
"an identical item already exists" was simply untrue under load. Found by writing the
|
||||
concurrency test rather than by observing it. The fix puts the guarantee where it belongs:
|
||||
a partial unique index on `(workspace_id, content_hash)` covering live, non-deliberate
|
||||
items, with the insert's duplicate-key error reported as exactly the same `DuplicateError`
|
||||
the check produces, so callers see one answer regardless of which path caught it.
|
||||
Deliberate duplicates carry `duplicate_of` and sit outside the constraint.
|
||||
|
||||
6. **A useless model answer failed the whole request.** Adding degeneracy validation to model
|
||||
output surfaced this: `Answer` returned `nil, err` for any generation failure, discarding
|
||||
the retrieved sources and the recorded contradictions between them. Both are produced
|
||||
deterministically and neither depends on the model, so the one case where a user most needs
|
||||
to read the sources themselves was the case that showed them nothing. Invalid output now
|
||||
degrades — no prose, sources and conflicts retained, rejection reason stated — matching the
|
||||
no-material and no-readable-text paths beside it. A timeout or unreachable provider still
|
||||
errors, because that is an operational condition rather than a bad answer.
|
||||
|
||||
A seventh, found by fuzzing rather than running: the Markdown parser set a heading block's `Text`
|
||||
to the trimmed line while its offsets spanned the untrimmed one, breaking the invariant that
|
||||
`block.Text == doc.Text[block.Start:block.End]` — on which every claim and entity source
|
||||
reference depends. The failing input is retained as a corpus seed.
|
||||
|
||||
### Deliberate changes from the plan
|
||||
|
||||
- **`MaxUnreviewedScore = 0.95`.** Not in the original design. Without it a candidate with
|
||||
enough accumulated signals reached 1.00 — the same score as a human confirmation — and the
|
||||
interface presented them identically. That conflates "much evidence" with "confirmed", which
|
||||
is the distinction the whole design exists to preserve.
|
||||
- **Alias signals require a *confirmed* alias.** The plan listed `alias_entity` as a signal
|
||||
without saying which aliases counted. Using unconfirmed candidates would have made a guess
|
||||
into evidence, defeating the point of proposing aliases for review.
|
||||
- **Claim subjects moved to a two-sided rule.** For a value the subject precedes the figure
|
||||
("the cost is £42,000"); for a quantity it follows ("ship 240 **units**"). The one-sided
|
||||
rule produced subjects like "ship" and "ltd", which paired unrelated claims.
|
||||
- **A model that says "unrelated" no longer also contributes its type suggestion.** Both
|
||||
signals were being recorded, so a model could simultaneously argue against and for a link.
|
||||
- **No `scripts/openapi` generator.** The specification is hand-written and a test asserts it
|
||||
matches the routes the router actually registers, plus every enum and error code. A
|
||||
generator would have produced a specification that could only agree with itself.
|
||||
|
||||
### Verified, with the caveats that apply
|
||||
|
||||
Run against MongoDB 7, Qdrant 1.12, and Ollama with `nomic-embed-text` and `llama3.1:8b` on a
|
||||
CPU-only machine.
|
||||
|
||||
- The full pipeline completes for all eleven demonstration documents.
|
||||
- The two superficially similar depot reports acquire **no** relationship, and the note sharing
|
||||
only the entity "Example Ltd" acquires none either — sharing wording, or an entity, is not a
|
||||
relationship.
|
||||
- The cost contradiction is detected deterministically, with both figures named, and is
|
||||
surfaced in generated answers **independently of what the model says**.
|
||||
- Deleting an item leaves no orphaned vectors, chunks, claims or entities.
|
||||
- Reprocessing twice changes no derived-record count, and a human review decision survives it.
|
||||
- Two Relata instances against one MongoDB cooperated correctly through the leased queue —
|
||||
unplanned, but a genuine test of the lease design.
|
||||
|
||||
Known weak points are listed in the README rather than here, so that a reader meets them
|
||||
before they meet a surprise.
|
||||
@@ -0,0 +1,273 @@
|
||||
# Ingestion pipeline
|
||||
|
||||
From "a user pastes a note" to "the note has evidence-backed relationships".
|
||||
|
||||
The governing rule: **the original is preserved.** Whatever a user supplies is stored verbatim
|
||||
before any parsing is attempted. If parsing later fails, the failure is recorded as a warning
|
||||
on the item and the original remains retrievable and reprocessable.
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
```
|
||||
HTTP ──▶ ingest.Service.Ingest ──▶ items collection + parse_item job
|
||||
│
|
||||
┌──────────────────────────────────────────────────┘
|
||||
▼
|
||||
parse_item ─▶ chunk_item ─▶ generate_embeddings ─▶ extract_entities
|
||||
│
|
||||
discover_relationships ◀── extract_claims
|
||||
```
|
||||
|
||||
Each stage is a **separate job type**, so progress is visible on the queue page and a failure
|
||||
is attributable to a specific stage. Stages hand off by enqueuing the next.
|
||||
|
||||
---
|
||||
|
||||
## Stage 0 — Acceptance
|
||||
|
||||
`internal/ingest/service.go`
|
||||
|
||||
Before anything is written:
|
||||
|
||||
1. **Admission control.** A semaphore bounds concurrent ingestion, because each in-flight
|
||||
request holds its content in memory.
|
||||
2. **Size check** against `RELATA_MAX_UPLOAD_BYTES`.
|
||||
3. **Content-based type detection.** `parsing.Sniff` decides from magic bytes. The browser's
|
||||
`Content-Type` and the filename are hints, never authorities.
|
||||
- PDF: `%PDF-`
|
||||
- DOCX: a zip containing `word/document.xml` — checked by reading the central directory
|
||||
- Archives (gzip, bzip2, xz, rar, 7z, plain zip): **rejected**
|
||||
- Executables (ELF, PE, Mach-O, `#!`): **rejected**
|
||||
- Text: must be valid UTF-8, no NUL bytes, under 2% control characters
|
||||
- JSON: parses as a single well-formed value
|
||||
- Markdown vs plain text: decided by filename or declared type, then by content signals
|
||||
4. **Validation** of title length, metadata keys and values, and the user-asserted creation
|
||||
date (a date far in the future is refused, since it would distort every temporal signal).
|
||||
5. **Exact-duplicate detection.** SHA-256 over the supplied bytes, looked up against
|
||||
`(workspace, content_hash)`. An identical item returns `409` with `existing_item_id`
|
||||
unless `allow_duplicate` is set, in which case `duplicate_of` records the link.
|
||||
6. **Original storage.** Binary formats are written to
|
||||
`{upload_dir}/{workspace_id}/{item_id}.{ext}` with mode `0600`. The filename is the item's
|
||||
own UUID and the extension comes from a fixed allow-list — **the user's filename never
|
||||
influences the path.** Text formats are kept inline on the item.
|
||||
7. **Item created**, then `parse_item` enqueued.
|
||||
|
||||
If the declared type and the detected type disagree, a `mime_mismatch` warning is stored. It
|
||||
may be innocent, or it may be an attempt to smuggle one format past a check for another; either
|
||||
way it is visible.
|
||||
|
||||
---
|
||||
|
||||
## Stage 1 — Parse
|
||||
|
||||
`internal/parsing`, handler in `pipeline.go`
|
||||
|
||||
Opens a **processing run**, then extracts text from the preserved original.
|
||||
|
||||
| Format | Approach | Retains |
|
||||
|---|---|---|
|
||||
| Plain text | Paragraph splitting on blank lines; RFC-822 headers recognised as a block | Paragraph offsets |
|
||||
| Markdown | Line-based structural parse: headings, lists, fences, quotes, tables | Heading hierarchy as a section path |
|
||||
| JSON | Depth-bounded streaming scan, then flattening to `path: value` lines, keys sorted | Object boundaries as sections |
|
||||
| PDF | Page by page; a page that fails is skipped with a warning | Page numbers |
|
||||
| DOCX | Streaming XML over `word/document.xml`; Word heading styles → structural headings | Heading hierarchy |
|
||||
|
||||
Every block records **byte offsets into the extracted text**, and the invariant
|
||||
`block.Text == doc.Text[block.Start:block.End]` holds. Every derived record's source
|
||||
reference depends on that identity; it is asserted in unit tests and fuzz-tested.
|
||||
|
||||
**Failure behaviour.** A parse failure is *not* a job failure. The warning is recorded, the
|
||||
item is marked `failed`, the original is untouched, and the job returns successfully — the job
|
||||
did its work correctly, and retrying would only repeat the same failure. The user sees the
|
||||
warning on the item and can reprocess after fixing the cause.
|
||||
|
||||
**Quality warnings** raised here:
|
||||
- `low_extractable_text` — a large file yielding almost no letters, usually a scan. Relata does
|
||||
not OCR, and says so rather than silently indexing nothing.
|
||||
- `truncated_text` — output exceeded the configured limit; blocks past the cut are dropped and
|
||||
the straddling one clamped, so offsets stay valid.
|
||||
- `json_depth_exceeded` — nesting or width beyond the limit.
|
||||
- `parse_failed` — recorded per page for PDFs, or for the whole document.
|
||||
|
||||
---
|
||||
|
||||
## Stage 2 — Chunk
|
||||
|
||||
`internal/chunking`
|
||||
|
||||
Structure first, character counts only as a ceiling.
|
||||
|
||||
**Rules, in order:**
|
||||
|
||||
1. A document below `RELATA_CHUNK_MIN_SPLIT_CHARS` becomes **one chunk**. Splitting a
|
||||
two-sentence note produces fragments that embed worse than the whole.
|
||||
2. A heading starts a new chunk, once the current one is large enough to stand alone.
|
||||
3. A page change (PDF) is a real boundary.
|
||||
4. A JSON object boundary is respected once the chunk has reached its target size.
|
||||
5. Adding a block that would exceed `RELATA_CHUNK_MAX_CHARS` starts a new chunk.
|
||||
6. A single block larger than the ceiling is split internally, preferring blank lines, then
|
||||
sentence terminators, then line breaks, then word breaks.
|
||||
7. Chunks below `RELATA_CHUNK_MIN_CHARS` are merged into a neighbour.
|
||||
|
||||
**Overlap.** Each chunk after the first extends backwards by up to
|
||||
`RELATA_CHUNK_OVERLAP_CHARS`, aligned to a word boundary, so a sentence spanning a boundary is
|
||||
retrievable from both sides. The chunk's `Start` moves back with it, so
|
||||
`chunk.Text == text[Start:End]` still holds; `OverlapChars` records how much was borrowed.
|
||||
|
||||
Each chunk stores: parent item, sequence, source offsets, page, section, heading, text,
|
||||
content hash and (after embedding) the embedding model.
|
||||
|
||||
**Chunk IDs are deterministic** — derived from `(workspace, item, sequence, content_hash)` — so
|
||||
re-chunking unchanged content reuses the same Qdrant point IDs and overwrites vectors rather
|
||||
than orphaning them. Before writing new chunks, the item's existing vectors are deleted.
|
||||
|
||||
---
|
||||
|
||||
## Stage 3 — Embed
|
||||
|
||||
`internal/embeddings`
|
||||
|
||||
- Chunks are embedded in batches of `RELATA_EMBED_BATCH_SIZE`.
|
||||
- An **AI semaphore** is held for the duration of the provider call, so concurrent model usage
|
||||
stays within `RELATA_AI_CONCURRENCY` regardless of worker count.
|
||||
- **Dimension validation.** The first response fixes the width; a later mismatch is a
|
||||
*permanent* error, because writing differently-sized vectors into one collection would
|
||||
corrupt it. The error names the fix.
|
||||
- **Bounded retry.** Only transport failures and explicitly transient HTTP statuses retry, with
|
||||
exponential backoff and full jitter, within `RELATA_AI_MAX_RETRIES`. A 400 or an unknown
|
||||
model never retries. Jitter matters because the worker pool dispatches in parallel: without
|
||||
it, a provider restart would be met by every worker retrying in lockstep.
|
||||
- Vectors go to Qdrant with the minimal payload; chunks are marked embedded with the model name.
|
||||
- **Document text is never logged.**
|
||||
|
||||
---
|
||||
|
||||
## Stage 4 — Entities
|
||||
|
||||
`internal/entities`
|
||||
|
||||
Two stages, separately labelled so a user can tell a matched pattern from a model's guess.
|
||||
|
||||
**Deterministic** (`method = deterministic`) — narrow by design, because a pattern that fires
|
||||
on ordinary prose would flood the index and make shared-entity evidence meaningless:
|
||||
|
||||
- organisations ending in a legal suffix (`Ltd`, `PLC`, `GmbH`, `Inc`, …)
|
||||
- people preceded by an honorific (`Dr`, `Prof`, `Ms`, …)
|
||||
- email addresses
|
||||
- dates in ISO, `14 March 2025`, `March 14, 2025` and `14/03/2025` forms, **normalised to
|
||||
ISO-8601** (day/month/year for the ambiguous numeric form, matching the application's
|
||||
British English conventions)
|
||||
|
||||
**Model-assisted** (`method = model`) — broader concepts, over a bounded sample of chunks.
|
||||
Output is enum-validated per row; one bad row is skipped rather than invalidating the response.
|
||||
A claimed evidence quotation that does not appear in the source is **dropped**, because a
|
||||
fabricated quotation would break the promise that source references point at real text.
|
||||
|
||||
**Resolution.** Entities are keyed on `(workspace, type, normalised_name)` and merged only on
|
||||
an **exact** key match. Normalisation is shallow — case folding, whitespace, punctuation,
|
||||
apostrophes, diacritics — and deliberately does *not* strip legal suffixes: "Example Ltd" and
|
||||
"Example Holdings Ltd" are different companies far more often than they are typos.
|
||||
|
||||
Names that look alike produce a **candidate alias** for a human to confirm, using a looser key
|
||||
plus token overlap, edit distance and acronym detection. Nothing is merged automatically.
|
||||
|
||||
Every mention carries item-relative byte offsets and the verbatim quote.
|
||||
|
||||
---
|
||||
|
||||
## Stage 5 — Claims
|
||||
|
||||
A claim records that a source **said** something. It is not a fact, and it never loses its link
|
||||
to the original text.
|
||||
|
||||
**Deterministic** — sentences carrying a figure or a date, because those are the statements
|
||||
that can actually agree or conflict with another source. Sentence splitting knows common
|
||||
abbreviations, so "Ltd. reported" stays one sentence.
|
||||
|
||||
For each, Relata derives comparison keys:
|
||||
- **subject** — the content word governing the figure ("cost" in "the total project cost is
|
||||
£42,000")
|
||||
- **numbers** — currency- and separator-aware
|
||||
- **dates** — normalised to ISO-8601
|
||||
- **polarity** — +1 affirmative, −1 negated
|
||||
|
||||
**Model-assisted** — broader claims, with attribution and a `quoted`/`paraphrased`/`inferred`
|
||||
nature. A claim whose quotation cannot be located in the source is **discarded**; whitespace
|
||||
reflow is tolerated, because that locates the same span rather than inventing one.
|
||||
|
||||
---
|
||||
|
||||
## Stage 6 — Discover relationships
|
||||
|
||||
`internal/relationships/discover.go`
|
||||
|
||||
**Candidates never come from an all-pairs comparison.** Three bounded, indexed sources:
|
||||
|
||||
1. **Vector neighbours** — a sample of the item's chunks is embedded and searched, filtered to
|
||||
the workspace and excluding the item itself.
|
||||
2. **Shared entities** — an indexed aggregation over `entities.item_ids`, ordered by how many
|
||||
entities are shared. Concepts are excluded as a *source* (too broad) while still
|
||||
contributing a signal.
|
||||
3. **Shared claim subjects** — an indexed lookup on `claims.subject`.
|
||||
|
||||
The union is capped at `RELATA_REL_MAX_CANDIDATES` before any scoring or model call.
|
||||
|
||||
**Evidence gathering** per candidate: shared entities (discounted by corpus frequency),
|
||||
concept overlap, temporal proximity, shared metadata, claim comparison and content-hash match.
|
||||
|
||||
**Claim comparison** is what makes "which sources disagree about the cost?" answerable without
|
||||
a language model: two claims about the same subject with materially different figures (beyond
|
||||
a 0.1% tolerance) or different dates are a **contradiction**, recorded with both quotations
|
||||
attached. Matching figures are **support**.
|
||||
|
||||
**Scoring**, then a bounded number of model classification calls — only for candidates that
|
||||
already have real evidence, so model time is not spent on noise. The model contributes one
|
||||
signal; a maximally confident opinion cannot on its own reach the storage threshold.
|
||||
|
||||
**Storage.** Candidates below `RELATA_REL_STORE_SCORE` are discarded. The rest are stored as
|
||||
`pending` review with full evidence. Non-directional types are stored in a canonical endpoint
|
||||
order so A~B and B~A are one relationship. A near-duplicate (similarity above
|
||||
`RELATA_NEAR_DUPLICATE_SIMILARITY`) is flagged with an explicit note — **never merged**.
|
||||
|
||||
---
|
||||
|
||||
## Failure and recovery
|
||||
|
||||
| Failure | Behaviour |
|
||||
|---|---|
|
||||
| Parse fails | Warning recorded, item marked `failed`, original preserved, job succeeds |
|
||||
| Embedding provider down | Transient — retried with backoff, then the job fails and can be retried from the queue page |
|
||||
| Model returns invalid JSON | Recorded with a reason, **not** retried; deterministic results are kept |
|
||||
| Model unavailable | Entity and claim extraction degrade to deterministic only; the pipeline continues |
|
||||
| Worker crashes | The lease expires and the job returns to the queue |
|
||||
| Job exceeds its attempts | Marked failed; retryable manually |
|
||||
| Item deleted mid-pipeline | Permanent failure — the stage stops rather than resurrecting derived data |
|
||||
|
||||
---
|
||||
|
||||
## Reprocessing
|
||||
|
||||
`RequestReprocess` opens a **new processing run** and restarts the chain at parsing, with the
|
||||
run ID in the job's idempotency key so it is never suppressed as a duplicate of the original
|
||||
ingest.
|
||||
|
||||
Nothing multiplies:
|
||||
|
||||
| Records | Why not |
|
||||
|---|---|
|
||||
| Chunks | Replaced wholesale |
|
||||
| Vectors | Deterministic IDs; the item's vectors are cleared before re-chunking |
|
||||
| Claims | Replaced wholesale; unique dedupe key |
|
||||
| Entities | Upserted on the normalised key; observations accumulate |
|
||||
| Relationships | Upserted on the dedupe key — and **a human review decision is never overwritten** |
|
||||
|
||||
Previous processing runs are **retained**, recording the parser, embedding model, analysis
|
||||
model and scoring version that produced each generation. That is the audit trail for "what did
|
||||
the old parser make of this?"
|
||||
|
||||
Reprocess when a parser changes, the embedding model changes, the analysis model changes, or
|
||||
the relationship scoring algorithm changes. For an embedding-model change across the whole
|
||||
corpus, use **Settings → Re-embed workspace**, which processes in bounded batches and
|
||||
re-enqueues itself rather than holding one lease for hours.
|
||||
@@ -0,0 +1,295 @@
|
||||
# Operations
|
||||
|
||||
Running Relata: what to watch, what the logs mean, and what to do when something is wrong.
|
||||
|
||||
---
|
||||
|
||||
## Health
|
||||
|
||||
| Endpoint | Purpose | Fails when |
|
||||
|---|---|---|
|
||||
| `GET /health/live` | Liveness | The process is gone |
|
||||
| `GET /health/ready` | Readiness | MongoDB or Qdrant is unreachable |
|
||||
|
||||
`/health/live` deliberately checks nothing else: a liveness probe that fails during a brief
|
||||
database blip causes an orchestrator to restart a process that was working perfectly well.
|
||||
|
||||
`/health/ready` is unauthenticated, so it names each dependency and whether it is reachable,
|
||||
but never why. **The reason is in the server log**, at `WARN` with `dependency` and the
|
||||
request ID:
|
||||
|
||||
```bash
|
||||
grep "readiness check failed" relata.log | tail
|
||||
```
|
||||
|
||||
Returning the driver's error to an unauthenticated caller would publish the database host,
|
||||
port and — when the connection string carries them — its credentials.
|
||||
|
||||
`/health/ready` reports the AI providers but **does not require them**. Ingestion, keyword
|
||||
search and browsing all work without a model, so taking the service out of rotation because
|
||||
Ollama restarted would be wrong.
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "ready",
|
||||
"checks": {
|
||||
"mongodb": {"status": "ok", "required": true},
|
||||
"qdrant": {"status": "ok", "required": true},
|
||||
"embedding_provider": {"status": "ok", "required": false},
|
||||
"analysis_provider": {"status": "unavailable", "required": false,
|
||||
"detail": "model \"llama3.1:8b\" is not installed — run: ollama pull llama3.1:8b"}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Metrics
|
||||
|
||||
Prometheus exposition at `GET /metrics`. These describe **the service**, not the stored
|
||||
knowledge: no item ID, entity name or query string ever becomes a label value.
|
||||
|
||||
| Metric | Type | Labels | Watch for |
|
||||
|---|---|---|---|
|
||||
| `relata_http_requests_total` | counter | route, method, status | A rise in `5xx` |
|
||||
| `relata_http_request_duration_seconds` | histogram | route, method | Search latency drift |
|
||||
| `relata_jobs_processed_total` | counter | type, outcome | `outcome="permanent"` climbing |
|
||||
| `relata_job_duration_seconds` | histogram | type | Embedding or analysis slowing |
|
||||
| `relata_jobs_queued` | gauge | status | A backlog that never drains |
|
||||
| `relata_ai_requests_total` | counter | provider, operation, outcome | Provider errors |
|
||||
| `relata_ai_request_duration_seconds` | histogram | provider, operation | Model host under strain |
|
||||
| `relata_storage_operations_total` | counter | store, operation, outcome | Database errors |
|
||||
| `relata_model_invalid_output_total` | counter | task, reason | A model producing unusable output |
|
||||
| `relata_auth_attempts_total` | counter | outcome | `bad_password` or `rate_limited` spikes |
|
||||
| `relata_ingest_items_total` | counter | source_type, outcome | Rejections |
|
||||
| `relata_search_duration_seconds` | histogram | — | Retrieval latency |
|
||||
|
||||
### Alerts worth having
|
||||
|
||||
```yaml
|
||||
- alert: RelataJobsFailing
|
||||
expr: rate(relata_jobs_processed_total{outcome=~"permanent|invalid_model_output"}[15m]) > 0.1
|
||||
annotations:
|
||||
summary: Jobs are failing permanently — check /jobs for the errors
|
||||
|
||||
- alert: RelataQueueBacklog
|
||||
expr: relata_jobs_queued{status="pending"} > 100
|
||||
for: 30m
|
||||
annotations:
|
||||
summary: The queue is not draining — is the AI provider reachable?
|
||||
|
||||
- alert: RelataModelOutputRejected
|
||||
expr: rate(relata_model_invalid_output_total[15m]) > 0.05
|
||||
annotations:
|
||||
summary: The model is producing output that fails validation — consider a larger model
|
||||
|
||||
- alert: RelataLoginAttacks
|
||||
expr: rate(relata_auth_attempts_total{outcome="rate_limited"}[5m]) > 1
|
||||
annotations:
|
||||
summary: Sustained failed sign-ins
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Logs
|
||||
|
||||
Structured, via `log/slog`. Set `RELATA_LOG_JSON=true` in production.
|
||||
|
||||
Every line carries `request_id` or `job_id`, so a user's report ("I got reference
|
||||
`3f2504e0-…`") leads straight to the failure.
|
||||
|
||||
```json
|
||||
{"time":"...","level":"INFO","msg":"job completed","job_id":"...","job_type":"extract_entities",
|
||||
"item_id":"...","worker":"host-a1b2-w0","duration_ms":4210}
|
||||
```
|
||||
|
||||
**Never logged:** passwords, session tokens, API credentials, complete uploaded documents,
|
||||
embedding vectors. Content appears as lengths and hashes; user agents as a truncated hash.
|
||||
|
||||
Levels: `debug` (per-chunk detail, noisy), `info` (requests, job outcomes — the default),
|
||||
`warn` (degraded: model unavailable, lease lost, parse warnings), `error` (failures needing
|
||||
attention).
|
||||
|
||||
---
|
||||
|
||||
## Common problems
|
||||
|
||||
### Items stay "Waiting" and the queue does not move
|
||||
|
||||
The workers are not running or cannot reach a dependency.
|
||||
|
||||
```bash
|
||||
grep "background workers started" relata.log # should appear at startup
|
||||
curl -s localhost:8080/health/ready | jq .checks
|
||||
curl -s localhost:11434/api/tags | jq '.models[].name'
|
||||
```
|
||||
|
||||
If `RELATA_WORKERS_ENABLED=false`, nothing will ever process. If the embedding model is
|
||||
missing, `ollama pull` it — jobs retry and will drain once it is back.
|
||||
|
||||
### "model ... is not installed"
|
||||
|
||||
```bash
|
||||
ollama pull nomic-embed-text
|
||||
ollama pull llama3.1:8b
|
||||
```
|
||||
|
||||
On a CPU-only machine, use `llama3.2:3b` for analysis and set `RELATA_ANALYSIS_MODEL`
|
||||
accordingly. Expect entity and relationship quality to be noticeably lower with a small model.
|
||||
|
||||
### Jobs fail with "invalid model output"
|
||||
|
||||
The model is returning JSON that fails schema validation. These are **not retried**, by design.
|
||||
|
||||
```bash
|
||||
curl -s -b cj.txt 'localhost:8080/api/v1/jobs?status=failed' | jq '.jobs[].last_error'
|
||||
```
|
||||
|
||||
Usually a model too small to follow a schema. Use a larger one, or set
|
||||
`RELATA_ANALYSIS_ENABLED=false` to run the deterministic pipeline only — ingestion, search and
|
||||
relationships still work.
|
||||
|
||||
### Search returns nothing for content that is clearly stored
|
||||
|
||||
1. Is the item `ready`? Check the queue page.
|
||||
2. Is meaning-based search available? Search responses carry a note when it is not.
|
||||
3. Do the vector and chunk counts match on the settings page? If not, re-embed.
|
||||
4. Did the embedding model change? Vectors written by a different model are excluded from
|
||||
search by design, because comparing across models is meaningless.
|
||||
|
||||
### "qdrant collection stores N-dimensional vectors but the model produces M"
|
||||
|
||||
The embedding model changed. Either restore the previous model, or:
|
||||
|
||||
```bash
|
||||
# set RELATA_EMBEDDING_DIMENSIONS to M, then
|
||||
curl -X DELETE http://127.0.0.1:6333/collections/relata_chunks
|
||||
# restart Relata, then Settings → Re-embed workspace
|
||||
```
|
||||
|
||||
### Answer generation times out
|
||||
|
||||
A large model on CPU can exceed the default 120-second analysis timeout when asked to read
|
||||
several passages. The API returns **504** with code `dependency_timeout` and names the fix.
|
||||
|
||||
Measured on a CPU-only machine with `llama3.1:8b`:
|
||||
|
||||
| Sources in the prompt | Time |
|
||||
|---|---|
|
||||
| 3 | ~33 s |
|
||||
| 6 | exceeds 120 s |
|
||||
|
||||
The shipped defaults already account for this — six passages, a 300-second analysis timeout
|
||||
and a 360-second write timeout. If you still hit it:
|
||||
|
||||
```bash
|
||||
RELATA_ANSWER_MAX_CHUNKS=3 # fewer passages per answer (cheapest fix)
|
||||
RELATA_ANALYSIS_TIMEOUT=600s # allow the model longer
|
||||
RELATA_HTTP_WRITE_TIMEOUT=660s # must exceed the analysis timeout
|
||||
RELATA_ANALYSIS_MODEL=llama3.2:3b # a smaller model
|
||||
```
|
||||
|
||||
`RELATA_HTTP_WRITE_TIMEOUT` must exceed `RELATA_ANALYSIS_TIMEOUT`, or the connection closes
|
||||
while the model is still working and the user sees nothing at all. Configuration validation
|
||||
refuses to start on that combination rather than letting it fail silently at request time.
|
||||
|
||||
### Everything is slow
|
||||
|
||||
Almost always the model host. Check `relata_ai_request_duration_seconds`. A 8B model on CPU
|
||||
takes seconds per call, and entity plus claim extraction makes several calls per chunk.
|
||||
|
||||
Options: a smaller analysis model; lower `RELATA_REL_MAX_MODEL_CALLS`; raise
|
||||
`RELATA_AI_CONCURRENCY` **only** if the model host has capacity; or disable analysis entirely.
|
||||
|
||||
### A worker died mid-job
|
||||
|
||||
Nothing to do. The lease expires, `ReleaseExpired` returns the job to the queue, and another
|
||||
worker picks it up. You will see:
|
||||
|
||||
```
|
||||
WARN returned jobs from expired leases to the queue count=2
|
||||
```
|
||||
|
||||
Jobs out of attempts are marked failed and can be retried from the queue page.
|
||||
|
||||
---
|
||||
|
||||
## Routine maintenance
|
||||
|
||||
**Daily** — check `/health/ready`; check failed jobs on the queue page.
|
||||
|
||||
**Weekly** — review relationships awaiting review (the dashboard shows the count); confirming
|
||||
and rejecting improves search, because confirmed links carry more weight in expansion and
|
||||
rejected ones stop pulling results in.
|
||||
|
||||
**Monthly** — `make security` (includes `govulncheck`); check the settings page for a
|
||||
vector/chunk mismatch; verify a backup restores.
|
||||
|
||||
**On any model change** — reprocess affected items, or re-embed the workspace. Both are
|
||||
idempotent.
|
||||
|
||||
---
|
||||
|
||||
## Capacity
|
||||
|
||||
Rough figures on a modern laptop, CPU-only, with `llama3.2:3b`:
|
||||
|
||||
| Operation | Time |
|
||||
|---|---|
|
||||
| Ingest and parse a small note | < 100 ms |
|
||||
| Chunk and embed one chunk | 0.3–1 s |
|
||||
| Model entity extraction per chunk | 2–10 s |
|
||||
| Model claim extraction per chunk | 2–10 s |
|
||||
| Relationship discovery per item | 5–30 s, depending on model calls |
|
||||
| Search (no answer generation) | 20–200 ms |
|
||||
| Answer generation | 5–30 s |
|
||||
|
||||
A GPU changes the model figures by an order of magnitude. Deterministic-only mode
|
||||
(`RELATA_ANALYSIS_ENABLED=false`) processes an item in well under a second.
|
||||
|
||||
Storage: roughly 3 KB per chunk in Qdrant at 768 dimensions, plus the original and extracted
|
||||
text in MongoDB.
|
||||
|
||||
---
|
||||
|
||||
## Deployment notes
|
||||
|
||||
**Keep the loopback binding** unless you have a specific reason not to. If you must expose it:
|
||||
|
||||
1. Put it behind a reverse proxy terminating TLS.
|
||||
2. Set `RELATA_BASE_URL` to the https URL — this enables `Secure` cookies and sets the trusted
|
||||
CSRF origin.
|
||||
3. Set `RELATA_HTTP_ADDR=0.0.0.0:8080` **and** `RELATA_ALLOW_EXTERNAL_BIND=true`; Relata
|
||||
refuses to start otherwise, so exposure is always deliberate.
|
||||
4. Set `RELATA_TRUST_PROXY_HEADERS=true` **only** if the proxy strips client-supplied
|
||||
forwarding headers. Otherwise anyone can defeat login rate limiting with a forged header.
|
||||
|
||||
**Least-privilege MongoDB:**
|
||||
|
||||
```javascript
|
||||
use relata
|
||||
db.createUser({
|
||||
user: "relata_app",
|
||||
pwd: passwordPrompt(),
|
||||
roles: [{ role: "readWrite", db: "relata" }]
|
||||
})
|
||||
```
|
||||
|
||||
Relata creates its own indexes at startup, which `readWrite` permits. It never needs
|
||||
`dbAdmin` or `clusterAdmin`.
|
||||
|
||||
**Container:** the image runs as UID 65532 on a read-only root filesystem with all
|
||||
capabilities dropped. The upload volume is the only writable path and must be writable by
|
||||
that UID.
|
||||
|
||||
---
|
||||
|
||||
## Upgrading
|
||||
|
||||
1. Back up (see [BACKUP_AND_RESTORE.md](BACKUP_AND_RESTORE.md)).
|
||||
2. Read the release notes for changes to the parser, scoring or embedding defaults.
|
||||
3. Deploy. Index creation is idempotent and runs at startup.
|
||||
4. If the parser version changed, reprocess items to benefit — old items keep working with
|
||||
their existing extraction.
|
||||
5. If the scoring version changed, existing relationships keep the version that produced them.
|
||||
Reprocess to recompute; human review decisions survive.
|
||||
@@ -0,0 +1,282 @@
|
||||
# Search and scoring
|
||||
|
||||
Relata computes **three different numbers** and keeps them apart everywhere — in storage, in
|
||||
the API, and on screen. Conflating them would make every explanation meaningless.
|
||||
|
||||
| Number | Question it answers | Range | Where it lives |
|
||||
|---|---|---|---|
|
||||
| **Semantic similarity** | How alike are these two pieces of text? | ~0–1 cosine | `VectorMatch.Score`, `Result.semantic_similarity` |
|
||||
| **Relationship strength** | How well evidenced is a link between two items? | 0–1 | `Relationship.score` + `score_version` |
|
||||
| **Query relevance** | How well does this item answer *this* query? | 0–1, relative | `Result.query_relevance` + `rank_version` |
|
||||
|
||||
None of them is a probability. All three are stated as such wherever they are displayed.
|
||||
|
||||
---
|
||||
|
||||
## 1. Semantic similarity
|
||||
|
||||
Cosine similarity between embedding vectors, computed by Qdrant. It is what it is: a measure
|
||||
of how alike two passages read.
|
||||
|
||||
The critical property: **similarity is not evidence of a factual relationship.** Two incident
|
||||
reports about the same warehouse, written by the same team in the same house style, will be
|
||||
highly similar and describe entirely different events. The demonstration dataset contains
|
||||
exactly that pair.
|
||||
|
||||
Relata therefore:
|
||||
|
||||
- reports similarity as its own field, never folded into another score;
|
||||
- rebases it before it becomes a relationship signal (see below);
|
||||
- uses `similar_to` as the relationship type when similarity is all there is.
|
||||
|
||||
---
|
||||
|
||||
## 2. Relationship strength
|
||||
|
||||
**Version: `relationship-score-v1`** (constant `relationships.ScoreVersion`)
|
||||
|
||||
A relationship's strength is the sum of weighted contributions from named signals, capped
|
||||
at 1.00:
|
||||
|
||||
```
|
||||
score = min(1, Σ (signal.strength × signal.weight × diminishing_factor))
|
||||
```
|
||||
|
||||
Every signal is stored on the relationship, so the arithmetic is always inspectable:
|
||||
|
||||
```json
|
||||
{
|
||||
"score": 0.72,
|
||||
"score_version": "relationship-score-v1",
|
||||
"signals": [
|
||||
{
|
||||
"type": "semantic_similarity",
|
||||
"value": "0.89 cosine",
|
||||
"strength": 0.76,
|
||||
"weight": 0.30,
|
||||
"detail": "How alike the wording and subject matter are. Similar wording is not evidence that two items describe the same thing."
|
||||
},
|
||||
{
|
||||
"type": "shared_entity",
|
||||
"value": "Example Ltd",
|
||||
"strength": 1.00,
|
||||
"weight": 0.25,
|
||||
"detail": "Both items refer to this organisation."
|
||||
},
|
||||
{
|
||||
"type": "claim_contradiction",
|
||||
"value": "cost",
|
||||
"strength": 1.00,
|
||||
"weight": 0.22,
|
||||
"detail": "The sources give different figures for cost: 42000 and 51500. Relata does not decide which is correct."
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Signal weights
|
||||
|
||||
| Signal | Weight | What it means |
|
||||
|---|---:|---|
|
||||
| `content_hash_match` | 0.45 | Byte-identical content |
|
||||
| `semantic_similarity` | 0.30 | Rebased cosine similarity |
|
||||
| `shared_entity` | 0.25 | An exact entity match, discounted by corpus frequency |
|
||||
| `claim_support` | 0.22 | Two claims agree about the same subject |
|
||||
| `claim_contradiction` | 0.22 | Two claims disagree about the same subject |
|
||||
| `model_classification` | 0.15 | A language model's opinion — one signal, never the score |
|
||||
| `shared_concept` | 0.12 | Overlapping concept vocabulary |
|
||||
| `alias_entity` | 0.10 | A match through an alias a person **confirmed**. Unconfirmed candidates contribute nothing. |
|
||||
| `temporal_proximity` | 0.10 | Created close together in time |
|
||||
| `shared_metadata` | 0.08 | Matching user-supplied metadata |
|
||||
| `user_confirmation` | **1.00** | A person confirmed it |
|
||||
| `user_rejection` | **−1.00** | A person rejected it |
|
||||
|
||||
### Why these weights
|
||||
|
||||
The weights encode three deliberate judgements.
|
||||
|
||||
**Similarity alone cannot look like evidence.** Cosine similarity between unrelated documents
|
||||
from one corpus routinely sits at 0.5–0.7, so raw cosine carries almost no information at the
|
||||
low end. The similarity signal rebases against a floor of 0.55:
|
||||
|
||||
```
|
||||
strength = max(0, (cosine − 0.55) / (1 − 0.55))
|
||||
```
|
||||
|
||||
A perfect 0.99 similarity therefore yields `0.98 × 0.30 ≈ 0.29` — below the default storage
|
||||
threshold of 0.25 only marginally, and far below anything that reads as strong. A pair of
|
||||
documents that are merely alike will be recorded as `similar_to` at a weak score, which is
|
||||
the honest description.
|
||||
|
||||
**A model cannot manufacture a relationship.** The model signal's weight is 0.15, so even a
|
||||
maximally confident classification contributes 0.15 — below the 0.25 storage threshold. A
|
||||
model opinion can *shape* a relationship that other evidence already supports; it cannot
|
||||
create one. There is a test asserting this.
|
||||
|
||||
**Human decisions dominate.** Confirmation adds 1.00, saturating the score. Rejection
|
||||
subtracts 1.00, driving it to zero. Neither deletes anything: the original proposal, its
|
||||
provenance and its signals are all retained.
|
||||
|
||||
### Diminishing returns
|
||||
|
||||
Repeated signals of one type contribute with a factor of 1.0, 0.5, 0.25, then nothing:
|
||||
|
||||
```
|
||||
shared_entity × 6 → 0.25 + 0.125 + 0.0625 = 0.44
|
||||
```
|
||||
|
||||
Breadth of evidence beats repetition. A document mentioning one company twelve times should
|
||||
not outrank one that relates on several independent grounds.
|
||||
|
||||
### Entity frequency discount
|
||||
|
||||
An entity appearing in most documents says little about any particular pair:
|
||||
|
||||
```
|
||||
strength ×= 1 / (1 + ln(corpus_frequency))
|
||||
```
|
||||
|
||||
Concepts and subjects start at 0.5 rather than 1.0, because they overlap by nature.
|
||||
|
||||
### Relationship type inference
|
||||
|
||||
Deterministic evidence outranks a model opinion:
|
||||
|
||||
1. `content_hash_match` → `derived_from`
|
||||
2. `claim_contradiction` → `contradicts`
|
||||
3. `claim_support` → `supports`
|
||||
4. the model's type, **except** `same_entity_as`, `supersedes` and `derived_from`, which are
|
||||
never established on a model's word alone
|
||||
5. shared entity plus differing dates → `occurred_before` / `occurred_after`
|
||||
6. shared entity → `mentions`
|
||||
7. otherwise → `similar_to`
|
||||
|
||||
Chronology yields only a chronological statement. Relata never infers `possibly_explains`,
|
||||
`updates` or `supersedes` from ordering alone — that would be inferring causation from
|
||||
sequence, which is precisely the mistake the design forbids.
|
||||
|
||||
### Thresholds
|
||||
|
||||
| Setting | Default | Effect |
|
||||
|---|---|---|
|
||||
| `RELATA_REL_MIN_SCORE` | 0.18 | Below this, a candidate is discarded before any model call |
|
||||
| `RELATA_REL_STORE_SCORE` | 0.25 | At or above this, stored as **pending review** |
|
||||
| `RELATA_NEAR_DUPLICATE_SIMILARITY` | 0.94 | Flagged as a likely near-duplicate — never merged |
|
||||
|
||||
Nothing is ever auto-confirmed. Every discovered relationship arrives as a proposal.
|
||||
|
||||
---
|
||||
|
||||
## 3. Query relevance
|
||||
|
||||
**Version: `query-rank-v1`** (constant `ranking.RankVersion`)
|
||||
|
||||
### Retrieval paths
|
||||
|
||||
A query runs through several independent paths, each producing its own ordering:
|
||||
|
||||
| Path | Weight | Source |
|
||||
|---|---:|---|
|
||||
| Lexical (items) | 1.00 | MongoDB `$text` index over title and extracted text |
|
||||
| Lexical (chunks) | 1.00 | MongoDB `$text` index over chunk text |
|
||||
| Vector | 1.00 | Qdrant cosine search, workspace-filtered |
|
||||
| Entity | 0.60 | Query n-grams matched against normalised entity keys |
|
||||
| Title | 0.50 | Substring match on the item title |
|
||||
| Relationship expansion | 0.35 | Items linked to a strong match |
|
||||
| Confirmed relationship | 0.25 | Bonus when a person confirmed the link |
|
||||
| Recency | 0.15 | Exponential decay, 180-day half-life |
|
||||
|
||||
### Fusion by rank, not by score
|
||||
|
||||
MongoDB text scores and cosine similarities are not on comparable scales — adding them
|
||||
directly would let whichever happens to have the larger range dominate. Relata fuses by
|
||||
**reciprocal rank**:
|
||||
|
||||
```
|
||||
contribution = weight × 1 / (60 + rank)
|
||||
```
|
||||
|
||||
Rank position is comparable by construction. Paths without a meaningful internal ordering
|
||||
(entity match, title match, expansion) contribute a scaled value pitched at roughly a
|
||||
mid-ranked hit, so they augment rather than swamp the ranked paths.
|
||||
|
||||
The final score is normalised against the top result, so the best result in any response is
|
||||
1.00. **This makes relevance comparable within one result set, not across queries.**
|
||||
|
||||
### Relationship expansion
|
||||
|
||||
After direct matching, Relata pulls in items connected to the strongest matches:
|
||||
|
||||
- only from the top `RELATA_SEARCH_MAX_EXPANDED_ITEMS` seeds;
|
||||
- only relationships at or above `RELATA_SEARCH_EXPANSION_MIN_SCORE` (default 0.45);
|
||||
- **never** through a relationship a person has rejected;
|
||||
- optionally only through confirmed relationships (`confirmed_only=true`).
|
||||
|
||||
An expanded result is labelled in the interface and in the API:
|
||||
|
||||
> Not a direct match. Reached because this item **updates** a matching item — strength 0.68
|
||||
> (suggested by analysis, not yet reviewed).
|
||||
|
||||
### Why each result matched
|
||||
|
||||
Every result carries a `why_matched` list and the raw `contributions`:
|
||||
|
||||
```json
|
||||
{
|
||||
"query_relevance": 0.83,
|
||||
"semantic_similarity": 0.89,
|
||||
"why_matched": [
|
||||
"A section of this item is close in meaning to your query (similarity 0.89).",
|
||||
"The item's text contains words from your query.",
|
||||
"This item refers to Example Ltd (organisation), which your query names.",
|
||||
"Created 12 days ago."
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Answer generation
|
||||
|
||||
"Answer from stored knowledge" retrieves, then generates strictly from what was retrieved.
|
||||
|
||||
**Guarantees that hold regardless of what the model returns:**
|
||||
|
||||
1. The model sees only retrieved passages, inside delimited untrusted blocks.
|
||||
2. Passages are labelled `S1`, `S2`… — **item IDs are never shown to the model**, so a
|
||||
fabricated citation cannot resolve to a real record. Citations naming an unsupplied
|
||||
reference are dropped and counted as invalid output.
|
||||
3. An answer claiming sufficiency while citing nothing is downgraded to insufficient.
|
||||
4. Contradictions Relata has recorded between the retrieved items are surfaced
|
||||
**independently of the model's own account**, so a model that quietly picked a side
|
||||
cannot hide the disagreement.
|
||||
5. Inferred relationships that influenced retrieval are listed as proposals, not facts.
|
||||
6. If a passage tried to instruct the model, that is reported rather than hidden.
|
||||
7. **A model that returns nothing usable does not cost you the retrieval.** Output rejected as
|
||||
invalid — including an answer that echoes the schema back (`{"answer": "sufficient"}`) —
|
||||
produces no prose, but the sources and any recorded contradictions between them are still
|
||||
returned, and the rejection reason is stated. Retrieval and contradiction detection are
|
||||
deterministic; neither depends on the model behaving. A transport failure (timeout,
|
||||
unreachable provider) is different and is reported as an error, because that is an
|
||||
operational condition rather than a bad answer.
|
||||
|
||||
Every answer has a **"Sources used"** section, and an **"Uncertainty and conflicts"** section
|
||||
whenever conflicts, insufficiency, inferred-relationship reliance or an injection attempt
|
||||
applies.
|
||||
|
||||
---
|
||||
|
||||
## Changing the scoring model
|
||||
|
||||
Weights live in `relationships.Weights`; retrieval weights in `ranking.Weights`.
|
||||
|
||||
If you change either:
|
||||
|
||||
1. **Bump the version constant.** `relationship-score-v1` → `-v2`. Every stored relationship
|
||||
records the version that produced its score, so mixed generations remain distinguishable.
|
||||
2. Existing relationships are **not** rescored automatically. Reprocess the affected items to
|
||||
recompute them; the dedupe key means this updates rather than duplicates.
|
||||
3. Update the weights table above and the settings page, which reads the live map.
|
||||
4. Re-run `internal/relationships/scoring_test.go`. Several tests encode the design
|
||||
invariants — similarity alone staying weak, a model opinion never reaching the storage
|
||||
threshold, human decisions dominating — and will fail if a change breaks them.
|
||||
@@ -0,0 +1,47 @@
|
||||
# 1. A modular monolith with in-process workers
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Relata has several distinct concerns — ingestion, parsing, embedding, analysis, search — and
|
||||
background work that can take tens of seconds per item. That shape invites splitting into
|
||||
services: an ingestion service, an analysis worker, a search API.
|
||||
|
||||
## Decision
|
||||
|
||||
One binary, one process. Modules are Go packages with enforced dependency boundaries, and
|
||||
background workers run in-process against a job queue stored in MongoDB.
|
||||
|
||||
## Rationale
|
||||
|
||||
The workload does not justify distribution. Relata targets a personal or small-team corpus;
|
||||
the bottleneck is the language model, and that is not helped by splitting the application
|
||||
around it.
|
||||
|
||||
Distribution would add: service discovery, inter-service authentication, distributed tracing
|
||||
to answer questions a single stack trace answers now, partial-failure handling on every call
|
||||
that is currently a function call, and a deployment story requiring an orchestrator.
|
||||
|
||||
The boundaries that matter are enforced without processes. `internal/domain` imports only the
|
||||
standard library; storage and AI adapters implement its ports. That gives the testability and
|
||||
replaceability people usually go to microservices for — the scoring model is unit-tested with
|
||||
nothing running — while keeping a stack trace that spans the whole request.
|
||||
|
||||
The job queue is the one piece that looks like distributed infrastructure, because it must
|
||||
survive a crash. It is a leased queue in MongoDB rather than a message broker: Relata already
|
||||
requires MongoDB, and adding Kafka or RabbitMQ for a queue that peaks at a few hundred jobs
|
||||
would be infrastructure without a purpose.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** one process to deploy, monitor and debug. Refactoring across modules is a compiler
|
||||
problem rather than a migration. Local development needs no orchestrator.
|
||||
|
||||
**Bad:** no independent scaling — a heavy analysis load competes with request handling for the
|
||||
same process, though the worker pool and AI semaphore bound that. No independent deployment.
|
||||
Horizontal scaling would need work, though the leased queue means multiple instances against
|
||||
one MongoDB would already cooperate correctly.
|
||||
|
||||
**If this needs revisiting:** the boundary to extract first is the worker pool, since it
|
||||
already communicates only through the queue.
|
||||
@@ -0,0 +1,47 @@
|
||||
# 2. Relationship strength is computed from signals, not asked of a model
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Relata must decide how strongly two items are related. The obvious approach is to show a
|
||||
language model both documents and ask for a confidence percentage.
|
||||
|
||||
## Decision
|
||||
|
||||
Strength is a weighted sum of named, individually-stored signals under a versioned scoring
|
||||
model. A language model may classify the relationship *type* or describe the connection; its
|
||||
opinion is **one signal**, weighted 0.15 — below the storage threshold, so it cannot create a
|
||||
relationship on its own.
|
||||
|
||||
## Rationale
|
||||
|
||||
A model's percentage cannot be explained, reproduced or debugged. Asked twice it may answer
|
||||
differently; asked why, it produces a plausible rationalisation rather than the reason. A user
|
||||
seeing "87%" has no way to judge it, and no recourse when it is wrong.
|
||||
|
||||
A weighted signal sum is inspectable: the interface shows every signal, its strength, its
|
||||
weight and its contribution, and the total is arithmetic the user can check. When a
|
||||
relationship looks wrong, the reason is visible — and often it is a signal firing incorrectly,
|
||||
which is fixable.
|
||||
|
||||
It is also *stable*. The same evidence yields the same score, so a change in score means a
|
||||
change in evidence, not a change in the model's mood.
|
||||
|
||||
The weights encode judgements that would otherwise be invisible: similarity alone tops out
|
||||
below the storage threshold; a maximally confident model opinion cannot reach it either; human
|
||||
confirmation dominates everything. Those are the design's central claims, and as weights they
|
||||
are both explicit and testable — `scoring_test.go` asserts each one.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** every score is explainable and reproducible. Changing behaviour means changing a
|
||||
weight, visibly. The system works without a language model at all.
|
||||
|
||||
**Bad:** the weights are judgement, not calibration — which is exactly why the output is called
|
||||
a *heuristic strength* and never a probability. Signals must be hand-built; a relationship
|
||||
whose evidence has no corresponding signal is invisible.
|
||||
|
||||
**Rejected alternative:** learning weights from user confirmations. Tempting, but it would make
|
||||
scores unstable, unexplainable, and dependent on a training set that does not exist on first
|
||||
run. Confirmations influence ranking directly instead.
|
||||
@@ -0,0 +1,54 @@
|
||||
# 3. Entities merge only on an exact normalised key
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
"Example Ltd", "Example Limited", "EXAMPLE LTD" and "Example" may all refer to one company.
|
||||
Merging them makes shared-entity evidence stronger and the entity browser tidier.
|
||||
|
||||
## Decision
|
||||
|
||||
Entities are keyed on `(workspace, type, normalised_name)` and merge only on an **exact** key
|
||||
match. Normalisation is shallow: case folding, whitespace, punctuation, apostrophes,
|
||||
diacritics. It deliberately does **not** strip legal suffixes or expand abbreviations.
|
||||
|
||||
Names that look alike produce a **candidate alias** for a human to confirm. Nothing merges
|
||||
automatically.
|
||||
|
||||
## Rationale
|
||||
|
||||
The failure modes are not symmetric.
|
||||
|
||||
Failing to merge costs a little evidence: two entity records where one would do, and a
|
||||
slightly weaker shared-entity signal. Visible, harmless, fixable.
|
||||
|
||||
Merging wrongly is **silent data corruption**. If "Example Ltd" and "Example Holdings Ltd"
|
||||
become one entity, every relationship built on that shared entity is now built on a false
|
||||
premise, and nothing in the interface reveals it. Those are genuinely different companies far
|
||||
more often than they are variant spellings of each other — parent and subsidiary, or two
|
||||
unrelated firms with a common word.
|
||||
|
||||
The same asymmetry applies to people. "Dr A Chen" and "Alice Chen" are probably the same
|
||||
person; "J Smith" and "John Smith" in a corpus of thousands are a coin toss.
|
||||
|
||||
So Relata proposes and a human disposes. The alias machinery uses a deliberately *looser* key —
|
||||
suffixes stripped, plus token overlap, edit distance and acronym detection — precisely because
|
||||
that key is unsafe for identity but useful for suggestion.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** no silent corruption of the entity graph. Shared-entity evidence means what it says.
|
||||
The alias list makes near-matches visible rather than hiding them behind a merge.
|
||||
|
||||
**Bad:** more entity records than strictly necessary. Shared-entity signals are weaker until
|
||||
someone confirms aliases. Confirmation is manual work.
|
||||
|
||||
**Note:** entity type is part of the key, so the place "Washington" and the person "Washington"
|
||||
never collide.
|
||||
|
||||
**Confirming an alias does not merge anything.** It records that a person judged two names to
|
||||
refer to the same thing, and that judgement becomes a `alias_entity` signal weighted at 0.10 —
|
||||
below an exact name match at 0.25. The two entity records remain separate and independently
|
||||
inspectable, so the decision is visible and reversible. An *unconfirmed* candidate contributes
|
||||
nothing at all: treating a proposal as evidence would defeat the purpose of proposing it.
|
||||
@@ -0,0 +1,50 @@
|
||||
# 4. A leased job queue in MongoDB, not a message broker
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Processing takes tens of seconds per item and must survive restarts and crashes. Options: a
|
||||
message broker (RabbitMQ, NATS, Kafka), a dedicated job library backed by Redis, or a queue
|
||||
in the database Relata already has.
|
||||
|
||||
## Decision
|
||||
|
||||
A leased queue in MongoDB. Claiming a job atomically sets an owner and an expiry; the worker
|
||||
renews the lease while it runs; an expired lease returns the job to the pending pool.
|
||||
|
||||
## Rationale
|
||||
|
||||
Relata already requires MongoDB. A broker would add a fourth service to deploy, monitor, back
|
||||
up and secure, for a queue that peaks at a few hundred jobs.
|
||||
|
||||
More importantly, the queue needs properties brokers make awkward:
|
||||
|
||||
- **Jobs are inspectable.** The queue page shows every job, its attempts, its last error and a
|
||||
retry button. That is a database query, not a management API.
|
||||
- **Idempotency is enforced by the store.** A partial unique index on `idempotency_key` for
|
||||
*pending* jobs means a duplicate enqueue is a no-op while completed jobs remain as history.
|
||||
- **Failure history is part of the record.** Each attempt appends to the job, so "why did this
|
||||
item never process?" is answerable from the job itself.
|
||||
|
||||
Leases rather than locks: a lock needs someone to release it, and a crashed worker releases
|
||||
nothing. A lease expires on its own, so crash recovery needs no external coordination. Losing
|
||||
a lease also cancels the worker's context, so a worker that was slow rather than dead abandons
|
||||
its attempt instead of writing a result another worker has superseded.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** no extra infrastructure. Jobs are queryable, inspectable and retryable through the
|
||||
ordinary interface. Crash recovery is automatic and testable. Multiple instances against one
|
||||
MongoDB would already cooperate correctly.
|
||||
|
||||
**Bad:** polling rather than push, so there is up to one poll interval of latency. MongoDB
|
||||
carries the write load of claims and heartbeats. No fan-out or routing — but Relata's pipeline
|
||||
is a linear chain, so neither is needed.
|
||||
|
||||
**Bug worth recording:** the first implementation deadlocked. `execute` waited for the
|
||||
heartbeat goroutine to exit, while that goroutine only exited when the job context was
|
||||
cancelled — which happened via `defer` after `execute` returned. Every worker wedged after one
|
||||
job. It was caught by running the application, not by the unit tests, and
|
||||
`TestWorkerCompletesJobsAndDoesNotDeadlock` now queues more jobs than there are workers so a
|
||||
regression fails immediately.
|
||||
@@ -0,0 +1,59 @@
|
||||
# 5. Model output is untrusted input
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Relata asks a language model for structured data: entities, claims, relationship
|
||||
classifications, answers. The convenient assumption is that a model asked for JSON returns
|
||||
usable JSON.
|
||||
|
||||
## Decision
|
||||
|
||||
Model output is validated exactly as strictly as an HTTP request body from an anonymous
|
||||
client: strict decoding with unknown fields rejected, trailing content refused, size bounded,
|
||||
enum values checked, referenced IDs verified against what was actually supplied, and quoted
|
||||
text confirmed to exist in the source. Invalid output is recorded and discarded — never
|
||||
repaired, never blindly retried.
|
||||
|
||||
## Rationale
|
||||
|
||||
Three distinct failure modes, and the third is the interesting one.
|
||||
|
||||
**Ordinary unreliability.** Models emit trailing prose, wrap JSON in fences, invent fields and
|
||||
occasionally return something that is not JSON at all. Any of these silently corrupts stored
|
||||
data if accepted.
|
||||
|
||||
**Hallucinated references.** A model asked to cite sources will sometimes cite one that does
|
||||
not exist. If citations were trusted, an answer could point at an item that has nothing to do
|
||||
with it — and look authoritative doing so. So the model is shown opaque handles (`S1`, `S2`)
|
||||
rather than item IDs: a fabricated citation cannot resolve to a real record, by construction.
|
||||
The same reasoning applies to claim quotations: a claim whose quote cannot be located in the
|
||||
source is discarded, which is what actually guarantees "a claim never loses its link to the
|
||||
original source text".
|
||||
|
||||
**Compromise.** If the provider were malicious — or if a prompt injection succeeded — the
|
||||
response is attacker-controlled. Strict schema validation means the worst it can do is
|
||||
populate typed fields. It cannot invent a relationship type outside the enum, reference an
|
||||
item it was not shown, or reach anything else, because the model has no tools, no database
|
||||
access and no credentials.
|
||||
|
||||
Deliberately **no regular expressions** are used to find or repair JSON. Regex repair is how
|
||||
malformed output becomes accepted-but-wrong data. A complete markdown fence is unwrapped —
|
||||
a structural operation on a delimiter — and if what remains is not valid JSON it is still
|
||||
rejected.
|
||||
|
||||
Invalid output is **not retried**, because a model that cannot follow a schema will not start
|
||||
on the third attempt; it would only burn the queue. It is recorded on
|
||||
`relata_model_invalid_output_total{task,reason}` so a systematically failing model is visible.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** a model cannot corrupt stored data by being unreliable. A fabricated citation cannot
|
||||
resolve. Rejection rates are measurable, which turns "the model is bad" into a number.
|
||||
|
||||
**Bad:** a strict schema rejects output that a lenient parser would have salvaged, so smaller
|
||||
models yield less. That is the intended trade: less data rather than wrong data.
|
||||
|
||||
**Consequence worth stating:** because rejection is per-row where possible — one invalid entity
|
||||
does not discard the whole response — a partly-usable response is partly used.
|
||||
@@ -0,0 +1,72 @@
|
||||
# 6. Duplicate detection is enforced by an index, not by a check
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Ingestion hashes the incoming content, looks for an existing item with that hash, and reports
|
||||
a duplicate if it finds one. Detection was therefore a read followed by a write, with only a
|
||||
non-unique index underneath.
|
||||
|
||||
That is a time-of-check-to-time-of-use gap. Concurrent requests carrying identical content all
|
||||
perform the read before any of them performs the write, so all of them find nothing and all of
|
||||
them store an item. Eight concurrent uploads of the same document produced four items, each
|
||||
having reported itself as the only copy.
|
||||
|
||||
The gap is not exotic. A retried upload, a double-clicked form, or two workers fed from the
|
||||
same source hit it without anyone trying.
|
||||
|
||||
## Decision
|
||||
|
||||
A partial unique index on `(workspace_id, content_hash)`, covering only items where
|
||||
`duplicate_of = ""` and `deleted = false`. The insert's duplicate-key rejection is mapped to
|
||||
`domain.ErrDuplicate` and reported as exactly the same `DuplicateError` the pre-check produces.
|
||||
|
||||
The pre-check stays. It is what makes the common case a clean, cheap answer that names the
|
||||
existing item; the index is what makes that answer *true*.
|
||||
|
||||
## Rationale
|
||||
|
||||
A uniqueness rule that a race can defeat is not a rule. The database is the only place where
|
||||
"at most one" can be decided, because it is the only place where concurrent writers are
|
||||
serialised. Anything above it is advisory.
|
||||
|
||||
The index is **partial** for two reasons, both of which a blanket unique index would break:
|
||||
|
||||
- A user may deliberately keep a second copy (`allow_duplicate: true`). Those items carry a
|
||||
non-empty `duplicate_of` and fall outside the filter, so any number of them can coexist —
|
||||
the link to the original is recorded rather than the copy being refused.
|
||||
- A tombstoned item still occupies its row until it is hard-deleted. Excluding
|
||||
`deleted = true` means removing an item frees its content hash immediately, rather than
|
||||
blocking re-ingestion for the length of the deletion job.
|
||||
|
||||
Losing the race is handled according to what the caller asked for, not according to which
|
||||
code path noticed. A request with `allow_duplicate: true` that loses the insert has still
|
||||
asked for a second copy to be kept: the race only revealed *which* item it duplicates, so the
|
||||
link is recorded and the insert retried, which succeeds because the item is now outside the
|
||||
partial filter. Refusing it would make the flag mean "unless you were unlucky".
|
||||
|
||||
Reporting the same error from both paths matters more than it appears. A caller that sees
|
||||
`DuplicateError` from the check and an opaque write failure from the race would have to handle
|
||||
two outcomes for one condition, and the seed command, the API and the web upload form would
|
||||
each get it subtly differently.
|
||||
|
||||
`duplicate_of` is stored unconditionally rather than with `omitempty`, because a partial index
|
||||
filter matches an empty string but not an absent field. Items written before that change need
|
||||
a one-off backfill, which `EnsureIndexes` performs itself; it is idempotent and logs what it
|
||||
touched.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** the guarantee holds under concurrency, and the enforcement lives next to the data
|
||||
rather than in whichever caller remembered to check. Deliberate duplicates are unaffected.
|
||||
|
||||
**Bad:** an existing deployment whose data already violates the constraint cannot build the
|
||||
index, and startup fails. That is deliberate — silently proceeding would leave the guarantee
|
||||
untrue while claiming otherwise — but it means the failure message has to name the fix, and
|
||||
`migrations/README.md` has to carry the query that finds the offending rows. Nothing is
|
||||
deleted automatically: which copy is canonical is not a decision Relata should make for you.
|
||||
|
||||
**Rejected alternative:** a transaction around check-and-insert. It would work, but it makes
|
||||
every ingest pay for a case that is rare, requires a replica set, and still leaves the
|
||||
invariant expressed in application code where a future caller can bypass it.
|
||||
@@ -0,0 +1,62 @@
|
||||
# 7. Unusable model output degrades the answer; it does not fail the request
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Model output is validated strictly: strict JSON decoding, enum checks, citations resolved
|
||||
against the references actually supplied, quotations grounded in the source. Anything that
|
||||
fails is rejected rather than repaired.
|
||||
|
||||
Adding degeneracy validation — rejecting free text that is a schema field name, a placeholder,
|
||||
or too short to carry meaning — made a pre-existing weakness visible. `Answer` returned
|
||||
`nil, err` for *any* generation failure. A model that satisfied the schema without answering
|
||||
therefore produced no sources, no citations and no conflicts: the retrieval was discarded
|
||||
along with the response.
|
||||
|
||||
`llama3.2:3b` returns `{"answer": "sufficient", "sufficient": true, "citations": []}` often
|
||||
enough that this was the normal experience on a small model, not an edge case.
|
||||
|
||||
## Decision
|
||||
|
||||
Output rejected as invalid produces an answer with no prose, `sufficient = false`, the
|
||||
retrieved sources intact, the recorded contradictions between them surfaced, and the rejection
|
||||
reason stated in the uncertainty section.
|
||||
|
||||
Transport failures — a timeout, an unreachable provider — still return an error.
|
||||
|
||||
## Rationale
|
||||
|
||||
Retrieval and contradiction detection are deterministic and do not involve the model at all.
|
||||
Discarding them because the model misbehaved throws away the part of the system that worked,
|
||||
in precisely the situation where the user most needs to read the sources themselves.
|
||||
|
||||
It also matches what the surrounding code already does. "Nothing was retrieved" and "the
|
||||
retrieved items had no readable text" both return an answer explaining the situation rather
|
||||
than an error. A useless model response is the third case of the same kind, and it was the
|
||||
only one that behaved differently.
|
||||
|
||||
The distinction between an invalid response and a transport failure is the distinction between
|
||||
a bad answer and a broken dependency. The first is a normal outcome of asking a small model a
|
||||
question, and the user can work around it by reading the sources. The second is an operational
|
||||
condition: it needs to reach the operator as a failure, with the 504 that names the remedy,
|
||||
because no amount of reading sources will fix Ollama being down.
|
||||
|
||||
Stating the rejection reason matters. Silently omitting the prose would look like the model
|
||||
had nothing to say about material that plainly addresses the question.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** a weak model degrades the experience instead of removing it. The guarantees that do
|
||||
not depend on the model — sources shown, contradictions surfaced independently of what the
|
||||
model said — hold in exactly the case where the model is least trustworthy.
|
||||
|
||||
**Bad:** a caller can no longer treat "no error" as "an answer was generated". `Sufficient` and
|
||||
the uncertainty list have to be read, which the API documents and the interface renders, but
|
||||
it is more to get right than a bare error would be.
|
||||
|
||||
**Rejected alternative:** retrying on rejection. A permanently invalid response does not become
|
||||
valid by repetition, and retrying spends the user's time — 100 seconds per attempt on CPU — to
|
||||
reach the same place. Rejections are counted on
|
||||
`relata_model_invalid_output_total{task,reason}` so a model that does this constantly is
|
||||
visible, which is the actionable signal.
|
||||
@@ -0,0 +1,85 @@
|
||||
# 8. Federated identity, keyed on the subject, with a private workspace each
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Relata needed to sit behind an organisation's identity provider, with a public description of
|
||||
the product in front of it. Two questions had to be answered before any code: what identifies
|
||||
an account, and what does a new person get.
|
||||
|
||||
## Decision
|
||||
|
||||
OAuth 2.0 Authorization Code with PKCE, upgraded to OpenID Connect when the provider offers an
|
||||
issuer. Accounts are keyed on `(provider, subject)`. Each person who signs in gets a **private
|
||||
workspace of their own**, in which they are admin.
|
||||
|
||||
Local password login remains, disabled whenever an identity provider is configured, as a way
|
||||
back in when that provider is misconfigured or unreachable.
|
||||
|
||||
## Rationale
|
||||
|
||||
### The subject, never the email
|
||||
|
||||
`sub` is the only identifier a provider undertakes not to reassign. Email addresses are
|
||||
recycled constantly — a person leaves, their address is given to someone new — and an account
|
||||
keyed on one hands the new holder the previous holder's documents. That is not a subtle
|
||||
failure: it is a silent, total disclosure of somebody's corpus to a stranger, and it looks like
|
||||
a successful login.
|
||||
|
||||
The email is still stored, for display and for the optional domain allowlist. It is never
|
||||
looked up.
|
||||
|
||||
### A private workspace each
|
||||
|
||||
Relata already enforces a workspace boundary in every storage query, from the first commit.
|
||||
Giving each person their own reuses that boundary rather than inventing a second notion of
|
||||
separation on top of it — and a mechanism exercised by every query is worth far more than one
|
||||
added for this feature alone.
|
||||
|
||||
The cost is real and worth stating rather than discovering: **relationship discovery never
|
||||
spans users.** Two people who upload documents about the same contract get no link between
|
||||
them, because as far as Relata is concerned they are separate corpora. The product's central
|
||||
value — finding connections you did not know about — stops at the workspace edge. Anyone
|
||||
choosing this model should choose it knowing that.
|
||||
|
||||
Making them admin of their own workspace follows from it. There is nobody else in there to
|
||||
protect anything from, and a person who cannot review the relationships in their own corpus
|
||||
cannot use the system at all.
|
||||
|
||||
### Provisioning must be idempotent
|
||||
|
||||
First sign-in creates a workspace and an account. Two clicks, two tabs, or a retried callback
|
||||
mean two simultaneous first sign-ins, and creating two workspaces for one person would split
|
||||
their corpus in a way nothing would later reconcile. A unique index on
|
||||
`(oauth_provider, oauth_subject)` decides it: the loser re-reads the winner's account.
|
||||
|
||||
A workspace created by a losing racer is abandoned rather than deleted. Deleting it would race
|
||||
with the winner, and an empty unreferenced workspace costs nothing.
|
||||
|
||||
### Local login stays, and defaults to off
|
||||
|
||||
An identity provider is a single point of failure for getting in at all. A wrong client secret,
|
||||
an expired registration, or an outage would otherwise mean editing the database to recover.
|
||||
|
||||
It defaults to on while no provider is configured and off once one is, so a fresh installation
|
||||
works out of the box and a federated one does not keep a second front door open by accident.
|
||||
Setting the variable explicitly always wins. A federated account has no password hash and is
|
||||
refused by the local path explicitly, rather than relying on hash verification to fail on an
|
||||
empty string — the guarantee should not depend on a parser's behaviour.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** identity is stable across email changes; isolation reuses a boundary already
|
||||
enforced everywhere; a broken provider is recoverable without database surgery.
|
||||
|
||||
**Bad:** no shared corpora, so cross-user discovery is impossible by construction. No account
|
||||
linking either — a local account and an OAuth account with the same email are two people with
|
||||
two workspaces, because merging them is a decision to make deliberately, not to infer from a
|
||||
matching address.
|
||||
|
||||
**Rejected alternative:** one shared workspace with roles mapped from provider groups. It would
|
||||
make discovery work across everyone's documents, which is where Relata is most useful — but it
|
||||
means every person who signs in can read everything anyone has uploaded, and group-to-role
|
||||
mapping is a poor place to discover you got the boundary wrong. If shared workspaces are added
|
||||
later, claim-based role mapping belongs with them.
|
||||
@@ -0,0 +1,80 @@
|
||||
# 9. API tokens are hashed, scoped, and bounded by their owner's role
|
||||
|
||||
**Status:** accepted
|
||||
|
||||
## Context
|
||||
|
||||
Programs need to ingest and query without a browser session. Sessions are the wrong shape for
|
||||
that: they expire on idleness, rotate, and are bound to a cookie and a CSRF token.
|
||||
|
||||
## Decision
|
||||
|
||||
Bearer tokens a person creates for themselves. Only a SHA-256 hash is stored. Each token
|
||||
carries explicit scopes (`ingest`, `query`, `review`), and its effective permission is the
|
||||
intersection of those scopes and its owner's **current** role, re-read on every request.
|
||||
|
||||
Tokens authenticate `/api/v1` only, and cannot manage tokens.
|
||||
|
||||
## Rationale
|
||||
|
||||
### Hash, not encrypt
|
||||
|
||||
An encrypted token is recoverable by whoever holds the key, which is the same process that
|
||||
holds the database connection. Hashing means a database read, a backup, a dump or a stolen
|
||||
snapshot yields nothing that authenticates. The cost — the secret cannot ever be shown again —
|
||||
is the point rather than a limitation, and the interface says so at the moment of creation.
|
||||
|
||||
Authentication is a lookup on a unique index over the whole hash. There is no comparison in
|
||||
application code for an attacker to time, and no way to probe for a prefix. The 12-character
|
||||
display prefix is kept in clear so a person can tell their tokens apart; it is far too short to
|
||||
be a usable fraction of a 32-byte secret.
|
||||
|
||||
### Intersection, not assignment
|
||||
|
||||
Scopes could have been a fixed grant checked on its own. Making permission the *intersection*
|
||||
with the owner's live role means one thing that a fixed grant cannot give: **demoting or
|
||||
disabling a person immediately narrows or kills every token they hold**, without anyone having
|
||||
to find those tokens. The user record is re-read per request precisely to make that true; a
|
||||
role cached on the token would mean the change took effect at expiry, which is exactly when it
|
||||
does not matter.
|
||||
|
||||
### Tokens cannot manage tokens
|
||||
|
||||
The single most valuable thing a leaked credential can do is make itself permanent. Requiring
|
||||
an interactive session for creation, listing and revocation means a leaked token has a fixed
|
||||
expiry it cannot extend, cannot mint replacements, and cannot revoke the tokens that would
|
||||
replace it.
|
||||
|
||||
### The CSRF exemption is about *how*, not *whether*
|
||||
|
||||
Bearer requests carry no CSRF token, and should not have to: CSRF defends against a
|
||||
third-party page spending *ambient* credentials, and an `Authorization` header is not ambient —
|
||||
a browser never attaches one by itself.
|
||||
|
||||
The dangerous way to implement that is to skip CSRF when an `Authorization` header is present,
|
||||
because then any cookie-authenticated request can disable the check by adding one. So the
|
||||
exemption keys off how the request actually authenticated, and authentication resolves the
|
||||
session cookie **first**. A request holding a cookie is never token-authenticated, so the
|
||||
header cannot switch anything off. This was verified against a running instance, not only
|
||||
reasoned about.
|
||||
|
||||
### Rejected: PATs that never expire
|
||||
|
||||
Convenient, and the reason so many leaked credentials stay live for years. Every token has an
|
||||
expiry, capped by configuration, and stops working with no further action.
|
||||
|
||||
### Rejected: tokens in query parameters
|
||||
|
||||
A token in a URL ends up in access logs, browser history, referrer headers and screenshots.
|
||||
Only the `Authorization` header is read.
|
||||
|
||||
## Consequences
|
||||
|
||||
**Good:** a stolen database yields no usable credential; a role change takes effect instantly
|
||||
across every token; a leaked token is bounded in scope, in lifetime and in what it can escalate
|
||||
to.
|
||||
|
||||
**Bad:** a lost token cannot be recovered, only revoked and replaced. Scopes are coarse — three
|
||||
of them, checked per route, with no per-item restriction. Creating a token through the
|
||||
interface puts its value in the browser's history for one page load, which the page warns
|
||||
about; the API route returns it in a response body and avoids that entirely.
|
||||
@@ -0,0 +1,19 @@
|
||||
# Architectural decision records
|
||||
|
||||
One file per decision that would be expensive to reverse or surprising to encounter without
|
||||
explanation. Each records the context, what was decided, why, and what it costs — including
|
||||
the alternative that was rejected, since that is usually the part a later reader wants.
|
||||
|
||||
| # | Decision | Turns on |
|
||||
|---|---|---|
|
||||
| [1](0001-modular-monolith.md) | Modular monolith with in-process workers | One binary and a real dependency rule beat a distributed system for a corpus this size |
|
||||
| [2](0002-scoring-not-model-percentage.md) | Relationship strength is computed from signals, not asked of a model | A model's percentage cannot be explained, reproduced or debugged |
|
||||
| [3](0003-conservative-entity-resolution.md) | Entities merge only on an exact normalised key; similar names become candidate aliases | "Example Ltd" and "Example Holdings Ltd" are different companies more often than they are typos |
|
||||
| [4](0004-leased-job-queue-in-mongodb.md) | A leased job queue in MongoDB rather than a broker | A lease expires by itself; a lock held by a dead worker does not |
|
||||
| [5](0005-untrusted-model-output.md) | Model output is parsed as untrusted input and never repaired | A regex that fixes malformed JSON is inventing content the model did not produce |
|
||||
| [6](0006-duplicate-detection-enforced-by-index.md) | Duplicate detection is enforced by a partial unique index, not by a check | A uniqueness rule that a race can defeat is not a rule |
|
||||
| [7](0007-unusable-model-output-degrades.md) | Unusable model output degrades the answer; it does not fail the request | Retrieval and contradiction detection do not involve the model, so a bad response should not discard them |
|
||||
| [8](0008-federated-identity-private-workspaces.md) | Federated identity keyed on the provider subject, with a private workspace each | Email addresses get reassigned; keying on one hands a recycled address the previous holder's documents |
|
||||
| [9](0009-api-tokens-hashed-and-scoped.md) | API tokens are hashed, scoped, and bounded by their owner's live role | A stolen database should yield no usable credential, and demoting someone should narrow every token they hold |
|
||||
|
||||
Superseding a decision means adding a record that says so, not editing the old one.
|
||||
+1349
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user