Code sync, security fixes
This commit is contained in:
Andy Dixon
@@ -0,0 +1,309 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// Role determines what a user may do.
|
||||
type Role string
|
||||
|
||||
const (
|
||||
RoleAdmin Role = "admin"
|
||||
RoleEditor Role = "editor"
|
||||
RoleViewer Role = "viewer"
|
||||
)
|
||||
|
||||
// Valid reports whether r is a recognised role.
|
||||
func (r Role) Valid() bool {
|
||||
switch r {
|
||||
case RoleAdmin, RoleEditor, RoleViewer:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// CanWrite reports whether the role may create or modify content.
|
||||
func (r Role) CanWrite() bool { return r == RoleAdmin || r == RoleEditor }
|
||||
|
||||
// Workspace is the data boundary. Every user-content record carries a
|
||||
// workspace ID and every repository query filters on it.
|
||||
type Workspace struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
Name string `bson:"name" json:"name"`
|
||||
Slug string `bson:"slug" json:"slug"`
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
}
|
||||
|
||||
// User is an account, either local or federated. Only the Argon2id encoded
|
||||
// hash is ever stored for a local account; a federated one has none.
|
||||
type User struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
Username string `bson:"username" json:"username"`
|
||||
// PasswordHash is an encoded Argon2id string including salt and parameters.
|
||||
// It is empty for a federated account, which can never sign in locally.
|
||||
PasswordHash string `bson:"password_hash" json:"-"`
|
||||
|
||||
// OAuthProvider and OAuthSubject identify a federated account.
|
||||
//
|
||||
// The subject is the key, never the email address: `sub` is the only
|
||||
// identifier a provider promises not to reassign, whereas email addresses
|
||||
// are recycled — and keying on one would hand a new holder the previous
|
||||
// holder's documents.
|
||||
OAuthProvider string `bson:"oauth_provider,omitempty" json:"oauth_provider,omitempty"`
|
||||
OAuthSubject string `bson:"oauth_subject,omitempty" json:"oauth_subject,omitempty"`
|
||||
// Email is stored for display and for the optional domain allowlist. It is
|
||||
// not an identifier and nothing is looked up by it.
|
||||
Email string `bson:"email,omitempty" json:"email,omitempty"`
|
||||
DisplayName string `bson:"display_name,omitempty" json:"display_name,omitempty"`
|
||||
Role Role `bson:"role" json:"role"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
Disabled bool `bson:"disabled" json:"disabled"`
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
LastLoginAt *time.Time `bson:"last_login_at,omitempty" json:"last_login_at,omitempty"`
|
||||
}
|
||||
|
||||
// Session is a server-side browser session. The cookie carries only an opaque
|
||||
// token; the stored record holds a hash of that token so a database read cannot
|
||||
// be replayed as a login.
|
||||
type Session struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
TokenHash string `bson:"token_hash" json:"-"`
|
||||
// CSRFToken is a per-session secret compared in constant time on
|
||||
// state-changing browser requests.
|
||||
CSRFToken string `bson:"csrf_token" json:"-"`
|
||||
|
||||
UserID string `bson:"user_id" json:"user_id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
LastSeenAt time.Time `bson:"last_seen_at" json:"last_seen_at"`
|
||||
ExpiresAt time.Time `bson:"expires_at" json:"expires_at"`
|
||||
AbsoluteExpiry time.Time `bson:"absolute_expiry" json:"absolute_expiry"`
|
||||
|
||||
UserAgentHash string `bson:"user_agent_hash,omitempty" json:"-"`
|
||||
Revoked bool `bson:"revoked" json:"revoked"`
|
||||
}
|
||||
|
||||
// Active reports whether the session may still be used at time now.
|
||||
func (s *Session) Active(now time.Time) bool {
|
||||
return !s.Revoked && now.Before(s.ExpiresAt) && now.Before(s.AbsoluteExpiry)
|
||||
}
|
||||
|
||||
// Principal is the authenticated caller carried through request contexts.
|
||||
// Every service call that touches user content takes one.
|
||||
type Principal struct {
|
||||
UserID string
|
||||
Username string
|
||||
DisplayName string
|
||||
Role Role
|
||||
WorkspaceID string
|
||||
SessionID string
|
||||
|
||||
// TokenID and TokenScopes are set when the request authenticated with an
|
||||
// API token rather than a session. A principal carries one or the other,
|
||||
// never both, so downstream code can tell an interactive actor from a
|
||||
// program without inspecting the request.
|
||||
TokenID string
|
||||
TokenScopes []TokenScope
|
||||
}
|
||||
|
||||
// ViaToken reports whether this principal came from an API token.
|
||||
func (p *Principal) ViaToken() bool { return p != nil && p.TokenID != "" }
|
||||
|
||||
// Permits reports whether the principal may perform an action needing scope s.
|
||||
//
|
||||
// A session-authenticated person is limited only by their role. A token is
|
||||
// limited by the intersection of its scopes and that role, so a token can never
|
||||
// do more than the person who created it.
|
||||
func (p *Principal) Permits(s TokenScope) bool {
|
||||
if p == nil || !RoleGrants(p.Role, s) {
|
||||
return false
|
||||
}
|
||||
if !p.ViaToken() {
|
||||
return true
|
||||
}
|
||||
for _, got := range p.TokenScopes {
|
||||
if got == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AuditEvent is an append-only record of a consequential action.
|
||||
type AuditEvent struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
At time.Time `bson:"at" json:"at"`
|
||||
|
||||
Action string `bson:"action" json:"action"`
|
||||
ActorID string `bson:"actor_id,omitempty" json:"actor_id,omitempty"`
|
||||
ActorName string `bson:"actor_name,omitempty" json:"actor_name,omitempty"`
|
||||
TargetKind string `bson:"target_kind,omitempty" json:"target_kind,omitempty"`
|
||||
TargetID string `bson:"target_id,omitempty" json:"target_id,omitempty"`
|
||||
RequestID string `bson:"request_id,omitempty" json:"request_id,omitempty"`
|
||||
Outcome string `bson:"outcome" json:"outcome"`
|
||||
// Detail holds short, non-sensitive descriptive fields. Never credentials,
|
||||
// never full document bodies.
|
||||
Detail map[string]string `bson:"detail,omitempty" json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Well-known audit actions.
|
||||
const (
|
||||
AuditLogin = "auth.login"
|
||||
AuditLoginFailed = "auth.login_failed"
|
||||
AuditLogout = "auth.logout"
|
||||
AuditUserCreated = "user.created"
|
||||
AuditItemCreated = "item.created"
|
||||
AuditItemDeleted = "item.deleted"
|
||||
AuditItemReprocessed = "item.reprocessed"
|
||||
AuditRelConfirmed = "relationship.confirmed"
|
||||
AuditRelRejected = "relationship.rejected"
|
||||
AuditRelEdited = "relationship.edited"
|
||||
AuditRelCreated = "relationship.created"
|
||||
AuditTokenCreated = "api_token.created"
|
||||
AuditTokenRevoked = "api_token.revoked"
|
||||
AuditOAuthLogin = "auth.oauth_login"
|
||||
AuditOAuthProvisioned = "auth.oauth_provisioned"
|
||||
AuditOAuthFailed = "auth.oauth_failed"
|
||||
AuditJobRetried = "job.retried"
|
||||
AuditModelBadOutput = "model.invalid_output"
|
||||
AuditAnswerGenerated = "answer.generated"
|
||||
AuditWorkspaceReembed = "workspace.reembed"
|
||||
)
|
||||
|
||||
// ProcessingRun records one pass of the pipeline over an item, retained for
|
||||
// audit so that a reprocess does not erase what previous versions produced.
|
||||
type ProcessingRun struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
ItemID string `bson:"item_id" json:"item_id"`
|
||||
|
||||
Reason string `bson:"reason" json:"reason"`
|
||||
StartedAt time.Time `bson:"started_at" json:"started_at"`
|
||||
EndedAt *time.Time `bson:"ended_at,omitempty" json:"ended_at,omitempty"`
|
||||
Outcome string `bson:"outcome" json:"outcome"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
|
||||
ParserVersion string `bson:"parser_version,omitempty" json:"parser_version,omitempty"`
|
||||
EmbeddingModel string `bson:"embedding_model,omitempty" json:"embedding_model,omitempty"`
|
||||
AnalysisModel string `bson:"analysis_model,omitempty" json:"analysis_model,omitempty"`
|
||||
ScoreVersion string `bson:"score_version,omitempty" json:"score_version,omitempty"`
|
||||
|
||||
Stats map[string]int `bson:"stats,omitempty" json:"stats,omitempty"`
|
||||
}
|
||||
|
||||
// ModelVersion records a model/algorithm version that has been used, so that
|
||||
// stale derived data can be identified when configuration changes.
|
||||
type ModelVersion struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
Kind string `bson:"kind" json:"kind"`
|
||||
Name string `bson:"name" json:"name"`
|
||||
Dimensions int `bson:"dimensions,omitempty" json:"dimensions,omitempty"`
|
||||
FirstSeenAt time.Time `bson:"first_seen_at" json:"first_seen_at"`
|
||||
LastSeenAt time.Time `bson:"last_seen_at" json:"last_seen_at"`
|
||||
}
|
||||
|
||||
// Federated reports whether the account signs in through an identity provider.
|
||||
func (u *User) Federated() bool { return u != nil && u.OAuthSubject != "" }
|
||||
|
||||
// CanLoginLocally reports whether a username and password could ever
|
||||
// authenticate this account. A federated account has no password hash, so no
|
||||
// submitted password can match one.
|
||||
func (u *User) CanLoginLocally() bool {
|
||||
return u != nil && u.PasswordHash != "" && !u.Federated()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// API tokens
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// TokenScope is a coarse permission carried by an API token.
|
||||
type TokenScope string
|
||||
|
||||
const (
|
||||
// ScopeIngest permits creating, reprocessing and deleting items.
|
||||
ScopeIngest TokenScope = "ingest"
|
||||
// ScopeQuery permits search, answer generation, and reading stored records.
|
||||
ScopeQuery TokenScope = "query"
|
||||
// ScopeReview permits acting on relationship and alias review decisions.
|
||||
ScopeReview TokenScope = "review"
|
||||
)
|
||||
|
||||
// Valid reports whether s is a recognised scope.
|
||||
func (s TokenScope) Valid() bool {
|
||||
switch s {
|
||||
case ScopeIngest, ScopeQuery, ScopeReview:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// AllTokenScopes lists every scope, for validation and for the interface.
|
||||
func AllTokenScopes() []TokenScope {
|
||||
return []TokenScope{ScopeIngest, ScopeQuery, ScopeReview}
|
||||
}
|
||||
|
||||
// RoleGrants reports whether a role is capable of a scope at all. A token can
|
||||
// never exceed the role of the user who created it, so the effective permission
|
||||
// is the intersection of the two — which means demoting a person immediately
|
||||
// narrows every token they hold, without touching the tokens.
|
||||
func RoleGrants(r Role, s TokenScope) bool {
|
||||
switch s {
|
||||
case ScopeQuery:
|
||||
return r.Valid()
|
||||
case ScopeIngest, ScopeReview:
|
||||
return r.CanWrite()
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// APIToken is a non-interactive credential for /api/v1.
|
||||
//
|
||||
// Only the hash of the secret is stored, so neither a database read nor a
|
||||
// backup yields a usable credential.
|
||||
type APIToken struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
UserID string `bson:"user_id" json:"user_id"`
|
||||
|
||||
// Name is the person's own label for the token.
|
||||
Name string `bson:"name" json:"name"`
|
||||
// Prefix is the leading, non-secret portion of the token, kept in clear so
|
||||
// the interface can identify a token without holding it.
|
||||
Prefix string `bson:"prefix" json:"prefix"`
|
||||
// TokenHash is the SHA-256 of the whole token. It is never returned.
|
||||
TokenHash string `bson:"token_hash" json:"-"`
|
||||
|
||||
Scopes []TokenScope `bson:"scopes" json:"scopes"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
ExpiresAt time.Time `bson:"expires_at" json:"expires_at"`
|
||||
LastUsedAt *time.Time `bson:"last_used_at,omitempty" json:"last_used_at,omitempty"`
|
||||
|
||||
RevokedAt *time.Time `bson:"revoked_at,omitempty" json:"revoked_at,omitempty"`
|
||||
// RevokedBy records who revoked it, for the audit trail.
|
||||
RevokedBy string `bson:"revoked_by,omitempty" json:"revoked_by,omitempty"`
|
||||
}
|
||||
|
||||
// Active reports whether the token may authenticate a request at time now.
|
||||
func (t *APIToken) Active(now time.Time) bool {
|
||||
if t == nil || t.RevokedAt != nil {
|
||||
return false
|
||||
}
|
||||
return t.ExpiresAt.IsZero() || now.Before(t.ExpiresAt)
|
||||
}
|
||||
|
||||
// HasScope reports whether the token was granted s.
|
||||
func (t *APIToken) HasScope(s TokenScope) bool {
|
||||
if t == nil {
|
||||
return false
|
||||
}
|
||||
for _, got := range t.Scopes {
|
||||
if got == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// EntityType classifies an extracted subject.
|
||||
type EntityType string
|
||||
|
||||
const (
|
||||
EntityPerson EntityType = "person"
|
||||
EntityOrganisation EntityType = "organisation"
|
||||
EntityLocation EntityType = "location"
|
||||
EntityProduct EntityType = "product"
|
||||
EntityEvent EntityType = "event"
|
||||
EntityDate EntityType = "date"
|
||||
EntitySubject EntityType = "subject"
|
||||
EntityConcept EntityType = "concept"
|
||||
)
|
||||
|
||||
// ValidEntityTypes is the closed set of accepted entity types.
|
||||
var ValidEntityTypes = []EntityType{
|
||||
EntityPerson, EntityOrganisation, EntityLocation, EntityProduct,
|
||||
EntityEvent, EntityDate, EntitySubject, EntityConcept,
|
||||
}
|
||||
|
||||
// Valid reports whether t is a recognised entity type.
|
||||
func (t EntityType) Valid() bool {
|
||||
for _, v := range ValidEntityTypes {
|
||||
if v == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ExtractionMethod records how a derived record was produced.
|
||||
type ExtractionMethod string
|
||||
|
||||
const (
|
||||
MethodDeterministic ExtractionMethod = "deterministic"
|
||||
MethodModel ExtractionMethod = "model"
|
||||
MethodUser ExtractionMethod = "user"
|
||||
)
|
||||
|
||||
// SourceRef points at the exact place a derived record came from.
|
||||
// A claim or entity must never lose this link back to original text.
|
||||
type SourceRef struct {
|
||||
ItemID string `bson:"item_id" json:"item_id"`
|
||||
ChunkID string `bson:"chunk_id,omitempty" json:"chunk_id,omitempty"`
|
||||
// Start and End are byte offsets into the item's extracted text.
|
||||
Start int `bson:"start" json:"start"`
|
||||
End int `bson:"end" json:"end"`
|
||||
// Quote is the verbatim text at those offsets.
|
||||
Quote string `bson:"quote,omitempty" json:"quote,omitempty"`
|
||||
Page int `bson:"page,omitempty" json:"page,omitempty"`
|
||||
}
|
||||
|
||||
// Entity is an identifiable subject extracted from source material.
|
||||
//
|
||||
// Entities are keyed on (workspace, type, normalised name). Two entities whose
|
||||
// names merely look alike are never merged automatically; a candidate alias is
|
||||
// proposed instead and awaits confirmation.
|
||||
type Entity struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
CanonicalName string `bson:"canonical_name" json:"canonical_name"`
|
||||
NormalisedKey string `bson:"normalised_key" json:"normalised_key"`
|
||||
Type EntityType `bson:"type" json:"type"`
|
||||
|
||||
Aliases []string `bson:"aliases,omitempty" json:"aliases,omitempty"`
|
||||
SourceRefs []SourceRef `bson:"source_refs,omitempty" json:"source_refs,omitempty"`
|
||||
ItemIDs []string `bson:"item_ids" json:"item_ids"`
|
||||
|
||||
Method ExtractionMethod `bson:"method" json:"method"`
|
||||
// EvidenceScore reflects how well-attested the entity is (mention count,
|
||||
// extraction agreement). It is not a probability.
|
||||
EvidenceScore float64 `bson:"evidence_score" json:"evidence_score"`
|
||||
Mentions int `bson:"mentions" json:"mentions"`
|
||||
ModelVersion string `bson:"model_version,omitempty" json:"model_version,omitempty"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
Deleted bool `bson:"deleted" json:"deleted"`
|
||||
}
|
||||
|
||||
// AliasStatus tracks whether a proposed alias link has been adjudicated.
|
||||
type AliasStatus string
|
||||
|
||||
const (
|
||||
AliasCandidate AliasStatus = "candidate"
|
||||
AliasConfirmed AliasStatus = "confirmed"
|
||||
AliasRejected AliasStatus = "rejected"
|
||||
)
|
||||
|
||||
// EntityAlias is a proposed "these two names may be the same thing" link.
|
||||
// It exists precisely so that name similarity does not silently merge entities.
|
||||
type EntityAlias struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
EntityID string `bson:"entity_id" json:"entity_id"`
|
||||
Alias string `bson:"alias" json:"alias"`
|
||||
Normalised string `bson:"normalised" json:"normalised"`
|
||||
// CandidateEntityID is set when the alias points at another existing entity.
|
||||
CandidateEntityID string `bson:"candidate_entity_id,omitempty" json:"candidate_entity_id,omitempty"`
|
||||
|
||||
Status AliasStatus `bson:"status" json:"status"`
|
||||
Method ExtractionMethod `bson:"method" json:"method"`
|
||||
Similarity float64 `bson:"similarity" json:"similarity"`
|
||||
Reason string `bson:"reason,omitempty" json:"reason,omitempty"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
}
|
||||
|
||||
// ClaimNature distinguishes how faithfully a claim reproduces its source.
|
||||
type ClaimNature string
|
||||
|
||||
const (
|
||||
ClaimQuoted ClaimNature = "quoted"
|
||||
ClaimParaphrased ClaimNature = "paraphrased"
|
||||
ClaimInferred ClaimNature = "inferred"
|
||||
)
|
||||
|
||||
// Valid reports whether n is a recognised claim nature.
|
||||
func (n ClaimNature) Valid() bool {
|
||||
switch n {
|
||||
case ClaimQuoted, ClaimParaphrased, ClaimInferred:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Claim is a statement made by a source item. A claim is not a fact: it is a
|
||||
// record that some source said something, and it always retains the exact text.
|
||||
type Claim struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
ItemID string `bson:"item_id" json:"item_id"`
|
||||
ChunkID string `bson:"chunk_id,omitempty" json:"chunk_id,omitempty"`
|
||||
|
||||
// Statement is a short normalised rendering; SourceRef.Quote is the original.
|
||||
Statement string `bson:"statement" json:"statement"`
|
||||
SourceRef SourceRef `bson:"source_ref" json:"source_ref"`
|
||||
|
||||
// Attribution is who the source says is making the claim, where known.
|
||||
Attribution string `bson:"attribution,omitempty" json:"attribution,omitempty"`
|
||||
Nature ClaimNature `bson:"nature" json:"nature"`
|
||||
|
||||
// Subject and Predicate are coarse normalised keys used to find claims that
|
||||
// talk about the same thing, which is how support/contradiction candidates
|
||||
// are located without comparing every claim to every other claim.
|
||||
Subject string `bson:"subject,omitempty" json:"subject,omitempty"`
|
||||
Predicate string `bson:"predicate,omitempty" json:"predicate,omitempty"`
|
||||
// Numbers and Dates hold normalised values mentioned in the claim, used to
|
||||
// detect disagreement about the same quantity or date.
|
||||
Numbers []float64 `bson:"numbers,omitempty" json:"numbers,omitempty"`
|
||||
Dates []string `bson:"dates,omitempty" json:"dates,omitempty"`
|
||||
// Polarity is +1 for affirmative, -1 for negated statements.
|
||||
Polarity int `bson:"polarity" json:"polarity"`
|
||||
|
||||
Method ExtractionMethod `bson:"method" json:"method"`
|
||||
ModelVersion string `bson:"model_version,omitempty" json:"model_version,omitempty"`
|
||||
Confidence float64 `bson:"confidence" json:"confidence"`
|
||||
DedupeKey string `bson:"dedupe_key" json:"-"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
Deleted bool `bson:"deleted" json:"deleted"`
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
// Package domain holds Relata's core types and the port interfaces the service
|
||||
// layer depends on. It imports only the standard library: no MongoDB, Qdrant,
|
||||
// Ollama or HTTP types may appear here.
|
||||
package domain
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SourceType classifies where an item came from.
|
||||
type SourceType string
|
||||
|
||||
const (
|
||||
SourceTypeNote SourceType = "note"
|
||||
SourceTypeDocument SourceType = "document"
|
||||
SourceTypeJSON SourceType = "json"
|
||||
SourceTypeEmail SourceType = "email"
|
||||
SourceTypeTranscript SourceType = "transcript"
|
||||
SourceTypeMarkdown SourceType = "markdown"
|
||||
)
|
||||
|
||||
// ValidSourceTypes enumerates every accepted source type.
|
||||
var ValidSourceTypes = []SourceType{
|
||||
SourceTypeNote, SourceTypeDocument, SourceTypeJSON,
|
||||
SourceTypeEmail, SourceTypeTranscript, SourceTypeMarkdown,
|
||||
}
|
||||
|
||||
// Valid reports whether s is a recognised source type.
|
||||
func (s SourceType) Valid() bool {
|
||||
for _, v := range ValidSourceTypes {
|
||||
if v == s {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProcessingStatus tracks an item's progress through the pipeline.
|
||||
type ProcessingStatus string
|
||||
|
||||
const (
|
||||
StatusPending ProcessingStatus = "pending"
|
||||
StatusParsing ProcessingStatus = "parsing"
|
||||
StatusChunking ProcessingStatus = "chunking"
|
||||
StatusEmbedding ProcessingStatus = "embedding"
|
||||
StatusAnalysing ProcessingStatus = "analysing"
|
||||
StatusRelating ProcessingStatus = "relating"
|
||||
StatusReady ProcessingStatus = "ready"
|
||||
StatusFailed ProcessingStatus = "failed"
|
||||
StatusPartial ProcessingStatus = "partial"
|
||||
StatusDeleting ProcessingStatus = "deleting"
|
||||
StatusTombstoned ProcessingStatus = "tombstoned"
|
||||
)
|
||||
|
||||
// ParseWarning records a non-fatal problem encountered while extracting text.
|
||||
// Warnings are retained on the item so that a degraded extraction is visible
|
||||
// rather than silently accepted.
|
||||
type ParseWarning struct {
|
||||
Code string `bson:"code" json:"code"`
|
||||
Message string `bson:"message" json:"message"`
|
||||
Detail string `bson:"detail,omitempty" json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Well-known parse warning codes.
|
||||
const (
|
||||
WarnLowExtractableText = "low_extractable_text"
|
||||
WarnParseFailed = "parse_failed"
|
||||
WarnTruncatedText = "truncated_text"
|
||||
WarnTruncatedChunks = "truncated_chunks"
|
||||
WarnJSONDepthExceeded = "json_depth_exceeded"
|
||||
WarnEncodingRepaired = "encoding_repaired"
|
||||
WarnNoPageInfo = "no_page_info"
|
||||
)
|
||||
|
||||
// Item is a complete piece of original information supplied by a user.
|
||||
// The original content is never rewritten; everything else is derived.
|
||||
type Item struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
Title string `bson:"title" json:"title"`
|
||||
SourceType SourceType `bson:"source_type" json:"source_type"`
|
||||
MIMEType string `bson:"mime_type" json:"mime_type"`
|
||||
|
||||
// OriginalFilename is the sanitised upload name; OriginalStoredPath points at
|
||||
// the preserved bytes on disk for binary uploads (PDF, DOCX).
|
||||
OriginalFilename string `bson:"original_filename,omitempty" json:"original_filename,omitempty"`
|
||||
OriginalStoredPath string `bson:"original_stored_path,omitempty" json:"-"`
|
||||
// OriginalText holds the verbatim user-supplied text for text-shaped inputs.
|
||||
OriginalText string `bson:"original_text,omitempty" json:"original_text,omitempty"`
|
||||
SizeBytes int64 `bson:"size_bytes" json:"size_bytes"`
|
||||
|
||||
// ExtractedText is the parser's output. For plain text it equals OriginalText.
|
||||
ExtractedText string `bson:"extracted_text,omitempty" json:"extracted_text,omitempty"`
|
||||
|
||||
ContentHash string `bson:"content_hash" json:"content_hash"`
|
||||
Metadata map[string]string `bson:"metadata,omitempty" json:"metadata,omitempty"`
|
||||
|
||||
// CreatedAt is the user-asserted creation time of the information itself
|
||||
// (may predate ingestion); IngestedAt is when Relata received it.
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
IngestedAt time.Time `bson:"ingested_at" json:"ingested_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
|
||||
Status ProcessingStatus `bson:"status" json:"status"`
|
||||
Warnings []ParseWarning `bson:"warnings,omitempty" json:"warnings,omitempty"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
|
||||
ParserVersion string `bson:"parser_version,omitempty" json:"parser_version,omitempty"`
|
||||
EmbeddingModel string `bson:"embedding_model,omitempty" json:"embedding_model,omitempty"`
|
||||
AnalysisModel string `bson:"analysis_model,omitempty" json:"analysis_model,omitempty"`
|
||||
ScoreVersion string `bson:"score_version,omitempty" json:"score_version,omitempty"`
|
||||
ChunkCount int `bson:"chunk_count" json:"chunk_count"`
|
||||
EntityCount int `bson:"entity_count" json:"entity_count"`
|
||||
ClaimCount int `bson:"claim_count" json:"claim_count"`
|
||||
RelationshipSeen int `bson:"relationship_count" json:"relationship_count"`
|
||||
|
||||
// DuplicateOf is set when the item was knowingly kept alongside an identical
|
||||
// existing item. It is informational; the duplicate is still fully processed.
|
||||
// The bson tag deliberately omits `omitempty`: the uniqueness index that
|
||||
// enforces duplicate detection is a partial index keyed on this field being
|
||||
// the empty string, and an absent field would not match it.
|
||||
DuplicateOf string `bson:"duplicate_of" json:"duplicate_of,omitempty"`
|
||||
|
||||
CreatedBy string `bson:"created_by,omitempty" json:"created_by,omitempty"`
|
||||
Deleted bool `bson:"deleted" json:"deleted"`
|
||||
}
|
||||
|
||||
// HasWarning reports whether the item carries a warning with the given code.
|
||||
func (i *Item) HasWarning(code string) bool {
|
||||
for _, w := range i.Warnings {
|
||||
if w.Code == code {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Chunk is a retrievable section of an item.
|
||||
type Chunk struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
ItemID string `bson:"item_id" json:"item_id"`
|
||||
|
||||
Sequence int `bson:"sequence" json:"sequence"`
|
||||
Text string `bson:"text" json:"text"`
|
||||
ContentHash string `bson:"content_hash" json:"content_hash"`
|
||||
|
||||
StartOffset int `bson:"start_offset" json:"start_offset"`
|
||||
EndOffset int `bson:"end_offset" json:"end_offset"`
|
||||
|
||||
// Page is 1-based and only set for PDF-derived chunks.
|
||||
Page int `bson:"page,omitempty" json:"page,omitempty"`
|
||||
Section string `bson:"section,omitempty" json:"section,omitempty"`
|
||||
Heading string `bson:"heading,omitempty" json:"heading,omitempty"`
|
||||
|
||||
EmbeddingModel string `bson:"embedding_model,omitempty" json:"embedding_model,omitempty"`
|
||||
Embedded bool `bson:"embedded" json:"embedded"`
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
}
|
||||
|
||||
// Errors returned by repositories and services.
|
||||
var (
|
||||
ErrNotFound = errors.New("not found")
|
||||
ErrDuplicate = errors.New("duplicate content")
|
||||
ErrForbidden = errors.New("forbidden")
|
||||
ErrInvalidInput = errors.New("invalid input")
|
||||
ErrTooLarge = errors.New("payload too large")
|
||||
ErrUnsupported = errors.New("unsupported content type")
|
||||
ErrConflict = errors.New("conflict")
|
||||
ErrRateLimited = errors.New("rate limited")
|
||||
ErrModelInvalid = errors.New("model returned invalid output")
|
||||
ErrUnavailable = errors.New("dependency unavailable")
|
||||
ErrWorkspaceMiss = errors.New("workspace mismatch")
|
||||
)
|
||||
@@ -0,0 +1,109 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// JobType names a unit of background work.
|
||||
type JobType string
|
||||
|
||||
const (
|
||||
JobParseItem JobType = "parse_item"
|
||||
JobChunkItem JobType = "chunk_item"
|
||||
JobEmbedItem JobType = "generate_embeddings"
|
||||
JobExtractEntity JobType = "extract_entities"
|
||||
JobExtractClaims JobType = "extract_claims"
|
||||
JobDiscoverRels JobType = "discover_relationships"
|
||||
JobReprocessItem JobType = "reprocess_item"
|
||||
JobDeleteItemData JobType = "delete_item_data"
|
||||
JobReembedWS JobType = "reembed_workspace"
|
||||
)
|
||||
|
||||
// ValidJobTypes is the closed set of job types the workers understand.
|
||||
var ValidJobTypes = []JobType{
|
||||
JobParseItem, JobChunkItem, JobEmbedItem, JobExtractEntity, JobExtractClaims,
|
||||
JobDiscoverRels, JobReprocessItem, JobDeleteItemData, JobReembedWS,
|
||||
}
|
||||
|
||||
// Valid reports whether t is a recognised job type.
|
||||
func (t JobType) Valid() bool {
|
||||
for _, v := range ValidJobTypes {
|
||||
if v == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// JobStatus is a job's lifecycle state.
|
||||
type JobStatus string
|
||||
|
||||
const (
|
||||
JobPending JobStatus = "pending"
|
||||
JobRunning JobStatus = "running"
|
||||
JobSucceeded JobStatus = "succeeded"
|
||||
JobFailed JobStatus = "failed"
|
||||
JobCancelled JobStatus = "cancelled"
|
||||
)
|
||||
|
||||
// JobAttempt records one execution attempt and its outcome.
|
||||
type JobAttempt struct {
|
||||
Number int `bson:"number" json:"number"`
|
||||
StartedAt time.Time `bson:"started_at" json:"started_at"`
|
||||
EndedAt time.Time `bson:"ended_at,omitempty" json:"ended_at,omitempty"`
|
||||
Worker string `bson:"worker,omitempty" json:"worker,omitempty"`
|
||||
Error string `bson:"error,omitempty" json:"error,omitempty"`
|
||||
// ErrorClass groups failures for metrics: transient, permanent, timeout, cancelled.
|
||||
ErrorClass string `bson:"error_class,omitempty" json:"error_class,omitempty"`
|
||||
DurationMS int64 `bson:"duration_ms" json:"duration_ms"`
|
||||
}
|
||||
|
||||
// Job is a persistent, leased unit of background work.
|
||||
//
|
||||
// Leases (not locks) are used so that a crashed worker's job becomes runnable
|
||||
// again once its lease expires, without any external coordination.
|
||||
type Job struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
Type JobType `bson:"type" json:"type"`
|
||||
Status JobStatus `bson:"status" json:"status"`
|
||||
|
||||
// ItemID is the primary subject for item-scoped jobs.
|
||||
ItemID string `bson:"item_id,omitempty" json:"item_id,omitempty"`
|
||||
// Payload carries small typed job parameters. It never holds document bodies.
|
||||
Payload map[string]string `bson:"payload,omitempty" json:"payload,omitempty"`
|
||||
|
||||
Priority int `bson:"priority" json:"priority"`
|
||||
RunAfter time.Time `bson:"run_after" json:"run_after"`
|
||||
Attempts int `bson:"attempts" json:"attempts"`
|
||||
MaxAttempts int `bson:"max_attempts" json:"max_attempts"`
|
||||
|
||||
LeaseOwner string `bson:"lease_owner,omitempty" json:"lease_owner,omitempty"`
|
||||
LeaseExpires *time.Time `bson:"lease_expires,omitempty" json:"lease_expires,omitempty"`
|
||||
|
||||
History []JobAttempt `bson:"history,omitempty" json:"history,omitempty"`
|
||||
LastError string `bson:"last_error,omitempty" json:"last_error,omitempty"`
|
||||
|
||||
// IdempotencyKey prevents duplicate enqueues of the same pending work.
|
||||
IdempotencyKey string `bson:"idempotency_key,omitempty" json:"-"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
CompletedAt *time.Time `bson:"completed_at,omitempty" json:"completed_at,omitempty"`
|
||||
RunID string `bson:"run_id,omitempty" json:"run_id,omitempty"`
|
||||
}
|
||||
|
||||
// Terminal reports whether the job will not run again without manual action.
|
||||
func (j *Job) Terminal() bool {
|
||||
return j.Status == JobSucceeded || j.Status == JobFailed || j.Status == JobCancelled
|
||||
}
|
||||
|
||||
// ErrorClass categorises a failure for retry decisions and metrics.
|
||||
type ErrorClass string
|
||||
|
||||
const (
|
||||
ErrClassTransient ErrorClass = "transient"
|
||||
ErrClassPermanent ErrorClass = "permanent"
|
||||
ErrClassTimeout ErrorClass = "timeout"
|
||||
ErrClassCancelled ErrorClass = "cancelled"
|
||||
ErrClassInvalid ErrorClass = "invalid_model_output"
|
||||
)
|
||||
@@ -0,0 +1,389 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// AI provider ports
|
||||
//
|
||||
// These exist so that Ollama can be replaced by an OpenAI-compatible API or any
|
||||
// other provider without touching the service layer. Adapters live in
|
||||
// internal/embeddings and internal/analysis.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Embedder turns text into vectors.
|
||||
type Embedder interface {
|
||||
// Embed returns one vector per input text, in the same order. Implementations
|
||||
// must honour ctx cancellation and must not retry indefinitely.
|
||||
Embed(ctx context.Context, texts []string) ([][]float32, error)
|
||||
// ModelID identifies the exact model, for stamping onto derived records.
|
||||
ModelID() string
|
||||
// Dimensions is the vector width this model produces.
|
||||
Dimensions() int
|
||||
}
|
||||
|
||||
// ChatMessage is one turn of a model conversation.
|
||||
type ChatMessage struct {
|
||||
Role string // "system" | "user" | "assistant"
|
||||
Content string
|
||||
}
|
||||
|
||||
// CompletionRequest asks the model for a bounded response.
|
||||
type CompletionRequest struct {
|
||||
Messages []ChatMessage
|
||||
// JSONSchema, when non-empty, requests structured output conforming to it.
|
||||
JSONSchema []byte
|
||||
// MaxTokens bounds the response so a runaway generation cannot exhaust
|
||||
// memory or time.
|
||||
MaxTokens int
|
||||
Temperature float64
|
||||
}
|
||||
|
||||
// CompletionResponse is a model reply plus what it cost.
|
||||
type CompletionResponse struct {
|
||||
Content string
|
||||
Model string
|
||||
PromptTokens int
|
||||
OutputTokens int
|
||||
Duration time.Duration
|
||||
}
|
||||
|
||||
// Analyser performs model-assisted structured tasks. Its output is always
|
||||
// treated as untrusted and validated against a schema before use.
|
||||
type Analyser interface {
|
||||
Complete(ctx context.Context, req CompletionRequest) (*CompletionResponse, error)
|
||||
ModelID() string
|
||||
}
|
||||
|
||||
// ProviderHealth describes whether an AI provider is reachable and configured.
|
||||
type ProviderHealth struct {
|
||||
Name string `json:"name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Available bool `json:"available"`
|
||||
Detail string `json:"detail,omitempty"`
|
||||
Models []string `json:"models,omitempty"`
|
||||
}
|
||||
|
||||
// HealthChecker is implemented by providers that can report reachability.
|
||||
type HealthChecker interface {
|
||||
Health(ctx context.Context) ProviderHealth
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Vector store port
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// VectorPoint is a stored embedding plus the minimal retrieval metadata needed
|
||||
// to resolve it back to MongoDB. Payloads deliberately exclude document text.
|
||||
type VectorPoint struct {
|
||||
ChunkID string
|
||||
WorkspaceID string
|
||||
ItemID string
|
||||
SourceType SourceType
|
||||
CreatedAt time.Time
|
||||
EmbeddingModel string
|
||||
Vector []float32
|
||||
}
|
||||
|
||||
// VectorMatch is a similarity search hit.
|
||||
type VectorMatch struct {
|
||||
ChunkID string
|
||||
ItemID string
|
||||
WorkspaceID string
|
||||
// Score is cosine similarity in roughly -1..1; higher is more similar.
|
||||
Score float64
|
||||
SourceType SourceType
|
||||
CreatedAt time.Time
|
||||
EmbeddingModel string
|
||||
}
|
||||
|
||||
// VectorFilter narrows a similarity search. All fields are optional and are
|
||||
// combined with AND. WorkspaceID is always applied by the caller.
|
||||
type VectorFilter struct {
|
||||
WorkspaceID string
|
||||
ItemIDs []string
|
||||
ExcludeItemIDs []string
|
||||
SourceTypes []SourceType
|
||||
EmbeddingModel string
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
}
|
||||
|
||||
// VectorStore persists and searches embeddings.
|
||||
type VectorStore interface {
|
||||
// EnsureCollection creates the collection if absent, sized for dim.
|
||||
EnsureCollection(ctx context.Context, dim int) error
|
||||
Upsert(ctx context.Context, points []VectorPoint) error
|
||||
Search(ctx context.Context, vector []float32, limit int, f VectorFilter) ([]VectorMatch, error)
|
||||
DeleteByItem(ctx context.Context, workspaceID, itemID string) error
|
||||
DeleteByChunks(ctx context.Context, workspaceID string, chunkIDs []string) error
|
||||
// CountByWorkspace reports how many vectors exist, for reconciliation.
|
||||
CountByWorkspace(ctx context.Context, workspaceID string) (int64, error)
|
||||
Health(ctx context.Context) error
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Repository ports
|
||||
//
|
||||
// Every method that touches user content takes a workspace ID and must include
|
||||
// it in the underlying query. There is intentionally no "get by ID" that skips
|
||||
// the workspace filter.
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Page describes a bounded slice of results.
|
||||
type Page struct {
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// Normalise clamps a page request to safe bounds.
|
||||
func (p Page) Normalise(defLimit, maxLimit int) Page {
|
||||
if p.Limit <= 0 {
|
||||
p.Limit = defLimit
|
||||
}
|
||||
if p.Limit > maxLimit {
|
||||
p.Limit = maxLimit
|
||||
}
|
||||
if p.Offset < 0 {
|
||||
p.Offset = 0
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// ItemFilter selects items using typed values only. Client-supplied MongoDB
|
||||
// operators are never accepted anywhere in this struct.
|
||||
type ItemFilter struct {
|
||||
SourceTypes []SourceType
|
||||
MIMETypes []string
|
||||
Statuses []ProcessingStatus
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
ContentHash string
|
||||
TitleContains string
|
||||
IncludeDeleted bool
|
||||
}
|
||||
|
||||
// ItemRepository stores source items.
|
||||
type ItemRepository interface {
|
||||
Create(ctx context.Context, it *Item) error
|
||||
Get(ctx context.Context, workspaceID, id string) (*Item, error)
|
||||
GetMany(ctx context.Context, workspaceID string, ids []string) ([]*Item, error)
|
||||
Update(ctx context.Context, it *Item) error
|
||||
SetStatus(ctx context.Context, workspaceID, id string, st ProcessingStatus, errMsg string) error
|
||||
AddWarning(ctx context.Context, workspaceID, id string, w ParseWarning) error
|
||||
List(ctx context.Context, workspaceID string, f ItemFilter, p Page) ([]*Item, int64, error)
|
||||
FindByContentHash(ctx context.Context, workspaceID, hash string) (*Item, error)
|
||||
// TextSearch is the lexical half of hybrid retrieval.
|
||||
TextSearch(ctx context.Context, workspaceID, query string, f ItemFilter, limit int) ([]LexicalHit, error)
|
||||
Tombstone(ctx context.Context, workspaceID, id string) error
|
||||
HardDelete(ctx context.Context, workspaceID, id string) error
|
||||
Count(ctx context.Context, workspaceID string) (int64, error)
|
||||
CountByStatus(ctx context.Context, workspaceID string) (map[ProcessingStatus]int64, error)
|
||||
}
|
||||
|
||||
// LexicalHit is a keyword-search result with its raw text score.
|
||||
type LexicalHit struct {
|
||||
ItemID string
|
||||
Score float64
|
||||
}
|
||||
|
||||
// ChunkRepository stores item chunks.
|
||||
type ChunkRepository interface {
|
||||
ReplaceForItem(ctx context.Context, workspaceID, itemID string, chunks []*Chunk) error
|
||||
ListByItem(ctx context.Context, workspaceID, itemID string) ([]*Chunk, error)
|
||||
GetMany(ctx context.Context, workspaceID string, ids []string) ([]*Chunk, error)
|
||||
MarkEmbedded(ctx context.Context, workspaceID string, ids []string, model string) error
|
||||
ListUnembedded(ctx context.Context, workspaceID, model string, limit int) ([]*Chunk, error)
|
||||
DeleteByItem(ctx context.Context, workspaceID, itemID string) error
|
||||
Count(ctx context.Context, workspaceID string) (int64, error)
|
||||
// TextSearch finds chunks lexically, used to produce excerpts.
|
||||
TextSearch(ctx context.Context, workspaceID, query string, limit int) ([]ChunkHit, error)
|
||||
}
|
||||
|
||||
// ChunkHit is a lexical chunk match.
|
||||
type ChunkHit struct {
|
||||
ChunkID string
|
||||
ItemID string
|
||||
Score float64
|
||||
Text string
|
||||
}
|
||||
|
||||
// EntityFilter selects entities by typed values.
|
||||
type EntityFilter struct {
|
||||
Types []EntityType
|
||||
NameContains string
|
||||
NormalisedKey string
|
||||
ItemID string
|
||||
MinMentions int
|
||||
}
|
||||
|
||||
// EntityRepository stores extracted entities.
|
||||
type EntityRepository interface {
|
||||
// UpsertByKey merges a newly observed entity into an existing one with the
|
||||
// same (workspace, type, normalised key), accumulating references. It never
|
||||
// merges entities whose keys merely look similar.
|
||||
UpsertByKey(ctx context.Context, e *Entity) (*Entity, error)
|
||||
Get(ctx context.Context, workspaceID, id string) (*Entity, error)
|
||||
GetMany(ctx context.Context, workspaceID string, ids []string) ([]*Entity, error)
|
||||
FindByNormalised(ctx context.Context, workspaceID string, keys []string) ([]*Entity, error)
|
||||
List(ctx context.Context, workspaceID string, f EntityFilter, p Page) ([]*Entity, int64, error)
|
||||
ListByItem(ctx context.Context, workspaceID, itemID string) ([]*Entity, error)
|
||||
// ItemsSharingEntities returns item IDs that reference any of the given
|
||||
// entities, excluding excludeItem. This is the indexed path that keeps
|
||||
// candidate discovery away from all-pairs comparison.
|
||||
ItemsSharingEntities(ctx context.Context, workspaceID string, entityIDs []string, excludeItem string, limit int) ([]string, error)
|
||||
RemoveItemRefs(ctx context.Context, workspaceID, itemID string) (orphaned []string, err error)
|
||||
DeleteMany(ctx context.Context, workspaceID string, ids []string) error
|
||||
Count(ctx context.Context, workspaceID string) (int64, error)
|
||||
}
|
||||
|
||||
// AliasRepository stores candidate and confirmed entity aliases.
|
||||
type AliasRepository interface {
|
||||
Upsert(ctx context.Context, a *EntityAlias) error
|
||||
ListByEntity(ctx context.Context, workspaceID, entityID string) ([]*EntityAlias, error)
|
||||
List(ctx context.Context, workspaceID string, status AliasStatus, p Page) ([]*EntityAlias, int64, error)
|
||||
SetStatus(ctx context.Context, workspaceID, id string, st AliasStatus) error
|
||||
DeleteByEntity(ctx context.Context, workspaceID, entityID string) error
|
||||
}
|
||||
|
||||
// ClaimRepository stores extracted claims.
|
||||
type ClaimRepository interface {
|
||||
ReplaceForItem(ctx context.Context, workspaceID, itemID string, claims []*Claim) error
|
||||
ListByItem(ctx context.Context, workspaceID, itemID string) ([]*Claim, error)
|
||||
// FindBySubjects locates claims about the same coarse subject, which is how
|
||||
// support/contradiction candidates are found cheaply.
|
||||
FindBySubjects(ctx context.Context, workspaceID string, subjects []string, excludeItem string, limit int) ([]*Claim, error)
|
||||
GetMany(ctx context.Context, workspaceID string, ids []string) ([]*Claim, error)
|
||||
DeleteByItem(ctx context.Context, workspaceID, itemID string) error
|
||||
Count(ctx context.Context, workspaceID string) (int64, error)
|
||||
}
|
||||
|
||||
// RelationshipFilter selects relationships by typed values.
|
||||
type RelationshipFilter struct {
|
||||
Types []RelationshipType
|
||||
Provenances []Provenance
|
||||
ReviewStatus []ReviewStatus
|
||||
MinScore float64
|
||||
NodeID string
|
||||
NodeKind NodeKind
|
||||
Direction string // "out" | "in" | "both"
|
||||
ConfirmedOnly bool
|
||||
}
|
||||
|
||||
// RelationshipRepository stores relationships and their review history.
|
||||
type RelationshipRepository interface {
|
||||
// UpsertByDedupeKey inserts or refreshes a relationship without creating a
|
||||
// duplicate on reprocessing. It never overwrites a human review decision.
|
||||
UpsertByDedupeKey(ctx context.Context, r *Relationship) (*Relationship, error)
|
||||
|
||||
// FindByDedupeKey returns the relationship with this dedupe key, or
|
||||
// ErrNotFound. It exists so callers can locate an existing link without
|
||||
// paging through a node's relationships — a well-connected item can have
|
||||
// more than one page of them, and a paged scan would miss the match and
|
||||
// then collide with the unique index.
|
||||
FindByDedupeKey(ctx context.Context, workspaceID, dedupeKey string) (*Relationship, error)
|
||||
Create(ctx context.Context, r *Relationship) error
|
||||
Get(ctx context.Context, workspaceID, id string) (*Relationship, error)
|
||||
List(ctx context.Context, workspaceID string, f RelationshipFilter, p Page) ([]*Relationship, int64, error)
|
||||
ListForNode(ctx context.Context, workspaceID string, kind NodeKind, id string, f RelationshipFilter) (out, in []*Relationship, err error)
|
||||
// ListForItems fetches relationships touching any of the given items, used
|
||||
// for relationship expansion during search.
|
||||
ListForItems(ctx context.Context, workspaceID string, itemIDs []string, minScore float64) ([]*Relationship, error)
|
||||
Update(ctx context.Context, r *Relationship) error
|
||||
AppendReview(ctx context.Context, workspaceID, id string, ev ReviewEvent, newStatus ReviewStatus, newType RelationshipType, newProv Provenance) (*Relationship, error)
|
||||
TombstoneForItem(ctx context.Context, workspaceID, itemID string) (int64, error)
|
||||
DeleteForItem(ctx context.Context, workspaceID, itemID string) (int64, error)
|
||||
Count(ctx context.Context, workspaceID string) (int64, error)
|
||||
CountPendingReview(ctx context.Context, workspaceID string) (int64, error)
|
||||
}
|
||||
|
||||
// JobRepository is the persistent queue.
|
||||
type JobRepository interface {
|
||||
Enqueue(ctx context.Context, j *Job) error
|
||||
// Claim atomically leases the next runnable job for owner.
|
||||
Claim(ctx context.Context, owner string, types []JobType, lease time.Duration) (*Job, error)
|
||||
// Heartbeat extends a lease held by owner; it fails if the lease was lost.
|
||||
Heartbeat(ctx context.Context, id, owner string, lease time.Duration) error
|
||||
Complete(ctx context.Context, id, owner string, att JobAttempt) error
|
||||
Fail(ctx context.Context, id, owner string, att JobAttempt, retryAfter *time.Time) error
|
||||
Get(ctx context.Context, id string) (*Job, error)
|
||||
List(ctx context.Context, workspaceID string, statuses []JobStatus, types []JobType, p Page) ([]*Job, int64, error)
|
||||
Retry(ctx context.Context, workspaceID, id string) (*Job, error)
|
||||
CountByStatus(ctx context.Context, workspaceID string) (map[JobStatus]int64, error)
|
||||
// ReleaseExpired returns jobs whose lease elapsed to the pending pool.
|
||||
ReleaseExpired(ctx context.Context, now time.Time) (int64, error)
|
||||
DeleteForItem(ctx context.Context, workspaceID, itemID string) (int64, error)
|
||||
}
|
||||
|
||||
// AuditRepository is append-only.
|
||||
type AuditRepository interface {
|
||||
Record(ctx context.Context, ev *AuditEvent) error
|
||||
List(ctx context.Context, workspaceID string, p Page) ([]*AuditEvent, int64, error)
|
||||
}
|
||||
|
||||
// RunRepository stores processing run history.
|
||||
type RunRepository interface {
|
||||
Create(ctx context.Context, r *ProcessingRun) error
|
||||
Finish(ctx context.Context, id, outcome, errMsg string, stats map[string]int) error
|
||||
ListByItem(ctx context.Context, workspaceID, itemID string, limit int) ([]*ProcessingRun, error)
|
||||
}
|
||||
|
||||
// ModelVersionRepository tracks which models have produced stored data.
|
||||
type ModelVersionRepository interface {
|
||||
Observe(ctx context.Context, workspaceID, kind, name string, dims int) error
|
||||
List(ctx context.Context, workspaceID string) ([]*ModelVersion, error)
|
||||
}
|
||||
|
||||
// UserRepository stores local accounts.
|
||||
type UserRepository interface {
|
||||
Create(ctx context.Context, u *User) error
|
||||
GetByUsername(ctx context.Context, username string) (*User, error)
|
||||
GetByID(ctx context.Context, id string) (*User, error)
|
||||
Update(ctx context.Context, u *User) error
|
||||
Count(ctx context.Context) (int64, error)
|
||||
|
||||
// GetByOAuthSubject finds a federated account by provider and subject, the
|
||||
// only pair a provider promises not to reassign. Returns ErrNotFound when
|
||||
// the person has not signed in before.
|
||||
GetByOAuthSubject(ctx context.Context, provider, subject string) (*User, error)
|
||||
}
|
||||
|
||||
// APITokenRepository stores non-interactive API credentials. Only hashes are
|
||||
// held; there is deliberately no method that returns a token's secret.
|
||||
type APITokenRepository interface {
|
||||
Create(ctx context.Context, t *APIToken) error
|
||||
// GetByHash finds a token by the SHA-256 of its secret. The lookup is by
|
||||
// the full hash, so a partial match cannot narrow the search space.
|
||||
GetByHash(ctx context.Context, hash string) (*APIToken, error)
|
||||
// ListForUser returns a person's tokens, newest first, including revoked
|
||||
// and expired ones so the history stays visible.
|
||||
ListForUser(ctx context.Context, workspaceID, userID string) ([]*APIToken, error)
|
||||
Get(ctx context.Context, workspaceID, id string) (*APIToken, error)
|
||||
Revoke(ctx context.Context, workspaceID, id, revokedBy string, at time.Time) error
|
||||
// TouchLastUsed records use. Callers throttle this so a busy token does not
|
||||
// turn every read into a write.
|
||||
TouchLastUsed(ctx context.Context, id string, at time.Time) error
|
||||
// DeleteForUser removes a user's tokens, used when an account is removed.
|
||||
DeleteForUser(ctx context.Context, workspaceID, userID string) (int64, error)
|
||||
}
|
||||
|
||||
// SessionRepository stores browser sessions.
|
||||
type SessionRepository interface {
|
||||
Create(ctx context.Context, s *Session) error
|
||||
GetByTokenHash(ctx context.Context, hash string) (*Session, error)
|
||||
Touch(ctx context.Context, id string, lastSeen, expires time.Time) error
|
||||
Revoke(ctx context.Context, id string) error
|
||||
RevokeAllForUser(ctx context.Context, userID string) error
|
||||
DeleteExpired(ctx context.Context, before time.Time) (int64, error)
|
||||
}
|
||||
|
||||
// WorkspaceRepository stores workspaces.
|
||||
type WorkspaceRepository interface {
|
||||
Create(ctx context.Context, w *Workspace) error
|
||||
Get(ctx context.Context, id string) (*Workspace, error)
|
||||
GetBySlug(ctx context.Context, slug string) (*Workspace, error)
|
||||
List(ctx context.Context) ([]*Workspace, error)
|
||||
}
|
||||
@@ -0,0 +1,239 @@
|
||||
package domain
|
||||
|
||||
import "time"
|
||||
|
||||
// RelationshipType is the semantic kind of connection between two nodes.
|
||||
type RelationshipType string
|
||||
|
||||
const (
|
||||
RelSimilarTo RelationshipType = "similar_to"
|
||||
RelMentions RelationshipType = "mentions"
|
||||
RelInvolves RelationshipType = "involves"
|
||||
RelSupports RelationshipType = "supports"
|
||||
RelContradicts RelationshipType = "contradicts"
|
||||
RelUpdates RelationshipType = "updates"
|
||||
RelSupersedes RelationshipType = "supersedes"
|
||||
RelProvidesContextFor RelationshipType = "provides_context_for"
|
||||
RelPossiblyExplains RelationshipType = "possibly_explains"
|
||||
RelOccurredBefore RelationshipType = "occurred_before"
|
||||
RelOccurredAfter RelationshipType = "occurred_after"
|
||||
RelDerivedFrom RelationshipType = "derived_from"
|
||||
RelSameEntityAs RelationshipType = "same_entity_as"
|
||||
)
|
||||
|
||||
// ValidRelationshipTypes is the closed set of accepted relationship types.
|
||||
// Model output proposing anything outside this set is rejected.
|
||||
var ValidRelationshipTypes = []RelationshipType{
|
||||
RelSimilarTo, RelMentions, RelInvolves, RelSupports, RelContradicts,
|
||||
RelUpdates, RelSupersedes, RelProvidesContextFor, RelPossiblyExplains,
|
||||
RelOccurredBefore, RelOccurredAfter, RelDerivedFrom, RelSameEntityAs,
|
||||
}
|
||||
|
||||
// Valid reports whether t is a recognised relationship type.
|
||||
func (t RelationshipType) Valid() bool {
|
||||
for _, v := range ValidRelationshipTypes {
|
||||
if v == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Directed reports whether the type implies an ordered source -> target reading.
|
||||
// Non-directed types are symmetric and are stored with a canonical ordering.
|
||||
func (t RelationshipType) Directed() bool {
|
||||
switch t {
|
||||
case RelSimilarTo, RelSameEntityAs:
|
||||
return false
|
||||
default:
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
// Inverse returns the type that describes the same connection read from the
|
||||
// target's point of view, and whether such an inverse exists.
|
||||
func (t RelationshipType) Inverse() (RelationshipType, bool) {
|
||||
switch t {
|
||||
case RelOccurredBefore:
|
||||
return RelOccurredAfter, true
|
||||
case RelOccurredAfter:
|
||||
return RelOccurredBefore, true
|
||||
case RelSimilarTo:
|
||||
return RelSimilarTo, true
|
||||
case RelSameEntityAs:
|
||||
return RelSameEntityAs, true
|
||||
}
|
||||
return "", false
|
||||
}
|
||||
|
||||
// Provenance records how a relationship came to exist. Model-inferred
|
||||
// relationships are proposals, not facts, and must be labelled as such.
|
||||
type Provenance string
|
||||
|
||||
const (
|
||||
ProvExplicit Provenance = "explicit"
|
||||
ProvDeterministic Provenance = "deterministic"
|
||||
ProvModelInferred Provenance = "model_inferred"
|
||||
ProvUserConfirmed Provenance = "user_confirmed"
|
||||
ProvUserRejected Provenance = "user_rejected"
|
||||
)
|
||||
|
||||
// ValidProvenances is the closed set of provenance classifications.
|
||||
var ValidProvenances = []Provenance{
|
||||
ProvExplicit, ProvDeterministic, ProvModelInferred, ProvUserConfirmed, ProvUserRejected,
|
||||
}
|
||||
|
||||
// Valid reports whether p is a recognised provenance.
|
||||
func (p Provenance) Valid() bool {
|
||||
for _, v := range ValidProvenances {
|
||||
if v == p {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Asserted reports whether the provenance represents something a human or a
|
||||
// deterministic rule stated, as opposed to a model's guess.
|
||||
func (p Provenance) Asserted() bool {
|
||||
return p == ProvExplicit || p == ProvDeterministic || p == ProvUserConfirmed
|
||||
}
|
||||
|
||||
// NodeKind is what a relationship endpoint refers to.
|
||||
type NodeKind string
|
||||
|
||||
const (
|
||||
NodeItem NodeKind = "item"
|
||||
NodeChunk NodeKind = "chunk"
|
||||
NodeEntity NodeKind = "entity"
|
||||
NodeClaim NodeKind = "claim"
|
||||
)
|
||||
|
||||
// Valid reports whether k is a recognised node kind.
|
||||
func (k NodeKind) Valid() bool {
|
||||
switch k {
|
||||
case NodeItem, NodeChunk, NodeEntity, NodeClaim:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ReviewStatus tracks human adjudication of a proposed relationship.
|
||||
type ReviewStatus string
|
||||
|
||||
const (
|
||||
ReviewPending ReviewStatus = "pending"
|
||||
ReviewConfirmed ReviewStatus = "confirmed"
|
||||
ReviewRejected ReviewStatus = "rejected"
|
||||
ReviewEdited ReviewStatus = "edited"
|
||||
)
|
||||
|
||||
// SignalType names a concrete, explainable contribution to a relationship score.
|
||||
type SignalType string
|
||||
|
||||
const (
|
||||
SigSemanticSimilarity SignalType = "semantic_similarity"
|
||||
SigSharedEntity SignalType = "shared_entity"
|
||||
SigAliasEntity SignalType = "alias_entity"
|
||||
SigSharedConcept SignalType = "shared_concept"
|
||||
SigTemporalProximity SignalType = "temporal_proximity"
|
||||
SigSharedMetadata SignalType = "shared_metadata"
|
||||
SigClaimSupport SignalType = "claim_support"
|
||||
SigClaimContradiction SignalType = "claim_contradiction"
|
||||
SigModelClassify SignalType = "model_classification"
|
||||
SigUserConfirmed SignalType = "user_confirmation"
|
||||
SigUserRejected SignalType = "user_rejection"
|
||||
SigContentHashMatch SignalType = "content_hash_match"
|
||||
)
|
||||
|
||||
// Signal is one explicit piece of evidence behind a relationship score.
|
||||
//
|
||||
// Value carries the human-readable evidence ("Example Ltd", "2 days"); Strength
|
||||
// is the normalised 0..1 contribution the signal makes before weighting. Keeping
|
||||
// them separate is what lets the UI show the arithmetic instead of a bare number.
|
||||
type Signal struct {
|
||||
Type SignalType `bson:"type" json:"type"`
|
||||
Value string `bson:"value" json:"value"`
|
||||
Strength float64 `bson:"strength" json:"strength"`
|
||||
Weight float64 `bson:"weight" json:"weight"`
|
||||
Detail string `bson:"detail,omitempty" json:"detail,omitempty"`
|
||||
}
|
||||
|
||||
// Contribution is the signal's weighted addition to the total score.
|
||||
func (s Signal) Contribution() float64 { return s.Strength * s.Weight }
|
||||
|
||||
// Evidence is the concrete supporting material for a proposed relationship:
|
||||
// the signals plus pointers at the exact chunks and entities involved.
|
||||
type Evidence struct {
|
||||
Signals []Signal `bson:"signals" json:"signals"`
|
||||
|
||||
SourceChunkIDs []string `bson:"source_chunk_ids,omitempty" json:"source_chunk_ids,omitempty"`
|
||||
TargetChunkIDs []string `bson:"target_chunk_ids,omitempty" json:"target_chunk_ids,omitempty"`
|
||||
SharedEntities []string `bson:"shared_entities,omitempty" json:"shared_entities,omitempty"`
|
||||
|
||||
// Excerpts are short verbatim quotations from each side, for display.
|
||||
SourceExcerpt string `bson:"source_excerpt,omitempty" json:"source_excerpt,omitempty"`
|
||||
TargetExcerpt string `bson:"target_excerpt,omitempty" json:"target_excerpt,omitempty"`
|
||||
|
||||
// ModelRationale is the model's own words about the connection. It is
|
||||
// displayed as a model opinion, never as an established fact.
|
||||
ModelRationale string `bson:"model_rationale,omitempty" json:"model_rationale,omitempty"`
|
||||
ModelVersion string `bson:"model_version,omitempty" json:"model_version,omitempty"`
|
||||
}
|
||||
|
||||
// ReviewEvent is one immutable entry in a relationship's decision history.
|
||||
// History is append-only: rejecting a relationship never erases the proposal.
|
||||
type ReviewEvent struct {
|
||||
At time.Time `bson:"at" json:"at"`
|
||||
Action string `bson:"action" json:"action"`
|
||||
ActorID string `bson:"actor_id" json:"actor_id"`
|
||||
ActorName string `bson:"actor_name,omitempty" json:"actor_name,omitempty"`
|
||||
Note string `bson:"note,omitempty" json:"note,omitempty"`
|
||||
FromValue string `bson:"from_value,omitempty" json:"from_value,omitempty"`
|
||||
ToValue string `bson:"to_value,omitempty" json:"to_value,omitempty"`
|
||||
}
|
||||
|
||||
// Relationship connects two nodes with a scored, evidenced, provenance-tagged link.
|
||||
type Relationship struct {
|
||||
ID string `bson:"_id" json:"id"`
|
||||
WorkspaceID string `bson:"workspace_id" json:"workspace_id"`
|
||||
|
||||
SourceKind NodeKind `bson:"source_kind" json:"source_kind"`
|
||||
SourceID string `bson:"source_id" json:"source_id"`
|
||||
TargetKind NodeKind `bson:"target_kind" json:"target_kind"`
|
||||
TargetID string `bson:"target_id" json:"target_id"`
|
||||
|
||||
Type RelationshipType `bson:"type" json:"type"`
|
||||
Provenance Provenance `bson:"provenance" json:"provenance"`
|
||||
|
||||
// Score is a heuristic relationship strength in 0..1. It is NOT a calibrated
|
||||
// probability and must never be presented as one.
|
||||
Score float64 `bson:"score" json:"score"`
|
||||
ScoreVersion string `bson:"score_version" json:"score_version"`
|
||||
Evidence Evidence `bson:"evidence" json:"evidence"`
|
||||
|
||||
// OriginalType and OriginalProvenance preserve what was first proposed, even
|
||||
// after a user edits the relationship.
|
||||
OriginalType RelationshipType `bson:"original_type" json:"original_type"`
|
||||
OriginalProvenance Provenance `bson:"original_provenance" json:"original_provenance"`
|
||||
|
||||
ReviewStatus ReviewStatus `bson:"review_status" json:"review_status"`
|
||||
ReviewHistory []ReviewEvent `bson:"review_history,omitempty" json:"review_history,omitempty"`
|
||||
|
||||
// DedupeKey makes relationship creation idempotent across reprocessing runs.
|
||||
DedupeKey string `bson:"dedupe_key" json:"-"`
|
||||
|
||||
CreatedAt time.Time `bson:"created_at" json:"created_at"`
|
||||
UpdatedAt time.Time `bson:"updated_at" json:"updated_at"`
|
||||
Deleted bool `bson:"deleted" json:"deleted"`
|
||||
}
|
||||
|
||||
// Confirmed reports whether a human affirmed this relationship.
|
||||
func (r *Relationship) Confirmed() bool { return r.ReviewStatus == ReviewConfirmed }
|
||||
|
||||
// Rejected reports whether a human rejected this relationship. Rejected
|
||||
// relationships are retained for audit and to inform future ranking.
|
||||
func (r *Relationship) Rejected() bool { return r.ReviewStatus == ReviewRejected }
|
||||
|
||||
// Inferred reports whether the link is a model proposal rather than an assertion.
|
||||
func (r *Relationship) Inferred() bool { return r.Provenance == ProvModelInferred }
|
||||
Reference in New Issue
Block a user