relata/internal/search/search.go

// Package search implements hybrid retrieval over stored items.
//
// A query runs through several independent paths — exact and lexical matching,
// vector similarity, entity matching, and expansion along confirmed
// relationships — whose results are fused by rank. Every result carries the
// reasons it was returned, and the three distinct numbers involved (query
// relevance, semantic similarity, relationship strength) are reported
// separately rather than merged into one figure.
package search

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"time"
	"unicode"

	"github.com/andydixon/relata/internal/config"
	"github.com/andydixon/relata/internal/domain"
	"github.com/andydixon/relata/internal/entities"
	"github.com/andydixon/relata/internal/observability"
	"github.com/andydixon/relata/internal/ranking"
	"github.com/andydixon/relata/internal/relationships"
)

// Query is a search request. Every field is a typed value; no part of it ever
// becomes a database operator.
type Query struct {
	Text string

	SourceTypes []domain.SourceType
	MIMETypes   []string
	EntityIDs   []string
	RelTypes    []domain.RelationshipType
	CreatedFrom *time.Time
	CreatedTo   *time.Time
	// ConfirmedOnly restricts relationship expansion to human-confirmed links.
	ConfirmedOnly bool
	// MinRelStrength filters expanded results by relationship strength.
	MinRelStrength float64
	// Expand enables relationship expansion. On by default for the UI.
	Expand bool

	Limit  int
	Offset int
}

// Result is one item returned for a query.
type Result struct {
	Item *domain.Item `json:"item"`

	// QueryRelevance is how well this item answers THIS query, 0..1 relative to
	// the best result. It is not a similarity and not a probability.
	QueryRelevance float64 `json:"query_relevance"`
	// SemanticSimilarity is the best cosine similarity of any matching chunk,
	// reported separately because similarity is not relevance.
	SemanticSimilarity float64 `json:"semantic_similarity,omitempty"`

	Excerpt         string                 `json:"excerpt"`
	MatchedChunks   []MatchedChunk         `json:"matched_chunks,omitempty"`
	MatchedEntities []MatchedEntity        `json:"matched_entities,omitempty"`
	Related         []RelatedItem          `json:"related_items,omitempty"`
	WhyMatched      []string               `json:"why_matched"`
	Contributions   []ranking.Contribution `json:"contributions,omitempty"`

	// ViaRelationship is set when this item was reached by expansion rather
	// than by matching the query itself.
	ViaRelationship *RelatedItem `json:"via_relationship,omitempty"`

	RankVersion string `json:"rank_version"`
}

// MatchedChunk is a chunk that matched, with its own similarity.
type MatchedChunk struct {
	ChunkID    string  `json:"chunk_id"`
	Sequence   int     `json:"sequence"`
	Text       string  `json:"text"`
	Similarity float64 `json:"semantic_similarity,omitempty"`
	Page       int     `json:"page,omitempty"`
	Section    string  `json:"section,omitempty"`
}

// MatchedEntity is an entity the query matched.
type MatchedEntity struct {
	ID   string            `json:"id"`
	Name string            `json:"name"`
	Type domain.EntityType `json:"type"`
}

// RelatedItem describes a relationship from a result to another item.
type RelatedItem struct {
	RelationshipID string                  `json:"relationship_id"`
	ItemID         string                  `json:"item_id"`
	Title          string                  `json:"title"`
	Type           domain.RelationshipType `json:"type"`
	Provenance     domain.Provenance       `json:"provenance"`
	// Strength is the relationship's heuristic strength, NOT query relevance.
	Strength     float64             `json:"relationship_strength"`
	ScoreVersion string              `json:"score_version"`
	ReviewStatus domain.ReviewStatus `json:"review_status"`
	Explanation  []string            `json:"explanation,omitempty"`
	Direction    string              `json:"direction"`
}

// Response is a complete search result set.
type Response struct {
	Query   string   `json:"query"`
	Results []Result `json:"results"`
	Total   int      `json:"total"`
	Limit   int      `json:"limit"`
	Offset  int      `json:"offset"`
	TookMS  int64    `json:"took_ms"`
	// Notes carries advisory messages, such as vector search being unavailable.
	Notes []string `json:"notes,omitempty"`
}

// Service performs searches.
type Service struct {
	items    domain.ItemRepository
	chunks   domain.ChunkRepository
	entities domain.EntityRepository
	rels     domain.RelationshipRepository
	vectors  domain.VectorStore
	embedder domain.Embedder
	cfg      *config.Config
	metrics  *observability.Metrics
}

// NewService builds the search service.
func NewService(items domain.ItemRepository, chunks domain.ChunkRepository,
	ents domain.EntityRepository, rels domain.RelationshipRepository,
	vectors domain.VectorStore, embedder domain.Embedder,
	cfg *config.Config, m *observability.Metrics) *Service {
	return &Service{
		items: items, chunks: chunks, entities: ents, rels: rels,
		vectors: vectors, embedder: embedder, cfg: cfg, metrics: m,
	}
}

// recencyHalfLife controls how gently recency nudges the ordering.
const recencyHalfLife = 180 * 24 * time.Hour

// Search runs the hybrid retrieval pipeline.
func (s *Service) Search(ctx context.Context, p *domain.Principal, q Query) (*Response, error) {
	if p == nil || p.WorkspaceID == "" {
		return nil, domain.ErrWorkspaceMiss
	}
	started := time.Now()

	limit := q.Limit
	if limit <= 0 {
		limit = s.cfg.Search.DefaultLimit
	}
	if limit > s.cfg.Search.MaxResults {
		limit = s.cfg.Search.MaxResults
	}
	offset := q.Offset
	if offset < 0 {
		offset = 0
	}

	resp := &Response{Query: q.Text, Limit: limit, Offset: offset}
	cands := map[string]*ranking.Candidate{}
	get := func(itemID string) *ranking.Candidate {
		c := cands[itemID]
		if c == nil {
			c = &ranking.Candidate{ItemID: itemID}
			cands[itemID] = c
		}
		return c
	}

	itemFilter := domain.ItemFilter{
		SourceTypes: q.SourceTypes,
		MIMETypes:   q.MIMETypes,
		CreatedFrom: q.CreatedFrom,
		CreatedTo:   q.CreatedTo,
		Statuses: []domain.ProcessingStatus{
			domain.StatusReady, domain.StatusPartial,
			domain.StatusAnalysing, domain.StatusRelating, domain.StatusEmbedding,
		},
	}

	trimmed := strings.TrimSpace(q.Text)

	// --- Path 1: lexical item search ---------------------------------------
	if trimmed != "" {
		hits, err := s.items.TextSearch(ctx, p.WorkspaceID, trimmed, itemFilter, s.cfg.Search.LexicalCandidates)
		if err != nil {
			resp.Notes = append(resp.Notes, "Keyword search was unavailable for this query.")
			observability.LoggerFrom(ctx).Warn("lexical item search failed", "error", err.Error())
		}
		for i, h := range hits {
			get(h.ItemID).Add(ranking.SourceLexical, i+1,
				"The item's text contains words from your query.")
		}
	}

	// --- Path 2: lexical chunk search, for excerpts -------------------------
	chunkHits := map[string][]domain.ChunkHit{}
	if trimmed != "" {
		hits, err := s.chunks.TextSearch(ctx, p.WorkspaceID, trimmed, s.cfg.Search.LexicalCandidates)
		if err == nil {
			for i, h := range hits {
				chunkHits[h.ItemID] = append(chunkHits[h.ItemID], h)
				c := get(h.ItemID)
				c.MatchedChunks = appendUnique(c.MatchedChunks, h.ChunkID, 5)
				if i < 20 {
					c.Add(ranking.SourceLexical, i+1,
						"A section of this item contains words from your query.")
				}
			}
		}
	}

	// --- Path 3: vector similarity -----------------------------------------
	vectorSims := map[string]float64{}
	chunkSims := map[string]float64{}
	if trimmed != "" {
		vecs, err := s.embedder.Embed(ctx, []string{trimmed})
		if err != nil || len(vecs) == 0 {
			resp.Notes = append(resp.Notes,
				"Meaning-based search was unavailable, so these results are from keyword matching only.")
			observability.LoggerFrom(ctx).Warn("could not embed the query", "error", errString(err))
		} else {
			matches, err := s.vectors.Search(ctx, vecs[0], s.cfg.Search.VectorCandidates, domain.VectorFilter{
				WorkspaceID:    p.WorkspaceID,
				SourceTypes:    q.SourceTypes,
				EmbeddingModel: s.embedder.ModelID(),
				CreatedFrom:    q.CreatedFrom,
				CreatedTo:      q.CreatedTo,
			})
			if err != nil {
				resp.Notes = append(resp.Notes, "Meaning-based search was unavailable for this query.")
				observability.LoggerFrom(ctx).Warn("vector search failed", "error", err.Error())
			}
			for i, m := range matches {
				c := get(m.ItemID)
				c.Add(ranking.SourceVector, i+1,
					fmt.Sprintf("A section of this item is close in meaning to your query (similarity %.2f).", m.Score))
				if m.Score > c.BestSimilarity {
					c.BestSimilarity = m.Score
				}
				if m.Score > vectorSims[m.ItemID] {
					vectorSims[m.ItemID] = m.Score
				}
				chunkSims[m.ChunkID] = m.Score
				c.MatchedChunks = appendUnique(c.MatchedChunks, m.ChunkID, 5)
			}
		}
	}

	// --- Path 4: entity matching -------------------------------------------
	matchedEntities := map[string][]*domain.Entity{}
	if err := s.matchEntities(ctx, p.WorkspaceID, q, trimmed, get, matchedEntities); err != nil {
		observability.LoggerFrom(ctx).Warn("entity matching failed", "error", err.Error())
	}

	// An empty query with filters is a legitimate browse request.
	if trimmed == "" && len(cands) == 0 {
		items, total, err := s.items.List(ctx, p.WorkspaceID, itemFilter,
			domain.Page{Limit: limit, Offset: offset})
		if err != nil {
			return nil, err
		}
		for i, it := range items {
			c := get(it.ID)
			c.CreatedAt = it.CreatedAt
			c.Add(ranking.SourceLexical, i+1, "Matched your filters.")
		}
		resp.Total = int(total)
	}

	if len(cands) == 0 {
		resp.TookMS = time.Since(started).Milliseconds()
		s.count("hybrid", "empty")
		return resp, nil
	}

	// Load the candidate items, applying the workspace and filter constraints
	// that the vector store cannot express.
	ids := make([]string, 0, len(cands))
	for id := range cands {
		ids = append(ids, id)
	}
	items, err := s.items.GetMany(ctx, p.WorkspaceID, ids)
	if err != nil {
		return nil, err
	}
	byID := make(map[string]*domain.Item, len(items))
	for _, it := range items {
		if it.Deleted || !matchesFilter(it, itemFilter, q) {
			delete(cands, it.ID)
			continue
		}
		byID[it.ID] = it
		cands[it.ID].CreatedAt = it.CreatedAt
	}
	// Drop candidates whose items were not returned at all: they are deleted,
	// in another workspace, or otherwise not visible to this principal.
	for id := range cands {
		if byID[id] == nil {
			delete(cands, id)
		}
	}

	// Title matching is cheap and a strong signal a user expects to work.
	if trimmed != "" {
		lower := strings.ToLower(trimmed)
		for id, it := range byID {
			if strings.Contains(strings.ToLower(it.Title), lower) {
				cands[id].AddScaled(ranking.SourceTitle, 1,
					fmt.Sprintf("The title %q contains your query.", it.Title))
			}
		}
	}

	// --- Path 5: relationship expansion ------------------------------------
	expansions := map[string]*RelatedItem{}
	relsByItem := map[string][]*domain.Relationship{}
	if q.Expand {
		s.expand(ctx, p.WorkspaceID, q, cands, byID, get, expansions, relsByItem)

		// Newly-expanded items still need their records.
		var missing []string
		for id := range cands {
			if byID[id] == nil {
				missing = append(missing, id)
			}
		}
		if len(missing) > 0 {
			more, err := s.items.GetMany(ctx, p.WorkspaceID, missing)
			if err == nil {
				for _, it := range more {
					if it.Deleted || !matchesFilter(it, itemFilter, q) {
						delete(cands, it.ID)
						continue
					}
					byID[it.ID] = it
					cands[it.ID].CreatedAt = it.CreatedAt
				}
			}
			for _, id := range missing {
				if byID[id] == nil {
					delete(cands, id)
				}
			}
		}
	}

	// --- Fuse and order -----------------------------------------------------
	list := make([]*ranking.Candidate, 0, len(cands))
	for _, c := range cands {
		list = append(list, c)
	}
	ranked := ranking.Rank(list, time.Now(), recencyHalfLife)

	if resp.Total == 0 {
		resp.Total = len(ranked)
	}

	// Apply paging after ranking, so a page boundary does not change ordering.
	start := offset
	if start > len(ranked) {
		start = len(ranked)
	}
	end := start + limit
	if end > len(ranked) {
		end = len(ranked)
	}
	page := ranked[start:end]

	// Fetch the chunks needed for excerpts, in one query.
	var wanted []string
	for _, c := range page {
		wanted = append(wanted, c.MatchedChunks...)
	}
	chunkByID := map[string]*domain.Chunk{}
	if len(wanted) > 0 {
		got, err := s.chunks.GetMany(ctx, p.WorkspaceID, wanted)
		if err == nil {
			for _, ch := range got {
				chunkByID[ch.ID] = ch
			}
		}
	}

	for _, c := range page {
		it := byID[c.ItemID]
		if it == nil {
			continue
		}
		r := Result{
			Item:               it,
			QueryRelevance:     c.Relevance,
			SemanticSimilarity: c.BestSimilarity,
			WhyMatched:         ranking.Explain(c),
			Contributions:      c.Contributions,
			RankVersion:        ranking.RankVersion,
		}

		for _, cid := range c.MatchedChunks {
			ch := chunkByID[cid]
			if ch == nil {
				continue
			}
			r.MatchedChunks = append(r.MatchedChunks, MatchedChunk{
				ChunkID:    ch.ID,
				Sequence:   ch.Sequence,
				Text:       ch.Text,
				Similarity: chunkSims[ch.ID],
				Page:       ch.Page,
				Section:    ch.Section,
			})
		}
		sort.SliceStable(r.MatchedChunks, func(i, j int) bool {
			return r.MatchedChunks[i].Similarity > r.MatchedChunks[j].Similarity
		})

		r.Excerpt = buildExcerpt(it, r.MatchedChunks, chunkHits[it.ID], trimmed)

		for _, e := range matchedEntities[it.ID] {
			r.MatchedEntities = append(r.MatchedEntities, MatchedEntity{
				ID: e.ID, Name: e.CanonicalName, Type: e.Type,
			})
		}

		if via, ok := expansions[it.ID]; ok {
			r.ViaRelationship = via
		}
		r.Related = s.relatedFor(it.ID, relsByItem[it.ID], byID)

		resp.Results = append(resp.Results, r)
	}

	resp.TookMS = time.Since(started).Milliseconds()
	s.count("hybrid", "success")
	if s.metrics != nil {
		s.metrics.SearchLatency.Observe(time.Since(started).Seconds())
	}
	return resp, nil
}

func (s *Service) count(mode, outcome string) {
	if s.metrics != nil {
		s.metrics.SearchOps.WithLabelValues(mode, outcome).Inc()
	}
}

// matchEntities finds entities the query names and adds the items citing them.
func (s *Service) matchEntities(ctx context.Context, workspaceID string, q Query, text string,
	get func(string) *ranking.Candidate, out map[string][]*domain.Entity) error {

	var found []*domain.Entity

	// Explicit entity filters are authoritative.
	if len(q.EntityIDs) > 0 {
		ents, err := s.entities.GetMany(ctx, workspaceID, q.EntityIDs)
		if err != nil {
			return err
		}
		found = append(found, ents...)
	}

	// Otherwise look for entity names appearing in the query text.
	if text != "" {
		keys := candidateEntityKeys(text)
		if len(keys) > 0 {
			ents, err := s.entities.FindByNormalised(ctx, workspaceID, keys)
			if err != nil {
				return err
			}
			found = append(found, ents...)
		}
	}

	seen := map[string]bool{}
	for _, e := range found {
		if seen[e.ID] {
			continue
		}
		seen[e.ID] = true
		for _, itemID := range e.ItemIDs {
			c := get(itemID)
			c.AddScaled(ranking.SourceEntity, 1,
				fmt.Sprintf("This item refers to %s (%s), which your query names.", e.CanonicalName, e.Type))
			c.MatchedEntities = appendUnique(c.MatchedEntities, e.ID, 8)
			out[itemID] = append(out[itemID], e)
		}
	}
	return nil
}

// candidateEntityKeys generates normalised keys for word sequences in the
// query, so "Example Ltd" matches the stored organisation.
//
// Only short n-grams are tried, and each becomes a value in an indexed $in
// lookup — never a pattern or an operator.
func candidateEntityKeys(text string) []string {
	words := strings.FieldsFunc(text, func(r rune) bool {
		return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\'' && r != '&'
	})
	if len(words) == 0 || len(words) > 40 {
		return nil
	}

	var keys []string
	seen := map[string]bool{}
	addKey := func(phrase string) {
		norm := entities.Normalise(phrase)
		if norm == "" || len(norm) < 3 {
			return
		}
		for _, t := range domain.ValidEntityTypes {
			k := string(t) + ":" + norm
			if !seen[k] {
				seen[k] = true
				keys = append(keys, k)
			}
		}
	}

	// Unigrams through trigrams.
	for n := 1; n <= 3; n++ {
		for i := 0; i+n <= len(words); i++ {
			if len(keys) > 600 {
				return keys
			}
			addKey(strings.Join(words[i:i+n], " "))
		}
	}
	return keys
}

// expand pulls in items connected to the current matches by a relationship.
//
// Expanded items are labelled with the relationship that brought them in, and
// their contribution is weighted well below a direct match: being related to a
// hit is weaker evidence than being one. Rejected relationships never expand.
func (s *Service) expand(ctx context.Context, workspaceID string, q Query,
	cands map[string]*ranking.Candidate, byID map[string]*domain.Item,
	get func(string) *ranking.Candidate, expansions map[string]*RelatedItem,
	relsByItem map[string][]*domain.Relationship) {

	// Expand only from the strongest current matches.
	seeds := make([]*ranking.Candidate, 0, len(cands))
	for _, c := range cands {
		seeds = append(seeds, c)
	}
	sort.SliceStable(seeds, func(i, j int) bool {
		return len(seeds[i].Contributions) > len(seeds[j].Contributions)
	})
	max := s.cfg.Search.MaxExpandedItems
	if max <= 0 {
		max = 20
	}
	if len(seeds) > max {
		seeds = seeds[:max]
	}

	seedIDs := make([]string, 0, len(seeds))
	for _, c := range seeds {
		seedIDs = append(seedIDs, c.ItemID)
	}

	minScore := q.MinRelStrength
	if minScore <= 0 {
		minScore = s.cfg.Search.ExpansionMinScore
	}

	rels, err := s.rels.ListForItems(ctx, workspaceID, seedIDs, minScore)
	if err != nil {
		observability.LoggerFrom(ctx).Warn("relationship expansion failed", "error", err.Error())
		return
	}

	isSeed := map[string]bool{}
	for _, id := range seedIDs {
		isSeed[id] = true
	}

	added := 0
	for _, rel := range rels {
		if q.ConfirmedOnly && rel.ReviewStatus != domain.ReviewConfirmed {
			continue
		}
		if len(q.RelTypes) > 0 && !containsRelType(q.RelTypes, rel.Type) {
			continue
		}

		relsByItem[rel.SourceID] = append(relsByItem[rel.SourceID], rel)
		relsByItem[rel.TargetID] = append(relsByItem[rel.TargetID], rel)

		// Identify the far end of the relationship.
		var otherID, direction string
		switch {
		case isSeed[rel.SourceID] && !isSeed[rel.TargetID]:
			otherID, direction = rel.TargetID, "outgoing"
		case isSeed[rel.TargetID] && !isSeed[rel.SourceID]:
			otherID, direction = rel.SourceID, "incoming"
		default:
			continue // both ends already matched, or neither did
		}
		if added >= max {
			break
		}
		added++

		c := get(otherID)
		c.ExpandedFrom = rel.SourceID
		label := "related to"
		if rel.ReviewStatus == domain.ReviewConfirmed {
			label = "confirmed as related to"
			c.AddScaled(ranking.SourceConfirmedRel, 1,
				"A person has confirmed this item's relationship to a matching item.")
		}
		c.AddScaled(ranking.SourceExpansion, rel.Score, fmt.Sprintf(
			"Not a direct match: this item is %s a matching item by a %s relationship (strength %.2f, %s).",
			label, rel.Type, rel.Score, provenanceLabel(rel.Provenance)))

		expansions[otherID] = &RelatedItem{
			RelationshipID: rel.ID,
			ItemID:         otherID,
			Type:           rel.Type,
			Provenance:     rel.Provenance,
			Strength:       rel.Score,
			ScoreVersion:   rel.ScoreVersion,
			ReviewStatus:   rel.ReviewStatus,
			Direction:      direction,
			Explanation:    relationships.Explain(rel),
		}
	}
}

// relatedFor renders an item's relationships for display.
func (s *Service) relatedFor(itemID string, rels []*domain.Relationship, byID map[string]*domain.Item) []RelatedItem {
	if len(rels) == 0 {
		return nil
	}
	seen := map[string]bool{}
	out := make([]RelatedItem, 0, len(rels))

	for _, rel := range rels {
		if seen[rel.ID] {
			continue
		}
		seen[rel.ID] = true

		otherID, direction := rel.TargetID, "outgoing"
		if rel.TargetID == itemID {
			otherID, direction = rel.SourceID, "incoming"
		}
		title := ""
		if it := byID[otherID]; it != nil {
			title = it.Title
		}
		out = append(out, RelatedItem{
			RelationshipID: rel.ID,
			ItemID:         otherID,
			Title:          title,
			Type:           rel.Type,
			Provenance:     rel.Provenance,
			Strength:       rel.Score,
			ScoreVersion:   rel.ScoreVersion,
			ReviewStatus:   rel.ReviewStatus,
			Direction:      direction,
			Explanation:    relationships.Explain(rel),
		})
	}
	sort.SliceStable(out, func(i, j int) bool { return out[i].Strength > out[j].Strength })
	if len(out) > 8 {
		out = out[:8]
	}
	return out
}

// buildExcerpt picks the most useful snippet to show for a result.
func buildExcerpt(item *domain.Item, matched []MatchedChunk, lexical []domain.ChunkHit, query string) string {
	const width = 320

	// Prefer a chunk that lexically contains a query term, so the excerpt shows
	// the user their own words where possible.
	if query != "" {
		terms := strings.Fields(strings.ToLower(query))
		for _, mc := range matched {
			if snippet := findTermWindow(mc.Text, terms, width); snippet != "" {
				return snippet
			}
		}
		for _, lh := range lexical {
			if snippet := findTermWindow(lh.Text, terms, width); snippet != "" {
				return snippet
			}
		}
	}
	if len(matched) > 0 {
		return truncateWords(matched[0].Text, width)
	}
	if len(lexical) > 0 {
		return truncateWords(lexical[0].Text, width)
	}
	return truncateWords(item.ExtractedText, width)
}

// findTermWindow returns a window of text around the first matching term.
func findTermWindow(text string, terms []string, width int) string {
	lower := strings.ToLower(text)
	best := -1
	for _, t := range terms {
		if len(t) < 3 {
			continue
		}
		if i := strings.Index(lower, t); i >= 0 && (best < 0 || i < best) {
			best = i
		}
	}
	if best < 0 {
		return ""
	}
	start := best - width/3
	if start < 0 {
		start = 0
	}
	for start > 0 && text[start]&0xC0 == 0x80 {
		start--
	}
	end := start + width
	if end > len(text) {
		end = len(text)
	}
	for end < len(text) && text[end]&0xC0 == 0x80 {
		end++
	}
	out := strings.TrimSpace(collapse(text[start:end]))
	if start > 0 {
		out = "…" + out
	}
	if end < len(text) {
		out += "…"
	}
	return out
}

func truncateWords(s string, n int) string {
	s = collapse(strings.TrimSpace(s))
	if len(s) <= n {
		return s
	}
	cut := n
	for cut > 0 && s[cut]&0xC0 == 0x80 {
		cut--
	}
	if i := strings.LastIndexByte(s[:cut], ' '); i > n/2 {
		cut = i
	}
	return s[:cut] + "…"
}

func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }

// matchesFilter applies the filters the storage query could not express.
func matchesFilter(it *domain.Item, f domain.ItemFilter, q Query) bool {
	if len(f.SourceTypes) > 0 && !containsSourceType(f.SourceTypes, it.SourceType) {
		return false
	}
	if len(f.MIMETypes) > 0 && !containsString(f.MIMETypes, it.MIMEType) {
		return false
	}
	if f.CreatedFrom != nil && it.CreatedAt.Before(*f.CreatedFrom) {
		return false
	}
	if f.CreatedTo != nil && it.CreatedAt.After(*f.CreatedTo) {
		return false
	}
	return true
}

func containsSourceType(list []domain.SourceType, v domain.SourceType) bool {
	for _, e := range list {
		if e == v {
			return true
		}
	}
	return false
}

func containsString(list []string, v string) bool {
	for _, e := range list {
		if e == v {
			return true
		}
	}
	return false
}

func containsRelType(list []domain.RelationshipType, v domain.RelationshipType) bool {
	for _, e := range list {
		if e == v {
			return true
		}
	}
	return false
}

func appendUnique(list []string, v string, max int) []string {
	if v == "" || len(list) >= max {
		return list
	}
	for _, e := range list {
		if e == v {
			return list
		}
	}
	return append(list, v)
}

func provenanceLabel(p domain.Provenance) string {
	switch p {
	case domain.ProvExplicit:
		return "stated explicitly"
	case domain.ProvDeterministic:
		return "derived from matching evidence"
	case domain.ProvModelInferred:
		return "suggested by analysis, not yet reviewed"
	case domain.ProvUserConfirmed:
		return "confirmed by a person"
	case domain.ProvUserRejected:
		return "rejected by a person"
	}
	return string(p)
}

func errString(err error) string {
	if err == nil {
		return ""
	}
	return err.Error()
}