fix: harden credential and input handling
This commit is contained in:
Andy Dixon
@@ -0,0 +1,839 @@
|
||||
// Package search implements hybrid retrieval over stored items.
|
||||
//
|
||||
// A query runs through several independent paths — exact and lexical matching,
|
||||
// vector similarity, entity matching, and expansion along confirmed
|
||||
// relationships — whose results are fused by rank. Every result carries the
|
||||
// reasons it was returned, and the three distinct numbers involved (query
|
||||
// relevance, semantic similarity, relationship strength) are reported
|
||||
// separately rather than merged into one figure.
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"github.com/andydixon/relata/internal/config"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/entities"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
"github.com/andydixon/relata/internal/ranking"
|
||||
"github.com/andydixon/relata/internal/relationships"
|
||||
)
|
||||
|
||||
// Query is a search request. Every field is a typed value; no part of it ever
|
||||
// becomes a database operator.
|
||||
type Query struct {
|
||||
Text string
|
||||
|
||||
SourceTypes []domain.SourceType
|
||||
MIMETypes []string
|
||||
EntityIDs []string
|
||||
RelTypes []domain.RelationshipType
|
||||
CreatedFrom *time.Time
|
||||
CreatedTo *time.Time
|
||||
// ConfirmedOnly restricts relationship expansion to human-confirmed links.
|
||||
ConfirmedOnly bool
|
||||
// MinRelStrength filters expanded results by relationship strength.
|
||||
MinRelStrength float64
|
||||
// Expand enables relationship expansion. On by default for the UI.
|
||||
Expand bool
|
||||
|
||||
Limit int
|
||||
Offset int
|
||||
}
|
||||
|
||||
// Result is one item returned for a query.
|
||||
type Result struct {
|
||||
Item *domain.Item `json:"item"`
|
||||
|
||||
// QueryRelevance is how well this item answers THIS query, 0..1 relative to
|
||||
// the best result. It is not a similarity and not a probability.
|
||||
QueryRelevance float64 `json:"query_relevance"`
|
||||
// SemanticSimilarity is the best cosine similarity of any matching chunk,
|
||||
// reported separately because similarity is not relevance.
|
||||
SemanticSimilarity float64 `json:"semantic_similarity,omitempty"`
|
||||
|
||||
Excerpt string `json:"excerpt"`
|
||||
MatchedChunks []MatchedChunk `json:"matched_chunks,omitempty"`
|
||||
MatchedEntities []MatchedEntity `json:"matched_entities,omitempty"`
|
||||
Related []RelatedItem `json:"related_items,omitempty"`
|
||||
WhyMatched []string `json:"why_matched"`
|
||||
Contributions []ranking.Contribution `json:"contributions,omitempty"`
|
||||
|
||||
// ViaRelationship is set when this item was reached by expansion rather
|
||||
// than by matching the query itself.
|
||||
ViaRelationship *RelatedItem `json:"via_relationship,omitempty"`
|
||||
|
||||
RankVersion string `json:"rank_version"`
|
||||
}
|
||||
|
||||
// MatchedChunk is a chunk that matched, with its own similarity.
|
||||
type MatchedChunk struct {
|
||||
ChunkID string `json:"chunk_id"`
|
||||
Sequence int `json:"sequence"`
|
||||
Text string `json:"text"`
|
||||
Similarity float64 `json:"semantic_similarity,omitempty"`
|
||||
Page int `json:"page,omitempty"`
|
||||
Section string `json:"section,omitempty"`
|
||||
}
|
||||
|
||||
// MatchedEntity is an entity the query matched.
|
||||
type MatchedEntity struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type domain.EntityType `json:"type"`
|
||||
}
|
||||
|
||||
// RelatedItem describes a relationship from a result to another item.
|
||||
type RelatedItem struct {
|
||||
RelationshipID string `json:"relationship_id"`
|
||||
ItemID string `json:"item_id"`
|
||||
Title string `json:"title"`
|
||||
Type domain.RelationshipType `json:"type"`
|
||||
Provenance domain.Provenance `json:"provenance"`
|
||||
// Strength is the relationship's heuristic strength, NOT query relevance.
|
||||
Strength float64 `json:"relationship_strength"`
|
||||
ScoreVersion string `json:"score_version"`
|
||||
ReviewStatus domain.ReviewStatus `json:"review_status"`
|
||||
Explanation []string `json:"explanation,omitempty"`
|
||||
Direction string `json:"direction"`
|
||||
}
|
||||
|
||||
// Response is a complete search result set.
|
||||
type Response struct {
|
||||
Query string `json:"query"`
|
||||
Results []Result `json:"results"`
|
||||
Total int `json:"total"`
|
||||
Limit int `json:"limit"`
|
||||
Offset int `json:"offset"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
// Notes carries advisory messages, such as vector search being unavailable.
|
||||
Notes []string `json:"notes,omitempty"`
|
||||
}
|
||||
|
||||
// Service performs searches.
|
||||
type Service struct {
|
||||
items domain.ItemRepository
|
||||
chunks domain.ChunkRepository
|
||||
entities domain.EntityRepository
|
||||
rels domain.RelationshipRepository
|
||||
vectors domain.VectorStore
|
||||
embedder domain.Embedder
|
||||
cfg *config.Config
|
||||
metrics *observability.Metrics
|
||||
}
|
||||
|
||||
// NewService builds the search service.
|
||||
func NewService(items domain.ItemRepository, chunks domain.ChunkRepository,
|
||||
ents domain.EntityRepository, rels domain.RelationshipRepository,
|
||||
vectors domain.VectorStore, embedder domain.Embedder,
|
||||
cfg *config.Config, m *observability.Metrics) *Service {
|
||||
return &Service{
|
||||
items: items, chunks: chunks, entities: ents, rels: rels,
|
||||
vectors: vectors, embedder: embedder, cfg: cfg, metrics: m,
|
||||
}
|
||||
}
|
||||
|
||||
// recencyHalfLife controls how gently recency nudges the ordering.
|
||||
const recencyHalfLife = 180 * 24 * time.Hour
|
||||
|
||||
// Search runs the hybrid retrieval pipeline.
|
||||
func (s *Service) Search(ctx context.Context, p *domain.Principal, q Query) (*Response, error) {
|
||||
if p == nil || p.WorkspaceID == "" {
|
||||
return nil, domain.ErrWorkspaceMiss
|
||||
}
|
||||
started := time.Now()
|
||||
|
||||
limit := q.Limit
|
||||
if limit <= 0 {
|
||||
limit = s.cfg.Search.DefaultLimit
|
||||
}
|
||||
if limit > s.cfg.Search.MaxResults {
|
||||
limit = s.cfg.Search.MaxResults
|
||||
}
|
||||
offset := q.Offset
|
||||
if offset < 0 {
|
||||
offset = 0
|
||||
}
|
||||
|
||||
resp := &Response{Query: q.Text, Limit: limit, Offset: offset}
|
||||
cands := map[string]*ranking.Candidate{}
|
||||
get := func(itemID string) *ranking.Candidate {
|
||||
c := cands[itemID]
|
||||
if c == nil {
|
||||
c = &ranking.Candidate{ItemID: itemID}
|
||||
cands[itemID] = c
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
itemFilter := domain.ItemFilter{
|
||||
SourceTypes: q.SourceTypes,
|
||||
MIMETypes: q.MIMETypes,
|
||||
CreatedFrom: q.CreatedFrom,
|
||||
CreatedTo: q.CreatedTo,
|
||||
Statuses: []domain.ProcessingStatus{
|
||||
domain.StatusReady, domain.StatusPartial,
|
||||
domain.StatusAnalysing, domain.StatusRelating, domain.StatusEmbedding,
|
||||
},
|
||||
}
|
||||
|
||||
trimmed := strings.TrimSpace(q.Text)
|
||||
|
||||
// --- Path 1: lexical item search ---------------------------------------
|
||||
if trimmed != "" {
|
||||
hits, err := s.items.TextSearch(ctx, p.WorkspaceID, trimmed, itemFilter, s.cfg.Search.LexicalCandidates)
|
||||
if err != nil {
|
||||
resp.Notes = append(resp.Notes, "Keyword search was unavailable for this query.")
|
||||
observability.LoggerFrom(ctx).Warn("lexical item search failed", "error", err.Error())
|
||||
}
|
||||
for i, h := range hits {
|
||||
get(h.ItemID).Add(ranking.SourceLexical, i+1,
|
||||
"The item's text contains words from your query.")
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 2: lexical chunk search, for excerpts -------------------------
|
||||
chunkHits := map[string][]domain.ChunkHit{}
|
||||
if trimmed != "" {
|
||||
hits, err := s.chunks.TextSearch(ctx, p.WorkspaceID, trimmed, s.cfg.Search.LexicalCandidates)
|
||||
if err == nil {
|
||||
for i, h := range hits {
|
||||
chunkHits[h.ItemID] = append(chunkHits[h.ItemID], h)
|
||||
c := get(h.ItemID)
|
||||
c.MatchedChunks = appendUnique(c.MatchedChunks, h.ChunkID, 5)
|
||||
if i < 20 {
|
||||
c.Add(ranking.SourceLexical, i+1,
|
||||
"A section of this item contains words from your query.")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 3: vector similarity -----------------------------------------
|
||||
vectorSims := map[string]float64{}
|
||||
chunkSims := map[string]float64{}
|
||||
if trimmed != "" {
|
||||
vecs, err := s.embedder.Embed(ctx, []string{trimmed})
|
||||
if err != nil || len(vecs) == 0 {
|
||||
resp.Notes = append(resp.Notes,
|
||||
"Meaning-based search was unavailable, so these results are from keyword matching only.")
|
||||
observability.LoggerFrom(ctx).Warn("could not embed the query", "error", errString(err))
|
||||
} else {
|
||||
matches, err := s.vectors.Search(ctx, vecs[0], s.cfg.Search.VectorCandidates, domain.VectorFilter{
|
||||
WorkspaceID: p.WorkspaceID,
|
||||
SourceTypes: q.SourceTypes,
|
||||
EmbeddingModel: s.embedder.ModelID(),
|
||||
CreatedFrom: q.CreatedFrom,
|
||||
CreatedTo: q.CreatedTo,
|
||||
})
|
||||
if err != nil {
|
||||
resp.Notes = append(resp.Notes, "Meaning-based search was unavailable for this query.")
|
||||
observability.LoggerFrom(ctx).Warn("vector search failed", "error", err.Error())
|
||||
}
|
||||
for i, m := range matches {
|
||||
c := get(m.ItemID)
|
||||
c.Add(ranking.SourceVector, i+1,
|
||||
fmt.Sprintf("A section of this item is close in meaning to your query (similarity %.2f).", m.Score))
|
||||
if m.Score > c.BestSimilarity {
|
||||
c.BestSimilarity = m.Score
|
||||
}
|
||||
if m.Score > vectorSims[m.ItemID] {
|
||||
vectorSims[m.ItemID] = m.Score
|
||||
}
|
||||
chunkSims[m.ChunkID] = m.Score
|
||||
c.MatchedChunks = appendUnique(c.MatchedChunks, m.ChunkID, 5)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 4: entity matching -------------------------------------------
|
||||
matchedEntities := map[string][]*domain.Entity{}
|
||||
if err := s.matchEntities(ctx, p.WorkspaceID, q, trimmed, get, matchedEntities); err != nil {
|
||||
observability.LoggerFrom(ctx).Warn("entity matching failed", "error", err.Error())
|
||||
}
|
||||
|
||||
// An empty query with filters is a legitimate browse request.
|
||||
if trimmed == "" && len(cands) == 0 {
|
||||
items, total, err := s.items.List(ctx, p.WorkspaceID, itemFilter,
|
||||
domain.Page{Limit: limit, Offset: offset})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for i, it := range items {
|
||||
c := get(it.ID)
|
||||
c.CreatedAt = it.CreatedAt
|
||||
c.Add(ranking.SourceLexical, i+1, "Matched your filters.")
|
||||
}
|
||||
resp.Total = int(total)
|
||||
}
|
||||
|
||||
if len(cands) == 0 {
|
||||
resp.TookMS = time.Since(started).Milliseconds()
|
||||
s.count("hybrid", "empty")
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
// Load the candidate items, applying the workspace and filter constraints
|
||||
// that the vector store cannot express.
|
||||
ids := make([]string, 0, len(cands))
|
||||
for id := range cands {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
items, err := s.items.GetMany(ctx, p.WorkspaceID, ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
byID := make(map[string]*domain.Item, len(items))
|
||||
for _, it := range items {
|
||||
if it.Deleted || !matchesFilter(it, itemFilter, q) {
|
||||
delete(cands, it.ID)
|
||||
continue
|
||||
}
|
||||
byID[it.ID] = it
|
||||
cands[it.ID].CreatedAt = it.CreatedAt
|
||||
}
|
||||
// Drop candidates whose items were not returned at all: they are deleted,
|
||||
// in another workspace, or otherwise not visible to this principal.
|
||||
for id := range cands {
|
||||
if byID[id] == nil {
|
||||
delete(cands, id)
|
||||
}
|
||||
}
|
||||
|
||||
// Title matching is cheap and a strong signal a user expects to work.
|
||||
if trimmed != "" {
|
||||
lower := strings.ToLower(trimmed)
|
||||
for id, it := range byID {
|
||||
if strings.Contains(strings.ToLower(it.Title), lower) {
|
||||
cands[id].AddScaled(ranking.SourceTitle, 1,
|
||||
fmt.Sprintf("The title %q contains your query.", it.Title))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Path 5: relationship expansion ------------------------------------
|
||||
expansions := map[string]*RelatedItem{}
|
||||
relsByItem := map[string][]*domain.Relationship{}
|
||||
if q.Expand {
|
||||
s.expand(ctx, p.WorkspaceID, q, cands, byID, get, expansions, relsByItem)
|
||||
|
||||
// Newly-expanded items still need their records.
|
||||
var missing []string
|
||||
for id := range cands {
|
||||
if byID[id] == nil {
|
||||
missing = append(missing, id)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
more, err := s.items.GetMany(ctx, p.WorkspaceID, missing)
|
||||
if err == nil {
|
||||
for _, it := range more {
|
||||
if it.Deleted || !matchesFilter(it, itemFilter, q) {
|
||||
delete(cands, it.ID)
|
||||
continue
|
||||
}
|
||||
byID[it.ID] = it
|
||||
cands[it.ID].CreatedAt = it.CreatedAt
|
||||
}
|
||||
}
|
||||
for _, id := range missing {
|
||||
if byID[id] == nil {
|
||||
delete(cands, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Fuse and order -----------------------------------------------------
|
||||
list := make([]*ranking.Candidate, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
list = append(list, c)
|
||||
}
|
||||
ranked := ranking.Rank(list, time.Now(), recencyHalfLife)
|
||||
|
||||
if resp.Total == 0 {
|
||||
resp.Total = len(ranked)
|
||||
}
|
||||
|
||||
// Apply paging after ranking, so a page boundary does not change ordering.
|
||||
start := offset
|
||||
if start > len(ranked) {
|
||||
start = len(ranked)
|
||||
}
|
||||
end := start + limit
|
||||
if end > len(ranked) {
|
||||
end = len(ranked)
|
||||
}
|
||||
page := ranked[start:end]
|
||||
|
||||
// Fetch the chunks needed for excerpts, in one query.
|
||||
var wanted []string
|
||||
for _, c := range page {
|
||||
wanted = append(wanted, c.MatchedChunks...)
|
||||
}
|
||||
chunkByID := map[string]*domain.Chunk{}
|
||||
if len(wanted) > 0 {
|
||||
got, err := s.chunks.GetMany(ctx, p.WorkspaceID, wanted)
|
||||
if err == nil {
|
||||
for _, ch := range got {
|
||||
chunkByID[ch.ID] = ch
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, c := range page {
|
||||
it := byID[c.ItemID]
|
||||
if it == nil {
|
||||
continue
|
||||
}
|
||||
r := Result{
|
||||
Item: it,
|
||||
QueryRelevance: c.Relevance,
|
||||
SemanticSimilarity: c.BestSimilarity,
|
||||
WhyMatched: ranking.Explain(c),
|
||||
Contributions: c.Contributions,
|
||||
RankVersion: ranking.RankVersion,
|
||||
}
|
||||
|
||||
for _, cid := range c.MatchedChunks {
|
||||
ch := chunkByID[cid]
|
||||
if ch == nil {
|
||||
continue
|
||||
}
|
||||
r.MatchedChunks = append(r.MatchedChunks, MatchedChunk{
|
||||
ChunkID: ch.ID,
|
||||
Sequence: ch.Sequence,
|
||||
Text: ch.Text,
|
||||
Similarity: chunkSims[ch.ID],
|
||||
Page: ch.Page,
|
||||
Section: ch.Section,
|
||||
})
|
||||
}
|
||||
sort.SliceStable(r.MatchedChunks, func(i, j int) bool {
|
||||
return r.MatchedChunks[i].Similarity > r.MatchedChunks[j].Similarity
|
||||
})
|
||||
|
||||
r.Excerpt = buildExcerpt(it, r.MatchedChunks, chunkHits[it.ID], trimmed)
|
||||
|
||||
for _, e := range matchedEntities[it.ID] {
|
||||
r.MatchedEntities = append(r.MatchedEntities, MatchedEntity{
|
||||
ID: e.ID, Name: e.CanonicalName, Type: e.Type,
|
||||
})
|
||||
}
|
||||
|
||||
if via, ok := expansions[it.ID]; ok {
|
||||
r.ViaRelationship = via
|
||||
}
|
||||
r.Related = s.relatedFor(it.ID, relsByItem[it.ID], byID)
|
||||
|
||||
resp.Results = append(resp.Results, r)
|
||||
}
|
||||
|
||||
resp.TookMS = time.Since(started).Milliseconds()
|
||||
s.count("hybrid", "success")
|
||||
if s.metrics != nil {
|
||||
s.metrics.SearchLatency.Observe(time.Since(started).Seconds())
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *Service) count(mode, outcome string) {
|
||||
if s.metrics != nil {
|
||||
s.metrics.SearchOps.WithLabelValues(mode, outcome).Inc()
|
||||
}
|
||||
}
|
||||
|
||||
// matchEntities finds entities the query names and adds the items citing them.
|
||||
func (s *Service) matchEntities(ctx context.Context, workspaceID string, q Query, text string,
|
||||
get func(string) *ranking.Candidate, out map[string][]*domain.Entity) error {
|
||||
|
||||
var found []*domain.Entity
|
||||
|
||||
// Explicit entity filters are authoritative.
|
||||
if len(q.EntityIDs) > 0 {
|
||||
ents, err := s.entities.GetMany(ctx, workspaceID, q.EntityIDs)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = append(found, ents...)
|
||||
}
|
||||
|
||||
// Otherwise look for entity names appearing in the query text.
|
||||
if text != "" {
|
||||
keys := candidateEntityKeys(text)
|
||||
if len(keys) > 0 {
|
||||
ents, err := s.entities.FindByNormalised(ctx, workspaceID, keys)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
found = append(found, ents...)
|
||||
}
|
||||
}
|
||||
|
||||
seen := map[string]bool{}
|
||||
for _, e := range found {
|
||||
if seen[e.ID] {
|
||||
continue
|
||||
}
|
||||
seen[e.ID] = true
|
||||
for _, itemID := range e.ItemIDs {
|
||||
c := get(itemID)
|
||||
c.AddScaled(ranking.SourceEntity, 1,
|
||||
fmt.Sprintf("This item refers to %s (%s), which your query names.", e.CanonicalName, e.Type))
|
||||
c.MatchedEntities = appendUnique(c.MatchedEntities, e.ID, 8)
|
||||
out[itemID] = append(out[itemID], e)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// candidateEntityKeys generates normalised keys for word sequences in the
|
||||
// query, so "Example Ltd" matches the stored organisation.
|
||||
//
|
||||
// Only short n-grams are tried, and each becomes a value in an indexed $in
|
||||
// lookup — never a pattern or an operator.
|
||||
func candidateEntityKeys(text string) []string {
|
||||
words := strings.FieldsFunc(text, func(r rune) bool {
|
||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && r != '\'' && r != '&'
|
||||
})
|
||||
if len(words) == 0 || len(words) > 40 {
|
||||
return nil
|
||||
}
|
||||
|
||||
var keys []string
|
||||
seen := map[string]bool{}
|
||||
addKey := func(phrase string) {
|
||||
norm := entities.Normalise(phrase)
|
||||
if norm == "" || len(norm) < 3 {
|
||||
return
|
||||
}
|
||||
for _, t := range domain.ValidEntityTypes {
|
||||
k := string(t) + ":" + norm
|
||||
if !seen[k] {
|
||||
seen[k] = true
|
||||
keys = append(keys, k)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Unigrams through trigrams.
|
||||
for n := 1; n <= 3; n++ {
|
||||
for i := 0; i+n <= len(words); i++ {
|
||||
if len(keys) > 600 {
|
||||
return keys
|
||||
}
|
||||
addKey(strings.Join(words[i:i+n], " "))
|
||||
}
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// expand pulls in items connected to the current matches by a relationship.
|
||||
//
|
||||
// Expanded items are labelled with the relationship that brought them in, and
|
||||
// their contribution is weighted well below a direct match: being related to a
|
||||
// hit is weaker evidence than being one. Rejected relationships never expand.
|
||||
func (s *Service) expand(ctx context.Context, workspaceID string, q Query,
|
||||
cands map[string]*ranking.Candidate, byID map[string]*domain.Item,
|
||||
get func(string) *ranking.Candidate, expansions map[string]*RelatedItem,
|
||||
relsByItem map[string][]*domain.Relationship) {
|
||||
|
||||
// Expand only from the strongest current matches.
|
||||
seeds := make([]*ranking.Candidate, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
seeds = append(seeds, c)
|
||||
}
|
||||
sort.SliceStable(seeds, func(i, j int) bool {
|
||||
return len(seeds[i].Contributions) > len(seeds[j].Contributions)
|
||||
})
|
||||
max := s.cfg.Search.MaxExpandedItems
|
||||
if max <= 0 {
|
||||
max = 20
|
||||
}
|
||||
if len(seeds) > max {
|
||||
seeds = seeds[:max]
|
||||
}
|
||||
|
||||
seedIDs := make([]string, 0, len(seeds))
|
||||
for _, c := range seeds {
|
||||
seedIDs = append(seedIDs, c.ItemID)
|
||||
}
|
||||
|
||||
minScore := q.MinRelStrength
|
||||
if minScore <= 0 {
|
||||
minScore = s.cfg.Search.ExpansionMinScore
|
||||
}
|
||||
|
||||
rels, err := s.rels.ListForItems(ctx, workspaceID, seedIDs, minScore)
|
||||
if err != nil {
|
||||
observability.LoggerFrom(ctx).Warn("relationship expansion failed", "error", err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
isSeed := map[string]bool{}
|
||||
for _, id := range seedIDs {
|
||||
isSeed[id] = true
|
||||
}
|
||||
|
||||
added := 0
|
||||
for _, rel := range rels {
|
||||
if q.ConfirmedOnly && rel.ReviewStatus != domain.ReviewConfirmed {
|
||||
continue
|
||||
}
|
||||
if len(q.RelTypes) > 0 && !containsRelType(q.RelTypes, rel.Type) {
|
||||
continue
|
||||
}
|
||||
|
||||
relsByItem[rel.SourceID] = append(relsByItem[rel.SourceID], rel)
|
||||
relsByItem[rel.TargetID] = append(relsByItem[rel.TargetID], rel)
|
||||
|
||||
// Identify the far end of the relationship.
|
||||
var otherID, direction string
|
||||
switch {
|
||||
case isSeed[rel.SourceID] && !isSeed[rel.TargetID]:
|
||||
otherID, direction = rel.TargetID, "outgoing"
|
||||
case isSeed[rel.TargetID] && !isSeed[rel.SourceID]:
|
||||
otherID, direction = rel.SourceID, "incoming"
|
||||
default:
|
||||
continue // both ends already matched, or neither did
|
||||
}
|
||||
if added >= max {
|
||||
break
|
||||
}
|
||||
added++
|
||||
|
||||
c := get(otherID)
|
||||
c.ExpandedFrom = rel.SourceID
|
||||
label := "related to"
|
||||
if rel.ReviewStatus == domain.ReviewConfirmed {
|
||||
label = "confirmed as related to"
|
||||
c.AddScaled(ranking.SourceConfirmedRel, 1,
|
||||
"A person has confirmed this item's relationship to a matching item.")
|
||||
}
|
||||
c.AddScaled(ranking.SourceExpansion, rel.Score, fmt.Sprintf(
|
||||
"Not a direct match: this item is %s a matching item by a %s relationship (strength %.2f, %s).",
|
||||
label, rel.Type, rel.Score, provenanceLabel(rel.Provenance)))
|
||||
|
||||
expansions[otherID] = &RelatedItem{
|
||||
RelationshipID: rel.ID,
|
||||
ItemID: otherID,
|
||||
Type: rel.Type,
|
||||
Provenance: rel.Provenance,
|
||||
Strength: rel.Score,
|
||||
ScoreVersion: rel.ScoreVersion,
|
||||
ReviewStatus: rel.ReviewStatus,
|
||||
Direction: direction,
|
||||
Explanation: relationships.Explain(rel),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// relatedFor renders an item's relationships for display.
|
||||
func (s *Service) relatedFor(itemID string, rels []*domain.Relationship, byID map[string]*domain.Item) []RelatedItem {
|
||||
if len(rels) == 0 {
|
||||
return nil
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
out := make([]RelatedItem, 0, len(rels))
|
||||
|
||||
for _, rel := range rels {
|
||||
if seen[rel.ID] {
|
||||
continue
|
||||
}
|
||||
seen[rel.ID] = true
|
||||
|
||||
otherID, direction := rel.TargetID, "outgoing"
|
||||
if rel.TargetID == itemID {
|
||||
otherID, direction = rel.SourceID, "incoming"
|
||||
}
|
||||
title := ""
|
||||
if it := byID[otherID]; it != nil {
|
||||
title = it.Title
|
||||
}
|
||||
out = append(out, RelatedItem{
|
||||
RelationshipID: rel.ID,
|
||||
ItemID: otherID,
|
||||
Title: title,
|
||||
Type: rel.Type,
|
||||
Provenance: rel.Provenance,
|
||||
Strength: rel.Score,
|
||||
ScoreVersion: rel.ScoreVersion,
|
||||
ReviewStatus: rel.ReviewStatus,
|
||||
Direction: direction,
|
||||
Explanation: relationships.Explain(rel),
|
||||
})
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool { return out[i].Strength > out[j].Strength })
|
||||
if len(out) > 8 {
|
||||
out = out[:8]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// buildExcerpt picks the most useful snippet to show for a result.
|
||||
func buildExcerpt(item *domain.Item, matched []MatchedChunk, lexical []domain.ChunkHit, query string) string {
|
||||
const width = 320
|
||||
|
||||
// Prefer a chunk that lexically contains a query term, so the excerpt shows
|
||||
// the user their own words where possible.
|
||||
if query != "" {
|
||||
terms := strings.Fields(strings.ToLower(query))
|
||||
for _, mc := range matched {
|
||||
if snippet := findTermWindow(mc.Text, terms, width); snippet != "" {
|
||||
return snippet
|
||||
}
|
||||
}
|
||||
for _, lh := range lexical {
|
||||
if snippet := findTermWindow(lh.Text, terms, width); snippet != "" {
|
||||
return snippet
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(matched) > 0 {
|
||||
return truncateWords(matched[0].Text, width)
|
||||
}
|
||||
if len(lexical) > 0 {
|
||||
return truncateWords(lexical[0].Text, width)
|
||||
}
|
||||
return truncateWords(item.ExtractedText, width)
|
||||
}
|
||||
|
||||
// findTermWindow returns a window of text around the first matching term.
|
||||
func findTermWindow(text string, terms []string, width int) string {
|
||||
lower := strings.ToLower(text)
|
||||
best := -1
|
||||
for _, t := range terms {
|
||||
if len(t) < 3 {
|
||||
continue
|
||||
}
|
||||
if i := strings.Index(lower, t); i >= 0 && (best < 0 || i < best) {
|
||||
best = i
|
||||
}
|
||||
}
|
||||
if best < 0 {
|
||||
return ""
|
||||
}
|
||||
start := best - width/3
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
for start > 0 && text[start]&0xC0 == 0x80 {
|
||||
start--
|
||||
}
|
||||
end := start + width
|
||||
if end > len(text) {
|
||||
end = len(text)
|
||||
}
|
||||
for end < len(text) && text[end]&0xC0 == 0x80 {
|
||||
end++
|
||||
}
|
||||
out := strings.TrimSpace(collapse(text[start:end]))
|
||||
if start > 0 {
|
||||
out = "…" + out
|
||||
}
|
||||
if end < len(text) {
|
||||
out += "…"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func truncateWords(s string, n int) string {
|
||||
s = collapse(strings.TrimSpace(s))
|
||||
if len(s) <= n {
|
||||
return s
|
||||
}
|
||||
cut := n
|
||||
for cut > 0 && s[cut]&0xC0 == 0x80 {
|
||||
cut--
|
||||
}
|
||||
if i := strings.LastIndexByte(s[:cut], ' '); i > n/2 {
|
||||
cut = i
|
||||
}
|
||||
return s[:cut] + "…"
|
||||
}
|
||||
|
||||
func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
|
||||
|
||||
// matchesFilter applies the filters the storage query could not express.
|
||||
func matchesFilter(it *domain.Item, f domain.ItemFilter, q Query) bool {
|
||||
if len(f.SourceTypes) > 0 && !containsSourceType(f.SourceTypes, it.SourceType) {
|
||||
return false
|
||||
}
|
||||
if len(f.MIMETypes) > 0 && !containsString(f.MIMETypes, it.MIMEType) {
|
||||
return false
|
||||
}
|
||||
if f.CreatedFrom != nil && it.CreatedAt.Before(*f.CreatedFrom) {
|
||||
return false
|
||||
}
|
||||
if f.CreatedTo != nil && it.CreatedAt.After(*f.CreatedTo) {
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func containsSourceType(list []domain.SourceType, v domain.SourceType) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsString(list []string, v string) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func containsRelType(list []domain.RelationshipType, v domain.RelationshipType) bool {
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func appendUnique(list []string, v string, max int) []string {
|
||||
if v == "" || len(list) >= max {
|
||||
return list
|
||||
}
|
||||
for _, e := range list {
|
||||
if e == v {
|
||||
return list
|
||||
}
|
||||
}
|
||||
return append(list, v)
|
||||
}
|
||||
|
||||
func provenanceLabel(p domain.Provenance) string {
|
||||
switch p {
|
||||
case domain.ProvExplicit:
|
||||
return "stated explicitly"
|
||||
case domain.ProvDeterministic:
|
||||
return "derived from matching evidence"
|
||||
case domain.ProvModelInferred:
|
||||
return "suggested by analysis, not yet reviewed"
|
||||
case domain.ProvUserConfirmed:
|
||||
return "confirmed by a person"
|
||||
case domain.ProvUserRejected:
|
||||
return "rejected by a person"
|
||||
}
|
||||
return string(p)
|
||||
}
|
||||
|
||||
func errString(err error) string {
|
||||
if err == nil {
|
||||
return ""
|
||||
}
|
||||
return err.Error()
|
||||
}
|
||||
Reference in New Issue
Block a user