Code sync, security fixes
This commit is contained in:
Andy Dixon
@@ -0,0 +1,305 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/relata/internal/analysis"
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
"github.com/andydixon/relata/internal/observability"
|
||||
)
|
||||
|
||||
// AnswerRequest asks a question of the stored material.
|
||||
type AnswerRequest struct {
|
||||
Question string
|
||||
Query
|
||||
}
|
||||
|
||||
// Answer is a generated answer, restricted to stored material and cited.
|
||||
type Answer struct {
|
||||
Question string `json:"question"`
|
||||
// Answer is the generated prose. It is rendered as escaped text, never HTML.
|
||||
Answer string `json:"answer"`
|
||||
// Sufficient reports whether the stored material actually supports an
|
||||
// answer. When false, the answer says so rather than guessing.
|
||||
Sufficient bool `json:"sufficient"`
|
||||
|
||||
// Sources used — the stored items the answer draws on.
|
||||
Sources []AnswerSource `json:"sources_used"`
|
||||
// Uncertainty and conflicts — disagreements between sources, and any
|
||||
// caveats about what the answer does and does not establish.
|
||||
Uncertainty []string `json:"uncertainty_and_conflicts,omitempty"`
|
||||
|
||||
// InferredRelationshipsUsed lists links that informed retrieval but are
|
||||
// model proposals, not verified facts.
|
||||
InferredRelationshipsUsed []RelatedItem `json:"inferred_relationships_used,omitempty"`
|
||||
|
||||
Model string `json:"model,omitempty"`
|
||||
Retrieved int `json:"retrieved_items"`
|
||||
TookMS int64 `json:"took_ms"`
|
||||
// InjectionNoticed reports that stored content tried to instruct the model.
|
||||
InjectionNoticed bool `json:"prompt_injection_noticed,omitempty"`
|
||||
}
|
||||
|
||||
// AnswerSource is one cited stored item.
|
||||
type AnswerSource struct {
|
||||
Ref string `json:"ref"`
|
||||
ItemID string `json:"item_id"`
|
||||
Title string `json:"title"`
|
||||
Excerpt string `json:"excerpt"`
|
||||
Date string `json:"date,omitempty"`
|
||||
// Cited reports whether the generated answer actually referenced it.
|
||||
Cited bool `json:"cited"`
|
||||
}
|
||||
|
||||
// ErrAnswerDisabled is returned when answer generation is switched off.
|
||||
var ErrAnswerDisabled = fmt.Errorf("answer generation is not enabled")
|
||||
|
||||
// Answerer generates answers from stored material.
|
||||
type Answerer struct {
|
||||
search *Service
|
||||
analysis *analysis.Service
|
||||
claims domain.ClaimRepository
|
||||
enabled bool
|
||||
}
|
||||
|
||||
// NewAnswerer builds the answer service.
|
||||
func NewAnswerer(s *Service, an *analysis.Service, claims domain.ClaimRepository, enabled bool) *Answerer {
|
||||
return &Answerer{search: s, analysis: an, claims: claims, enabled: enabled}
|
||||
}
|
||||
|
||||
// Enabled reports whether answers can be generated.
|
||||
func (a *Answerer) Enabled() bool {
|
||||
return a != nil && a.enabled && a.analysis.Enabled()
|
||||
}
|
||||
|
||||
// Answer retrieves relevant material and generates a cited answer from it.
|
||||
//
|
||||
// Several guarantees hold regardless of what the model returns:
|
||||
// - the answer is built only from retrieved passages;
|
||||
// - citations are resolved against the passages actually supplied, so a
|
||||
// fabricated reference cannot point at a real item;
|
||||
// - conflicting sources are surfaced, never reconciled away;
|
||||
// - any inferred relationship that influenced retrieval is listed as a
|
||||
// proposal rather than presented as fact.
|
||||
func (a *Answerer) Answer(ctx context.Context, p *domain.Principal, req AnswerRequest) (*Answer, error) {
|
||||
if !a.Enabled() {
|
||||
return nil, ErrAnswerDisabled
|
||||
}
|
||||
started := time.Now()
|
||||
|
||||
question := strings.TrimSpace(req.Question)
|
||||
if question == "" {
|
||||
return nil, domain.ErrInvalidInput
|
||||
}
|
||||
|
||||
q := req.Query
|
||||
q.Text = question
|
||||
q.Expand = true
|
||||
if q.Limit <= 0 || q.Limit > a.search.cfg.Search.AnswerMaxChunks {
|
||||
q.Limit = a.search.cfg.Search.AnswerMaxChunks
|
||||
}
|
||||
|
||||
results, err := a.search.Search(ctx, p, q)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out := &Answer{Question: question, Retrieved: len(results.Results)}
|
||||
|
||||
if len(results.Results) == 0 {
|
||||
out.Answer = "There is nothing in the stored material that addresses this question. " +
|
||||
"Nothing has been inferred or supplied from outside the stored items."
|
||||
out.Sufficient = false
|
||||
out.TookMS = time.Since(started).Milliseconds()
|
||||
a.search.count("answer", "no_material")
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Build the passages. The model sees opaque refs (S1, S2…) rather than item
|
||||
// IDs, so it cannot invent a citation that resolves to a real record.
|
||||
maxChars := a.search.cfg.Search.AnswerMaxCharsPerChunk
|
||||
if maxChars <= 0 {
|
||||
maxChars = 1200
|
||||
}
|
||||
passages := make([]analysis.Passage, 0, len(results.Results))
|
||||
for i, r := range results.Results {
|
||||
ref := fmt.Sprintf("S%d", i+1)
|
||||
excerpt := r.Excerpt
|
||||
if len(r.MatchedChunks) > 0 {
|
||||
// The matched chunk carries more context than the short excerpt.
|
||||
excerpt = truncateWords(r.MatchedChunks[0].Text, maxChars)
|
||||
}
|
||||
if strings.TrimSpace(excerpt) == "" {
|
||||
excerpt = truncateWords(r.Item.ExtractedText, maxChars)
|
||||
}
|
||||
if strings.TrimSpace(excerpt) == "" {
|
||||
continue
|
||||
}
|
||||
|
||||
date := ""
|
||||
if !r.Item.CreatedAt.IsZero() {
|
||||
date = r.Item.CreatedAt.Format("2 January 2006")
|
||||
}
|
||||
passages = append(passages, analysis.Passage{
|
||||
Ref: ref, ItemID: r.Item.ID, Title: r.Item.Title, Excerpt: excerpt, Date: date,
|
||||
})
|
||||
out.Sources = append(out.Sources, AnswerSource{
|
||||
Ref: ref, ItemID: r.Item.ID, Title: r.Item.Title, Excerpt: excerpt, Date: date,
|
||||
})
|
||||
|
||||
// Record any inferred relationship that pulled this item in, so the
|
||||
// answer can disclose that it was reached by a proposal.
|
||||
if r.ViaRelationship != nil && r.ViaRelationship.Provenance == domain.ProvModelInferred {
|
||||
out.InferredRelationshipsUsed = append(out.InferredRelationshipsUsed, *r.ViaRelationship)
|
||||
}
|
||||
}
|
||||
|
||||
if len(passages) == 0 {
|
||||
out.Answer = "Items were found, but none contained readable text to answer from."
|
||||
out.Sufficient = false
|
||||
out.TookMS = time.Since(started).Milliseconds()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
generated, err := a.analysis.GenerateAnswer(ctx, question, passages)
|
||||
if err != nil {
|
||||
// The truncated sample of what the model actually said is the whole
|
||||
// diagnostic here: "rejected as degenerate" tells an operator nothing
|
||||
// about whether the model is broken, the prompt is wrong, or the check
|
||||
// is too strict. It is already bounded to 300 characters.
|
||||
log := observability.LoggerFrom(ctx)
|
||||
var sampled *analysis.InvalidOutputError
|
||||
if errors.As(err, &sampled) {
|
||||
log.Warn("answer generation failed",
|
||||
"error", err.Error(), "reason", sampled.Reason, "model_output", sampled.Sample)
|
||||
} else {
|
||||
log.Warn("answer generation failed", "error", err.Error())
|
||||
}
|
||||
|
||||
// A model that returned something unusable is not a failed request.
|
||||
// Retrieval already found the material and Relata has already recorded
|
||||
// any contradictions between those items — the deterministic work is
|
||||
// the part worth keeping. Report that no prose could be generated and
|
||||
// hand back the sources and conflicts, as the no-material and
|
||||
// no-readable-text paths above do.
|
||||
var invalid *analysis.InvalidOutputError
|
||||
if errors.As(err, &invalid) {
|
||||
a.search.count("answer", "invalid_output")
|
||||
out.Answer = "The answering model did not return a usable answer, so none is shown. " +
|
||||
"The sources below were retrieved for this question and can be read directly; " +
|
||||
"nothing has been inferred or supplied from outside the stored items."
|
||||
out.Sufficient = false
|
||||
out.Uncertainty = append(out.Uncertainty,
|
||||
"No answer was generated: the model's response was rejected as invalid "+
|
||||
"("+invalid.Reason+"). The retrieved sources are unaffected.")
|
||||
if conflicts := a.storedConflicts(ctx, p, results.Results); len(conflicts) > 0 {
|
||||
out.Uncertainty = append(out.Uncertainty, conflicts...)
|
||||
}
|
||||
out.TookMS = time.Since(started).Milliseconds()
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Anything else — a timeout, an unreachable provider — is an
|
||||
// operational condition the caller needs to see as a failure.
|
||||
a.search.count("answer", "error")
|
||||
return nil, err
|
||||
}
|
||||
|
||||
out.Answer = generated.Answer
|
||||
out.Sufficient = generated.Sufficient
|
||||
out.Model = generated.Model
|
||||
out.InjectionNoticed = generated.InjectionNoticed
|
||||
|
||||
cited := map[string]bool{}
|
||||
for _, c := range generated.Citations {
|
||||
cited[c.Ref] = true
|
||||
}
|
||||
for i := range out.Sources {
|
||||
out.Sources[i].Cited = cited[out.Sources[i].Ref]
|
||||
}
|
||||
|
||||
out.Uncertainty = append(out.Uncertainty, generated.Conflicts...)
|
||||
|
||||
// Independently of what the model said, surface contradictions Relata has
|
||||
// recorded between the retrieved items. A model that quietly picked one
|
||||
// side must not be able to hide the disagreement.
|
||||
if conflicts := a.storedConflicts(ctx, p, results.Results); len(conflicts) > 0 {
|
||||
out.Uncertainty = append(out.Uncertainty, conflicts...)
|
||||
}
|
||||
|
||||
if !out.Sufficient {
|
||||
out.Uncertainty = append(out.Uncertainty,
|
||||
"The stored material does not fully answer this question. Treat the answer above as partial.")
|
||||
}
|
||||
if len(out.InferredRelationshipsUsed) > 0 {
|
||||
out.Uncertainty = append(out.Uncertainty, fmt.Sprintf(
|
||||
"%d of the sources were reached through relationships that Relata inferred and no one has confirmed. "+
|
||||
"An inferred relationship is a proposal, not an established fact.",
|
||||
len(out.InferredRelationshipsUsed)))
|
||||
}
|
||||
if generated.InjectionNoticed {
|
||||
out.Uncertainty = append(out.Uncertainty,
|
||||
"One or more stored documents contained text that attempted to instruct the answering model. "+
|
||||
"Those instructions were ignored, but the affected sources should be reviewed.")
|
||||
}
|
||||
|
||||
out.TookMS = time.Since(started).Milliseconds()
|
||||
a.search.count("answer", "success")
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// storedConflicts reports contradictions Relata recorded between the retrieved
|
||||
// items, independently of the model's own account.
|
||||
func (a *Answerer) storedConflicts(ctx context.Context, p *domain.Principal, results []Result) []string {
|
||||
if len(results) < 2 {
|
||||
return nil
|
||||
}
|
||||
ids := make([]string, 0, len(results))
|
||||
titles := make(map[string]string, len(results))
|
||||
for _, r := range results {
|
||||
ids = append(ids, r.Item.ID)
|
||||
titles[r.Item.ID] = r.Item.Title
|
||||
}
|
||||
|
||||
rels, err := a.search.rels.ListForItems(ctx, p.WorkspaceID, ids, 0)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var out []string
|
||||
seen := map[string]bool{}
|
||||
for _, rel := range rels {
|
||||
if rel.Type != domain.RelContradicts || rel.ReviewStatus == domain.ReviewRejected {
|
||||
continue
|
||||
}
|
||||
// Both ends must be in the retrieved set for the conflict to be
|
||||
// relevant to this answer.
|
||||
srcTitle, okA := titles[rel.SourceID]
|
||||
tgtTitle, okB := titles[rel.TargetID]
|
||||
if !okA || !okB || seen[rel.ID] {
|
||||
continue
|
||||
}
|
||||
seen[rel.ID] = true
|
||||
|
||||
detail := ""
|
||||
for _, s := range rel.Evidence.Signals {
|
||||
if s.Type == domain.SigClaimContradiction {
|
||||
detail = " " + s.Detail
|
||||
break
|
||||
}
|
||||
}
|
||||
out = append(out, fmt.Sprintf(
|
||||
"%q and %q record conflicting information (relationship strength %.2f, %s).%s",
|
||||
srcTitle, tgtTitle, rel.Score, provenanceLabel(rel.Provenance), detail))
|
||||
}
|
||||
sort.Strings(out)
|
||||
if len(out) > 6 {
|
||||
out = out[:6]
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/andydixon/relata/internal/domain"
|
||||
)
|
||||
|
||||
// Probe the pure helpers with adversarial input.
|
||||
func TestExcerptHelpersOnAdversarialInput(t *testing.T) {
|
||||
inputs := []string{"", " ", "\x00", "é", "ééééééééééé", "a\nb\tc", "🌍🌍🌍"}
|
||||
for _, in := range inputs {
|
||||
for _, n := range []int{0, 1, 2, 5, 1000} {
|
||||
got := truncateWords(in, n)
|
||||
for _, r := range got {
|
||||
if r == '�' {
|
||||
t.Fatalf("truncateWords(%q,%d) produced a replacement char: %q", in, n, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFindTermWindowOnAdversarialInput(t *testing.T) {
|
||||
texts := []string{"", "short", "ééééé match ééééé", "🌍 match 🌍"}
|
||||
for _, txt := range texts {
|
||||
for _, terms := range [][]string{nil, {}, {""}, {"match"}, {"zzz"}, {"é"}} {
|
||||
got := findTermWindow(txt, terms, 20)
|
||||
for _, r := range got {
|
||||
if r == '�' {
|
||||
t.Fatalf("findTermWindow(%q,%v) produced a replacement char: %q", txt, terms, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMatchesFilterHandlesZeroValues(t *testing.T) {
|
||||
it := &domain.Item{}
|
||||
if !matchesFilter(it, domain.ItemFilter{}, Query{}) {
|
||||
t.Fatal("an empty filter rejected an item")
|
||||
}
|
||||
future := time.Now().Add(time.Hour)
|
||||
if matchesFilter(it, domain.ItemFilter{CreatedFrom: &future}, Query{}) {
|
||||
t.Fatal("a zero-time item passed a future CreatedFrom filter")
|
||||
}
|
||||
}
|
||||
|
||||
// Excerpt building slices UTF-8 text at computed offsets, so it is exactly the
|
||||
// kind of code that produces mangled output on unusual input.
|
||||
func FuzzExcerptHelpers(f *testing.F) {
|
||||
f.Add("The revised delivery date is 14 March 2025.", "delivery", 40)
|
||||
f.Add("ééééé", "é", 3)
|
||||
f.Add("🌍🌍🌍", "🌍", 2)
|
||||
f.Add("", "", 0)
|
||||
|
||||
f.Fuzz(func(t *testing.T, text, term string, width int) {
|
||||
if width < 0 || width > 1<<16 {
|
||||
return
|
||||
}
|
||||
// Slicing valid UTF-8 must not produce invalid UTF-8.
|
||||
if !utf8.ValidString(text) {
|
||||
return
|
||||
}
|
||||
for name, got := range map[string]string{
|
||||
"truncateWords": truncateWords(text, width),
|
||||
"findTermWindow": findTermWindow(text, []string{term}, width),
|
||||
"collapse": collapse(text),
|
||||
} {
|
||||
if !utf8.ValidString(got) {
|
||||
t.Fatalf("%s produced invalid UTF-8 from text=%q term=%q width=%d -> %q",
|
||||
name, text, term, width, got)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user