relata/internal/search/answer.go

package search

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"strings"
	"time"

	"github.com/andydixon/relata/internal/analysis"
	"github.com/andydixon/relata/internal/domain"
	"github.com/andydixon/relata/internal/observability"
)

// AnswerRequest asks a question of the stored material.
type AnswerRequest struct {
	Question string
	Query
}

// Answer is a generated answer, restricted to stored material and cited.
type Answer struct {
	Question string `json:"question"`
	// Answer is the generated prose. It is rendered as escaped text, never HTML.
	Answer string `json:"answer"`
	// Sufficient reports whether the stored material actually supports an
	// answer. When false, the answer says so rather than guessing.
	Sufficient bool `json:"sufficient"`

	// Sources used — the stored items the answer draws on.
	Sources []AnswerSource `json:"sources_used"`
	// Uncertainty and conflicts — disagreements between sources, and any
	// caveats about what the answer does and does not establish.
	Uncertainty []string `json:"uncertainty_and_conflicts,omitempty"`

	// InferredRelationshipsUsed lists links that informed retrieval but are
	// model proposals, not verified facts.
	InferredRelationshipsUsed []RelatedItem `json:"inferred_relationships_used,omitempty"`

	Model     string `json:"model,omitempty"`
	Retrieved int    `json:"retrieved_items"`
	TookMS    int64  `json:"took_ms"`
	// InjectionNoticed reports that stored content tried to instruct the model.
	InjectionNoticed bool `json:"prompt_injection_noticed,omitempty"`
}

// AnswerSource is one cited stored item.
type AnswerSource struct {
	Ref     string `json:"ref"`
	ItemID  string `json:"item_id"`
	Title   string `json:"title"`
	Excerpt string `json:"excerpt"`
	Date    string `json:"date,omitempty"`
	// Cited reports whether the generated answer actually referenced it.
	Cited bool `json:"cited"`
}

// ErrAnswerDisabled is returned when answer generation is switched off.
var ErrAnswerDisabled = fmt.Errorf("answer generation is not enabled")

// Answerer generates answers from stored material.
type Answerer struct {
	search   *Service
	analysis *analysis.Service
	claims   domain.ClaimRepository
	enabled  bool
}

// NewAnswerer builds the answer service.
func NewAnswerer(s *Service, an *analysis.Service, claims domain.ClaimRepository, enabled bool) *Answerer {
	return &Answerer{search: s, analysis: an, claims: claims, enabled: enabled}
}

// Enabled reports whether answers can be generated.
func (a *Answerer) Enabled() bool {
	return a != nil && a.enabled && a.analysis.Enabled()
}

// Answer retrieves relevant material and generates a cited answer from it.
//
// Several guarantees hold regardless of what the model returns:
//   - the answer is built only from retrieved passages;
//   - citations are resolved against the passages actually supplied, so a
//     fabricated reference cannot point at a real item;
//   - conflicting sources are surfaced, never reconciled away;
//   - any inferred relationship that influenced retrieval is listed as a
//     proposal rather than presented as fact.
func (a *Answerer) Answer(ctx context.Context, p *domain.Principal, req AnswerRequest) (*Answer, error) {
	if !a.Enabled() {
		return nil, ErrAnswerDisabled
	}
	started := time.Now()

	question := strings.TrimSpace(req.Question)
	if question == "" {
		return nil, domain.ErrInvalidInput
	}

	q := req.Query
	q.Text = question
	q.Expand = true
	if q.Limit <= 0 || q.Limit > a.search.cfg.Search.AnswerMaxChunks {
		q.Limit = a.search.cfg.Search.AnswerMaxChunks
	}

	results, err := a.search.Search(ctx, p, q)
	if err != nil {
		return nil, err
	}

	out := &Answer{Question: question, Retrieved: len(results.Results)}

	if len(results.Results) == 0 {
		out.Answer = "There is nothing in the stored material that addresses this question. " +
			"Nothing has been inferred or supplied from outside the stored items."
		out.Sufficient = false
		out.TookMS = time.Since(started).Milliseconds()
		a.search.count("answer", "no_material")
		return out, nil
	}

	// Build the passages. The model sees opaque refs (S1, S2…) rather than item
	// IDs, so it cannot invent a citation that resolves to a real record.
	maxChars := a.search.cfg.Search.AnswerMaxCharsPerChunk
	if maxChars <= 0 {
		maxChars = 1200
	}
	passages := make([]analysis.Passage, 0, len(results.Results))
	for i, r := range results.Results {
		ref := fmt.Sprintf("S%d", i+1)
		excerpt := r.Excerpt
		if len(r.MatchedChunks) > 0 {
			// The matched chunk carries more context than the short excerpt.
			excerpt = truncateWords(r.MatchedChunks[0].Text, maxChars)
		}
		if strings.TrimSpace(excerpt) == "" {
			excerpt = truncateWords(r.Item.ExtractedText, maxChars)
		}
		if strings.TrimSpace(excerpt) == "" {
			continue
		}

		date := ""
		if !r.Item.CreatedAt.IsZero() {
			date = r.Item.CreatedAt.Format("2 January 2006")
		}
		passages = append(passages, analysis.Passage{
			Ref: ref, ItemID: r.Item.ID, Title: r.Item.Title, Excerpt: excerpt, Date: date,
		})
		out.Sources = append(out.Sources, AnswerSource{
			Ref: ref, ItemID: r.Item.ID, Title: r.Item.Title, Excerpt: excerpt, Date: date,
		})

		// Record any inferred relationship that pulled this item in, so the
		// answer can disclose that it was reached by a proposal.
		if r.ViaRelationship != nil && r.ViaRelationship.Provenance == domain.ProvModelInferred {
			out.InferredRelationshipsUsed = append(out.InferredRelationshipsUsed, *r.ViaRelationship)
		}
	}

	if len(passages) == 0 {
		out.Answer = "Items were found, but none contained readable text to answer from."
		out.Sufficient = false
		out.TookMS = time.Since(started).Milliseconds()
		return out, nil
	}

	generated, err := a.analysis.GenerateAnswer(ctx, question, passages)
	if err != nil {
		// The truncated sample of what the model actually said is the whole
		// diagnostic here: "rejected as degenerate" tells an operator nothing
		// about whether the model is broken, the prompt is wrong, or the check
		// is too strict. It is already bounded to 300 characters.
		log := observability.LoggerFrom(ctx)
		var sampled *analysis.InvalidOutputError
		if errors.As(err, &sampled) {
			log.Warn("answer generation failed",
				"error", err.Error(), "reason", sampled.Reason, "model_output", sampled.Sample)
		} else {
			log.Warn("answer generation failed", "error", err.Error())
		}

		// A model that returned something unusable is not a failed request.
		// Retrieval already found the material and Relata has already recorded
		// any contradictions between those items — the deterministic work is
		// the part worth keeping. Report that no prose could be generated and
		// hand back the sources and conflicts, as the no-material and
		// no-readable-text paths above do.
		var invalid *analysis.InvalidOutputError
		if errors.As(err, &invalid) {
			a.search.count("answer", "invalid_output")
			out.Answer = "The answering model did not return a usable answer, so none is shown. " +
				"The sources below were retrieved for this question and can be read directly; " +
				"nothing has been inferred or supplied from outside the stored items."
			out.Sufficient = false
			out.Uncertainty = append(out.Uncertainty,
				"No answer was generated: the model's response was rejected as invalid "+
					"("+invalid.Reason+"). The retrieved sources are unaffected.")
			if conflicts := a.storedConflicts(ctx, p, results.Results); len(conflicts) > 0 {
				out.Uncertainty = append(out.Uncertainty, conflicts...)
			}
			out.TookMS = time.Since(started).Milliseconds()
			return out, nil
		}

		// Anything else — a timeout, an unreachable provider — is an
		// operational condition the caller needs to see as a failure.
		a.search.count("answer", "error")
		return nil, err
	}

	out.Answer = generated.Answer
	out.Sufficient = generated.Sufficient
	out.Model = generated.Model
	out.InjectionNoticed = generated.InjectionNoticed

	cited := map[string]bool{}
	for _, c := range generated.Citations {
		cited[c.Ref] = true
	}
	for i := range out.Sources {
		out.Sources[i].Cited = cited[out.Sources[i].Ref]
	}

	out.Uncertainty = append(out.Uncertainty, generated.Conflicts...)

	// Independently of what the model said, surface contradictions Relata has
	// recorded between the retrieved items. A model that quietly picked one
	// side must not be able to hide the disagreement.
	if conflicts := a.storedConflicts(ctx, p, results.Results); len(conflicts) > 0 {
		out.Uncertainty = append(out.Uncertainty, conflicts...)
	}

	if !out.Sufficient {
		out.Uncertainty = append(out.Uncertainty,
			"The stored material does not fully answer this question. Treat the answer above as partial.")
	}
	if len(out.InferredRelationshipsUsed) > 0 {
		out.Uncertainty = append(out.Uncertainty, fmt.Sprintf(
			"%d of the sources were reached through relationships that Relata inferred and no one has confirmed. "+
				"An inferred relationship is a proposal, not an established fact.",
			len(out.InferredRelationshipsUsed)))
	}
	if generated.InjectionNoticed {
		out.Uncertainty = append(out.Uncertainty,
			"One or more stored documents contained text that attempted to instruct the answering model. "+
				"Those instructions were ignored, but the affected sources should be reviewed.")
	}

	out.TookMS = time.Since(started).Milliseconds()
	a.search.count("answer", "success")
	return out, nil
}

// storedConflicts reports contradictions Relata recorded between the retrieved
// items, independently of the model's own account.
func (a *Answerer) storedConflicts(ctx context.Context, p *domain.Principal, results []Result) []string {
	if len(results) < 2 {
		return nil
	}
	ids := make([]string, 0, len(results))
	titles := make(map[string]string, len(results))
	for _, r := range results {
		ids = append(ids, r.Item.ID)
		titles[r.Item.ID] = r.Item.Title
	}

	rels, err := a.search.rels.ListForItems(ctx, p.WorkspaceID, ids, 0)
	if err != nil {
		return nil
	}

	var out []string
	seen := map[string]bool{}
	for _, rel := range rels {
		if rel.Type != domain.RelContradicts || rel.ReviewStatus == domain.ReviewRejected {
			continue
		}
		// Both ends must be in the retrieved set for the conflict to be
		// relevant to this answer.
		srcTitle, okA := titles[rel.SourceID]
		tgtTitle, okB := titles[rel.TargetID]
		if !okA || !okB || seen[rel.ID] {
			continue
		}
		seen[rel.ID] = true

		detail := ""
		for _, s := range rel.Evidence.Signals {
			if s.Type == domain.SigClaimContradiction {
				detail = " " + s.Detail
				break
			}
		}
		out = append(out, fmt.Sprintf(
			"%q and %q record conflicting information (relationship strength %.2f, %s).%s",
			srcTitle, tgtTitle, rel.Score, provenanceLabel(rel.Provenance), detail))
	}
	sort.Strings(out)
	if len(out) > 6 {
		out = out[:6]
	}
	return out
}