Code sync, security fixes
This commit is contained in:
Andy Dixon
@@ -0,0 +1,204 @@
|
||||
// Package ranking fuses the several retrieval signals a search produces into a
|
||||
// single ordered result list, and records why each result ranked where it did.
|
||||
//
|
||||
// The output is a QUERY RELEVANCE score. It is deliberately a different number
|
||||
// from semantic similarity (how alike two texts are) and from relationship
|
||||
// strength (how well evidenced a link between two items is). Conflating the
|
||||
// three would make every explanation meaningless, so they are kept apart all
|
||||
// the way to the interface.
|
||||
package ranking
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"sort"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Source names a retrieval path that contributed to a result.
|
||||
type Source string
|
||||
|
||||
const (
|
||||
SourceLexical Source = "lexical"
|
||||
SourceVector Source = "vector"
|
||||
SourceEntity Source = "entity"
|
||||
SourceExpansion Source = "relationship"
|
||||
SourceRecency Source = "recency"
|
||||
SourceConfirmedRel Source = "confirmed_relationship"
|
||||
SourceTitle Source = "title"
|
||||
)
|
||||
|
||||
// Weights control how much each retrieval path contributes to query relevance.
|
||||
var Weights = map[Source]float64{
|
||||
SourceLexical: 1.00,
|
||||
SourceVector: 1.00,
|
||||
SourceEntity: 0.60,
|
||||
SourceTitle: 0.50,
|
||||
SourceExpansion: 0.35,
|
||||
SourceRecency: 0.15,
|
||||
SourceConfirmedRel: 0.25,
|
||||
}
|
||||
|
||||
// RankVersion identifies the fusion model, so a change is visible in results.
|
||||
const RankVersion = "query-rank-v1"
|
||||
|
||||
// rrfK is the reciprocal-rank-fusion constant.
|
||||
//
|
||||
// Fusion is by rank rather than by raw score because MongoDB text scores and
|
||||
// cosine similarities are not on comparable scales: adding them directly would
|
||||
// let whichever happened to have the larger range dominate. Rank position is
|
||||
// comparable by construction.
|
||||
const rrfK = 60.0
|
||||
|
||||
// Contribution is one retrieval path's effect on a result's rank.
|
||||
type Contribution struct {
|
||||
Source Source `json:"source"`
|
||||
// Rank is this result's position in that path's own ordering, 1-based.
|
||||
// Zero means the path contributed without an ordering of its own.
|
||||
Rank int `json:"rank,omitempty"`
|
||||
Weight float64 `json:"weight"`
|
||||
Score float64 `json:"score"`
|
||||
// Detail is the human-readable reason, shown as "why this matched".
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
// Candidate is one item under consideration, with everything known about it.
|
||||
type Candidate struct {
|
||||
ItemID string
|
||||
Contributions []Contribution
|
||||
// Relevance is the fused query relevance, filled in by Rank.
|
||||
Relevance float64
|
||||
// BestSimilarity is the highest cosine similarity of any matching chunk.
|
||||
// It is reported separately and never folded into Relevance.
|
||||
BestSimilarity float64
|
||||
MatchedChunks []string
|
||||
MatchedEntities []string
|
||||
// ExpandedFrom names the item that pulled this one in by relationship.
|
||||
ExpandedFrom string
|
||||
CreatedAt time.Time
|
||||
}
|
||||
|
||||
// Add records a contribution from one retrieval path.
|
||||
func (c *Candidate) Add(src Source, rank int, detail string) {
|
||||
weight := Weights[src]
|
||||
score := weight
|
||||
if rank > 0 {
|
||||
score = weight * (1.0 / (rrfK + float64(rank)))
|
||||
}
|
||||
c.Contributions = append(c.Contributions, Contribution{
|
||||
Source: src, Rank: rank, Weight: weight, Score: score, Detail: detail,
|
||||
})
|
||||
}
|
||||
|
||||
// weakestEffectiveRank is the rank a zero-strength scaled contribution maps to.
|
||||
const weakestEffectiveRank = 50
|
||||
|
||||
// AddScaled records a contribution whose strength is already known in 0..1,
|
||||
// used for paths without a meaningful ordering of their own.
|
||||
//
|
||||
// The strength is converted to an *effective rank* so that scaled and ranked
|
||||
// contributions sit on the same 1/(k+rank) curve, differing only by weight.
|
||||
// Scoring them on a separate curve made them incomparable: a full-strength
|
||||
// expansion could outrank a direct keyword hit, which would mean "related to a
|
||||
// match" beating "is a match".
|
||||
func (c *Candidate) AddScaled(src Source, strength float64, detail string) {
|
||||
weight := Weights[src]
|
||||
if strength < 0 {
|
||||
strength = 0
|
||||
}
|
||||
if strength > 1 {
|
||||
strength = 1
|
||||
}
|
||||
// A path reporting zero strength contributes nothing, but is still recorded
|
||||
// so the explanation shows that it ran and found nothing.
|
||||
score := 0.0
|
||||
if strength > 0 {
|
||||
effectiveRank := 1 + (1-strength)*(weakestEffectiveRank-1)
|
||||
score = weight * (1.0 / (rrfK + effectiveRank))
|
||||
}
|
||||
|
||||
c.Contributions = append(c.Contributions, Contribution{
|
||||
Source: src, Weight: weight, Score: score, Detail: detail,
|
||||
})
|
||||
}
|
||||
|
||||
// Rank fuses contributions and returns candidates ordered by query relevance.
|
||||
func Rank(cands []*Candidate, now time.Time, recencyHalfLife time.Duration) []*Candidate {
|
||||
for _, c := range cands {
|
||||
total := 0.0
|
||||
for _, contrib := range c.Contributions {
|
||||
total += contrib.Score
|
||||
}
|
||||
|
||||
// Recency is a gentle tie-breaker, not a ranking principle: a highly
|
||||
// relevant old document must still outrank a barely-relevant new one.
|
||||
if recencyHalfLife > 0 && !c.CreatedAt.IsZero() {
|
||||
age := now.Sub(c.CreatedAt)
|
||||
if age > 0 {
|
||||
decay := math.Exp(-math.Ln2 * float64(age) / float64(recencyHalfLife))
|
||||
if decay > 0.01 {
|
||||
// On the same curve as every other contribution.
|
||||
effectiveRank := 1 + (1-decay)*(weakestEffectiveRank-1)
|
||||
w := Weights[SourceRecency] * (1.0 / (rrfK + effectiveRank))
|
||||
c.Contributions = append(c.Contributions, Contribution{
|
||||
Source: SourceRecency, Weight: Weights[SourceRecency], Score: w,
|
||||
Detail: fmt.Sprintf("Created %s ago.", humanise(age)),
|
||||
})
|
||||
total += w
|
||||
}
|
||||
}
|
||||
}
|
||||
c.Relevance = total
|
||||
}
|
||||
|
||||
sort.SliceStable(cands, func(i, j int) bool {
|
||||
if cands[i].Relevance != cands[j].Relevance {
|
||||
return cands[i].Relevance > cands[j].Relevance
|
||||
}
|
||||
// A stable, meaningful tie-break keeps result order reproducible.
|
||||
if cands[i].BestSimilarity != cands[j].BestSimilarity {
|
||||
return cands[i].BestSimilarity > cands[j].BestSimilarity
|
||||
}
|
||||
return cands[i].ItemID < cands[j].ItemID
|
||||
})
|
||||
|
||||
// Normalise to 0..1 against the top result so the displayed figure is
|
||||
// interpretable. It remains a relative ranking score, not a probability.
|
||||
if len(cands) > 0 && cands[0].Relevance > 0 {
|
||||
top := cands[0].Relevance
|
||||
for _, c := range cands {
|
||||
c.Relevance = c.Relevance / top
|
||||
}
|
||||
}
|
||||
return cands
|
||||
}
|
||||
|
||||
// Explain renders a candidate's contributions as ordered plain-English lines.
|
||||
func Explain(c *Candidate) []string {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
contribs := append([]Contribution(nil), c.Contributions...)
|
||||
sort.SliceStable(contribs, func(i, j int) bool { return contribs[i].Score > contribs[j].Score })
|
||||
|
||||
out := make([]string, 0, len(contribs))
|
||||
for _, ct := range contribs {
|
||||
out = append(out, ct.Detail)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func humanise(d time.Duration) string {
|
||||
switch {
|
||||
case d < time.Hour:
|
||||
return fmt.Sprintf("%d minutes", int(d.Minutes()))
|
||||
case d < 48*time.Hour:
|
||||
return fmt.Sprintf("%d hours", int(d.Hours()))
|
||||
case d < 60*24*time.Hour:
|
||||
return fmt.Sprintf("%d days", int(d.Hours()/24))
|
||||
case d < 730*24*time.Hour:
|
||||
return fmt.Sprintf("%d months", int(d.Hours()/24/30))
|
||||
default:
|
||||
return fmt.Sprintf("%d years", int(d.Hours()/24/365))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package ranking
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newCandidate(id string) *Candidate {
|
||||
return &Candidate{ItemID: id, CreatedAt: time.Now()}
|
||||
}
|
||||
|
||||
func TestHigherRankedHitsContributeMore(t *testing.T) {
|
||||
first := newCandidate("a")
|
||||
first.Add(SourceLexical, 1, "ranked first")
|
||||
|
||||
tenth := newCandidate("b")
|
||||
tenth.Add(SourceLexical, 10, "ranked tenth")
|
||||
|
||||
ranked := Rank([]*Candidate{tenth, first}, time.Now(), 0)
|
||||
if ranked[0].ItemID != "a" {
|
||||
t.Fatalf("the better-ranked hit did not come first: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
// Fusion must be by rank, not raw score: MongoDB text scores and cosine
|
||||
// similarities are not on comparable scales, so adding them directly would let
|
||||
// whichever has the larger range dominate.
|
||||
func TestAgreementBetweenPathsOutranksASinglePath(t *testing.T) {
|
||||
both := newCandidate("both")
|
||||
both.Add(SourceLexical, 3, "keyword match")
|
||||
both.Add(SourceVector, 3, "meaning match")
|
||||
|
||||
one := newCandidate("one")
|
||||
one.Add(SourceVector, 1, "meaning match only")
|
||||
|
||||
ranked := Rank([]*Candidate{one, both}, time.Now(), 0)
|
||||
if ranked[0].ItemID != "both" {
|
||||
t.Fatalf("a result found by two paths did not outrank one found by a single path: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRelevanceIsNormalisedAgainstTheBestResult(t *testing.T) {
|
||||
a := newCandidate("a")
|
||||
a.Add(SourceLexical, 1, "x")
|
||||
b := newCandidate("b")
|
||||
b.Add(SourceLexical, 50, "y")
|
||||
|
||||
ranked := Rank([]*Candidate{a, b}, time.Now(), 0)
|
||||
if ranked[0].Relevance != 1.0 {
|
||||
t.Fatalf("top result relevance = %v, want 1.00", ranked[0].Relevance)
|
||||
}
|
||||
for _, c := range ranked {
|
||||
if c.Relevance < 0 || c.Relevance > 1 {
|
||||
t.Fatalf("relevance %v is outside 0..1", c.Relevance)
|
||||
}
|
||||
}
|
||||
if ranked[1].Relevance >= ranked[0].Relevance {
|
||||
t.Fatal("normalisation flattened the ordering")
|
||||
}
|
||||
}
|
||||
|
||||
// Recency is a tie-breaker, not a ranking principle: a highly relevant old
|
||||
// document must still beat a barely-relevant new one.
|
||||
func TestRecencyDoesNotOverwhelmRelevance(t *testing.T) {
|
||||
old := newCandidate("old-but-relevant")
|
||||
old.Add(SourceLexical, 1, "strong match")
|
||||
old.Add(SourceVector, 1, "strong match")
|
||||
old.CreatedAt = time.Now().AddDate(-3, 0, 0)
|
||||
|
||||
fresh := newCandidate("new-but-weak")
|
||||
fresh.Add(SourceLexical, 45, "weak match")
|
||||
fresh.CreatedAt = time.Now()
|
||||
|
||||
ranked := Rank([]*Candidate{fresh, old}, time.Now(), 180*24*time.Hour)
|
||||
if ranked[0].ItemID != "old-but-relevant" {
|
||||
t.Fatalf("recency outranked relevance: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRecencyBreaksTiesBetweenEqualMatches(t *testing.T) {
|
||||
older := newCandidate("older")
|
||||
older.Add(SourceLexical, 5, "match")
|
||||
older.CreatedAt = time.Now().AddDate(0, -6, 0)
|
||||
|
||||
newer := newCandidate("newer")
|
||||
newer.Add(SourceLexical, 5, "match")
|
||||
newer.CreatedAt = time.Now()
|
||||
|
||||
ranked := Rank([]*Candidate{older, newer}, time.Now(), 180*24*time.Hour)
|
||||
if ranked[0].ItemID != "newer" {
|
||||
t.Fatalf("recency did not break the tie: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
// Relationship expansion is weaker evidence than a direct match: being related
|
||||
// to a hit is not the same as being one.
|
||||
func TestExpansionRanksBelowADirectMatch(t *testing.T) {
|
||||
direct := newCandidate("direct")
|
||||
direct.Add(SourceLexical, 20, "direct keyword match")
|
||||
|
||||
expanded := newCandidate("expanded")
|
||||
expanded.AddScaled(SourceExpansion, 1.0, "reached through a relationship")
|
||||
|
||||
ranked := Rank([]*Candidate{expanded, direct}, time.Now(), 0)
|
||||
if ranked[0].ItemID != "direct" {
|
||||
t.Fatalf("an expanded result outranked a direct match: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestConfirmedRelationshipsAddWeight(t *testing.T) {
|
||||
plain := newCandidate("plain")
|
||||
plain.AddScaled(SourceExpansion, 0.6, "inferred link")
|
||||
|
||||
confirmed := newCandidate("confirmed")
|
||||
confirmed.AddScaled(SourceExpansion, 0.6, "inferred link")
|
||||
confirmed.AddScaled(SourceConfirmedRel, 1.0, "a person confirmed this link")
|
||||
|
||||
ranked := Rank([]*Candidate{plain, confirmed}, time.Now(), 0)
|
||||
if ranked[0].ItemID != "confirmed" {
|
||||
t.Fatalf("a human-confirmed link did not outrank an unreviewed one: %v", ids(ranked))
|
||||
}
|
||||
}
|
||||
|
||||
func TestOrderingIsStableAndReproducible(t *testing.T) {
|
||||
build := func() []*Candidate {
|
||||
var out []*Candidate
|
||||
for _, id := range []string{"c", "a", "b"} {
|
||||
c := newCandidate(id)
|
||||
c.Add(SourceLexical, 5, "identical match") // deliberately tied
|
||||
c.CreatedAt = time.Time{} // no recency contribution
|
||||
out = append(out, c)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
first := ids(Rank(build(), time.Now(), 0))
|
||||
for i := 0; i < 5; i++ {
|
||||
if got := ids(Rank(build(), time.Now(), 0)); got != first {
|
||||
t.Fatalf("ordering is not reproducible: %q then %q", first, got)
|
||||
}
|
||||
}
|
||||
// Ties break on item ID, so the order is meaningful rather than arbitrary.
|
||||
if first != "a,b,c" {
|
||||
t.Fatalf("tied results are not ordered deterministically: %q", first)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSimilarityIsNeverFoldedIntoRelevance(t *testing.T) {
|
||||
// The two numbers answer different questions and must stay separate.
|
||||
c := newCandidate("x")
|
||||
c.Add(SourceLexical, 1, "keyword match")
|
||||
c.BestSimilarity = 0.97
|
||||
|
||||
ranked := Rank([]*Candidate{c}, time.Now(), 0)
|
||||
if ranked[0].BestSimilarity != 0.97 {
|
||||
t.Fatal("similarity was modified by ranking")
|
||||
}
|
||||
if ranked[0].Relevance == ranked[0].BestSimilarity {
|
||||
t.Fatal("relevance and similarity are the same number; they must be reported separately")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplainListsEveryContributionStrongestFirst(t *testing.T) {
|
||||
c := newCandidate("x")
|
||||
c.AddScaled(SourceEntity, 0.2, "weak entity match")
|
||||
c.Add(SourceVector, 1, "strong meaning match")
|
||||
c.Add(SourceLexical, 2, "keyword match")
|
||||
|
||||
lines := Explain(c)
|
||||
if len(lines) != 3 {
|
||||
t.Fatalf("expected 3 explanations, got %d", len(lines))
|
||||
}
|
||||
if !strings.Contains(lines[0], "strong meaning match") {
|
||||
t.Fatalf("explanations are not ordered by contribution:\n%s", strings.Join(lines, "\n"))
|
||||
}
|
||||
}
|
||||
|
||||
func TestExplainOnNilCandidate(t *testing.T) {
|
||||
if got := Explain(nil); got != nil {
|
||||
t.Fatalf("Explain(nil) = %v, want nil", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestScaledContributionsAreClamped(t *testing.T) {
|
||||
c := newCandidate("x")
|
||||
c.AddScaled(SourceEntity, 99, "out of range")
|
||||
c.AddScaled(SourceEntity, -5, "negative")
|
||||
|
||||
if c.Contributions[0].Score <= 0 {
|
||||
t.Fatal("an over-range strength produced no contribution")
|
||||
}
|
||||
if c.Contributions[1].Score != 0 {
|
||||
t.Fatalf("a negative strength contributed %v, want 0", c.Contributions[1].Score)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEmptyInputIsHandled(t *testing.T) {
|
||||
if got := Rank(nil, time.Now(), time.Hour); len(got) != 0 {
|
||||
t.Fatalf("Rank(nil) returned %d candidates", len(got))
|
||||
}
|
||||
if got := Rank([]*Candidate{}, time.Now(), time.Hour); len(got) != 0 {
|
||||
t.Fatal("Rank on an empty slice returned candidates")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEveryWeightedSourceHasAWeight(t *testing.T) {
|
||||
for _, s := range []Source{
|
||||
SourceLexical, SourceVector, SourceEntity, SourceExpansion,
|
||||
SourceRecency, SourceConfirmedRel, SourceTitle,
|
||||
} {
|
||||
if _, ok := Weights[s]; !ok {
|
||||
t.Errorf("source %q has no configured weight", s)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func ids(cands []*Candidate) string {
|
||||
out := make([]string, 0, len(cands))
|
||||
for _, c := range cands {
|
||||
out = append(out, c.ItemID)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
Reference in New Issue
Block a user