andy/relata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Code sync, security fixes

Browse Source
This commit is contained in:
Andy Dixon
2026-08-11 10:40:10 +01:00
parent 4e00deedf8
commit 53dbc28383
141 changed files with 36194 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/entities/claims.go
+330
View File
@@ -0,0 +1,330 @@
package entities
import (
"regexp"
"strconv"
"strings"
)
// DeterministicClaim is a claim identified by rule rather than by a model.
type DeterministicClaim struct {
Statement string
Quote string
Start int
End int
// Subject is a coarse normalised key naming what the claim is about. Claims
// sharing a subject are the ones worth comparing for agreement or conflict.
Subject string
Predicate string
Numbers []float64
Dates []string
// Polarity is +1 for an affirmative statement, -1 when negated.
Polarity int
}
// maxSentencesScanned bounds work on a very long chunk.
const maxSentencesScanned = 400
// ExtractDeterministicClaims finds sentences that assert something checkable.
//
// The rule is narrow by design: a sentence qualifies when it carries a figure
// or a date, because those are the statements that can actually agree or
// conflict with another source. A sentence of pure prose is left to the model
// stage rather than recorded as a claim on weak grounds.
func ExtractDeterministicClaims(text string) []DeterministicClaim {
var out []DeterministicClaim
for i, sp := range splitSentences(text) {
if i >= maxSentencesScanned {
break
}
sentence := text[sp.start:sp.end]
trimmed := strings.TrimSpace(sentence)
if len(trimmed) < 12 || len(trimmed) > 600 {
continue
}
subject, predicate, numbers, dates, polarity := AnalyseStatement(sentence)
if len(numbers) == 0 && len(dates) == 0 {
continue
}
if subject == "" {
continue
}
out = append(out, DeterministicClaim{
Statement: collapse(trimmed),
Quote: sentence,
Start: sp.start,
End: sp.end,
Subject: subject,
Predicate: predicate,
Numbers: numbers,
Dates: dates,
Polarity: polarity,
})
}
return out
}
// AnalyseStatement derives the comparison keys for a statement.
//
// The subject is the content word immediately governing the figure or date —
// "cost" in "the total project cost is £42,000", "date" in "the delivery date
// is 14 March". Two claims sharing a subject are candidates for comparison; the
// comparison itself only ever happens between items already linked by other
// evidence, so a broad subject cannot manufacture spurious pairs.
func AnalyseStatement(sentence string) (subject, predicate string, numbers []float64, dates []string, polarity int) {
polarity = 1
lower := strings.ToLower(sentence)
for _, neg := range []string{" not ", " no ", "n't ", " never ", " cancelled", " withdrawn"} {
if strings.Contains(lower, neg) {
polarity = -1
break
}
}
// Dates first, so a year is not also counted as a bare number.
dateSpans := map[int]bool{}
for _, m := range ExtractDeterministic(sentence) {
if m.Type == "date" && m.Normalised != "" {
dates = append(dates, m.Normalised)
for i := m.Start; i < m.End; i++ {
dateSpans[i] = true
}
}
}
var anchor = -1
for _, loc := range numberPattern.FindAllStringIndex(sentence, 40) {
// Skip any match that OVERLAPS a date, not merely one that starts on
// it: the number pattern can begin at the whitespace before a date and
// take "202" out of "2025-02-10", which would then be compared against
// another document's figures as if it were a quantity.
if overlapsSpan(dateSpans, loc[0], loc[1]) {
continue
}
raw := sentence[loc[0]:loc[1]]
v, ok := parseNumber(raw)
if !ok {
continue
}
numbers = append(numbers, v)
if anchor < 0 {
anchor = loc[0]
}
}
if anchor < 0 && len(dates) > 0 {
for i := 0; i < len(sentence); i++ {
if dateSpans[i] {
anchor = i
break
}
}
}
if anchor < 0 {
return "", "", numbers, dates, polarity
}
// Decide which side of the figure the subject sits on.
//
// "the total project cost is £42,000" — the subject precedes the figure.
// "will ship 240 units" — the figure is a quantity and the
// unit noun follows it.
// The distinguishing signal is what sits immediately before the number: a
// verb means the figure is a direct object, so the noun after it is what
// the claim is about.
before := sentence[:anchor]
words := wordPattern.FindAllString(before, -1)
precededByVerb := false
for i := len(words) - 1; i >= 0; i-- {
w := strings.ToLower(words[i])
if stopWords[w] {
continue // skip "will", "is", "are"…
}
precededByVerb = badSubjects[w]
break
}
var content []string
if !precededByVerb {
for i := len(words) - 1; i >= 0 && len(content) < 3; i-- {
w := strings.ToLower(words[i])
if stopWords[w] || badSubjects[w] || len(w) < 3 {
continue
}
content = append([]string{w}, content...)
}
}
if len(content) == 0 {
// Either the figure is a quantity, or nothing meaningful precedes it.
for _, w := range wordPattern.FindAllString(sentence[anchor:], 8) {
lw := strings.ToLower(w)
if !stopWords[lw] && !badSubjects[lw] && len(lw) >= 3 {
content = append(content, lw)
break
}
}
}
if len(content) == 0 {
return "", "", numbers, dates, polarity
}
// The last content word is the subject key; keeping it single-word makes
// "the cost is" and "the total project cost is" agree on "cost".
subject = Normalise(content[len(content)-1])
predicate = Normalise(strings.Join(content, " "))
return subject, predicate, numbers, dates, polarity
}
var (
// Currency-aware number matching: "£42,000", "42000", "3.5", "42%".
numberPattern = regexp.MustCompile(`[£$€]?\s?\d{1,3}(?:,\d{3})*(?:\.\d+)?%?|[£$€]?\s?\d+(?:\.\d+)?%?`)
wordPattern = regexp.MustCompile(`[\p{L}][\p{L}'-]*`)
)
// parseNumber converts a matched numeric string to a float.
func parseNumber(raw string) (float64, bool) {
s := strings.TrimSpace(raw)
s = strings.NewReplacer("£", "", "$", "", "€", "", ",", "", "%", "", " ", "").Replace(s)
if s == "" {
return 0, false
}
v, err := strconv.ParseFloat(s, 64)
if err != nil {
return 0, false
}
return v, true
}
// stopWords are function words that never make a useful subject key.
var stopWords = map[string]bool{
"the": true, "a": true, "an": true, "and": true, "or": true, "but": true,
"is": true, "are": true, "was": true, "were": true, "be": true, "been": true,
"being": true, "will": true, "would": true, "shall": true, "should": true,
"can": true, "could": true, "may": true, "might": true, "must": true,
"has": true, "have": true, "had": true, "do": true, "does": true, "did": true,
"of": true, "to": true, "in": true, "on": true, "at": true, "by": true,
"for": true, "with": true, "from": true, "as": true, "that": true, "this": true,
"these": true, "those": true, "it": true, "its": true, "we": true, "our": true,
"they": true, "their": true, "he": true, "she": true, "his": true, "her": true,
"there": true, "here": true, "now": true, "then": true, "than": true,
"about": true, "into": true, "within": true, "per": true, "over": true, "under": true, "up": true, "down": true,
"out": true, "off": true, "all": true, "any": true, "some": true, "no": true,
"not": true, "only": true, "just": true, "also": true, "very": true, "more": true,
"most": true, "such": true, "which": true, "who": true, "whom": true, "whose": true,
"what": true, "when": true, "where": true, "how": true, "why": true,
"total": true, "approximately": true, "around": true, "about-": true,
"new": true, "old": true, "current": true, "previous": true,
}
// badSubjects are words that make a useless comparison key.
//
// A subject is what decides which claims get compared for agreement or
// conflict, so a verb or a company suffix here produces spurious pairs: two
// documents both containing "will ship 240 units" and "will ship 500 units"
// would otherwise be compared under the subject "ship" rather than "units".
var badSubjects = map[string]bool{
// Legal forms — part of a company's name, never what a figure is about.
"ltd": true, "limited": true, "plc": true, "llp": true, "llc": true,
"inc": true, "incorporated": true, "corp": true, "corporation": true,
"gmbh": true, "pty": true, "holdings": true,
// Common verbs that sit immediately before a quantity.
"ship": true, "ships": true, "shipped": true, "send": true, "sent": true,
"deliver": true, "delivers": true, "delivered": true, "cover": true,
"covers": true, "covered": true, "issue": true, "issued": true,
"report": true, "reported": true, "state": true, "stated": true,
"say": true, "says": true, "said": true, "give": true, "given": true,
"take": true, "taken": true, "make": true, "made": true, "pay": true,
"paid": true, "run": true, "runs": true, "use": true, "used": true,
"include": true, "includes": true, "included": true,
"attracts": true, "agreed": true, "prepared": true, "conducted": true,
// Honorifics.
"mr": true, "mrs": true, "dr": true, "prof": true, "professor": true,
}
// overlapsSpan reports whether [start,end) intersects any offset in spans.
func overlapsSpan(spans map[int]bool, start, end int) bool {
for i := start; i < end; i++ {
if spans[i] {
return true
}
}
return false
}
type sentenceSpan struct{ start, end int }
// splitSentences divides text at sentence terminators, leaving offsets intact.
//
// Common abbreviations are not treated as terminators, so "Ltd. reported" stays
// one sentence rather than becoming two fragments whose claim is lost.
func splitSentences(text string) []sentenceSpan {
var out []sentenceSpan
start := 0
for i := 0; i < len(text); i++ {
c := text[i]
if c != '.' && c != '!' && c != '?' && c != '\n' {
continue
}
if c == '.' && isAbbreviation(text, i) {
continue
}
// A terminator must be followed by whitespace or the end of the text.
if i+1 < len(text) && !isSpaceByte(text[i+1]) {
continue
}
end := i + 1
if strings.TrimSpace(text[start:end]) != "" {
out = append(out, sentenceSpan{start, end})
}
// Skip the whitespace that follows.
j := end
for j < len(text) && isSpaceByte(text[j]) {
j++
}
start = j
i = j - 1
}
if start < len(text) && strings.TrimSpace(text[start:]) != "" {
out = append(out, sentenceSpan{start, len(text)})
}
return out
}
// abbreviations that end in a full stop without ending a sentence.
var abbreviations = map[string]bool{
"mr": true, "mrs": true, "ms": true, "dr": true, "prof": true, "st": true,
"ltd": true, "inc": true, "co": true, "corp": true, "plc": true,
"e.g": true, "i.e": true, "etc": true, "vs": true, "no": true,
"jan": true, "feb": true, "mar": true, "apr": true, "jun": true,
"jul": true, "aug": true, "sep": true, "sept": true, "oct": true,
"nov": true, "dec": true, "approx": true, "dept": true, "est": true,
}
func isAbbreviation(text string, dot int) bool {
// Walk back over the word preceding the dot.
start := dot
for start > 0 && !isSpaceByte(text[start-1]) {
start--
}
word := strings.ToLower(strings.Trim(text[start:dot], ".,;:!?()"))
if word == "" {
return false
}
if abbreviations[word] {
return true
}
// A single letter followed by a dot is an initial ("J. Smith").
return len([]rune(word)) == 1
}
func isSpaceByte(b byte) bool {
return b == ' ' || b == '\t' || b == '\n' || b == '\r' || b == '\v' || b == '\f'
}
func collapse(s string) string {
return strings.Join(strings.Fields(s), " ")
}
internal/entities/deterministic.go
+221
View File
@@ -0,0 +1,221 @@
package entities
import (
"regexp"
"strings"
"time"
"github.com/andydixon/relata/internal/domain"
)
// Mention is one occurrence of an entity in a piece of text.
type Mention struct {
Name string
Type domain.EntityType
// Start and End are byte offsets into the text the mention was found in.
Start, End int
Quote string
Method domain.ExtractionMethod
// Normalised is the ISO-8601 form for date mentions.
Normalised string
}
// Deterministic patterns.
//
// These are kept narrow on purpose. A pattern that fires on ordinary prose
// would flood the entity index with noise and make shared-entity evidence
// meaningless, so only high-precision shapes are matched here. Everything
// broader is left to the model stage, whose output is marked as such.
var (
// A capitalised run terminated by a legal suffix: "Example Ltd",
// "Northern Rail Holdings PLC".
orgPattern = regexp.MustCompile(
`\b((?:[A-Z][\p{L}&.'-]*\s+){0,5}[A-Z][\p{L}&.'-]*)\s+(Ltd\.?|Limited|PLC|plc|LLP|LLC|Inc\.?|Incorporated|Corp\.?|Corporation|GmbH|AG|B\.?V\.?|N\.?V\.?|Pty|Co\.)(?:\s|$|[.,;:!?)])`)
// An honorific followed by capitalised names: "Dr Alice Chen".
personPattern = regexp.MustCompile(
`\b(Mr|Mrs|Ms|Miss|Dr|Prof|Professor|Sir|Dame|Lord|Lady)\.?\s+((?:[A-Z][\p{L}'-]+\s+){0,2}[A-Z][\p{L}'-]+)\b`)
// Email addresses identify a correspondent unambiguously.
emailPattern = regexp.MustCompile(`\b[\w.!#$%&'*+/=?^_` + "`" + `{|}~-]+@[\p{L}\d](?:[\p{L}\d-]{0,61}[\p{L}\d])?(?:\.[\p{L}\d](?:[\p{L}\d-]{0,61}[\p{L}\d])?)+\b`)
// Dates in the formats Relata can normalise with confidence.
isoDatePattern = regexp.MustCompile(`\b(\d{4})-(\d{2})-(\d{2})\b`)
dmyPattern = regexp.MustCompile(`\b(\d{1,2})(?:st|nd|rd|th)?\s+(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{4})\b`)
mdyPattern = regexp.MustCompile(`\b(January|February|March|April|May|June|July|August|September|October|November|December)\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(\d{4})\b`)
slashDate = regexp.MustCompile(`\b(\d{1,2})/(\d{1,2})/(\d{4})\b`)
)
// maxDeterministicMentions bounds output from a pathological document.
const maxDeterministicMentions = 500
// ExtractDeterministic finds entities that can be identified by rule alone.
//
// Every mention carries exact source offsets, so the caller can attach a
// verifiable source reference. Mentions are returned in document order.
func ExtractDeterministic(text string) []Mention {
var out []Mention
add := func(m Mention) {
if len(out) >= maxDeterministicMentions {
return
}
m.Method = domain.MethodDeterministic
m.Quote = safeSlice(text, m.Start, m.End)
out = append(out, m)
}
for _, loc := range orgPattern.FindAllStringSubmatchIndex(text, maxDeterministicMentions) {
// Group 1 is the name body, group 2 the suffix; the entity is both.
start, end := loc[2], loc[5]
if start < 0 || end < 0 || end <= start {
continue
}
name := strings.TrimSpace(text[start:end])
if !plausibleName(name) {
continue
}
add(Mention{Name: name, Type: domain.EntityOrganisation, Start: start, End: end})
}
for _, loc := range personPattern.FindAllStringSubmatchIndex(text, maxDeterministicMentions) {
start, end := loc[4], loc[5] // the name without the honorific
if start < 0 || end < 0 {
continue
}
name := strings.TrimSpace(text[start:end])
if !plausibleName(name) {
continue
}
add(Mention{Name: name, Type: domain.EntityPerson, Start: start, End: end})
}
for _, loc := range emailPattern.FindAllStringIndex(text, maxDeterministicMentions) {
addr := text[loc[0]:loc[1]]
if len(addr) > 254 {
continue
}
add(Mention{Name: strings.ToLower(addr), Type: domain.EntityPerson, Start: loc[0], End: loc[1]})
}
out = append(out, extractDates(text)...)
sortMentions(out)
return dedupeOverlapping(out)
}
// extractDates finds and normalises dates to ISO-8601.
//
// Ambiguous numeric formats are resolved as day/month/year, matching the
// application's British English conventions. The normalised value is what
// temporal comparisons use, so getting this wrong would silently distort
// relationship evidence — hence only unambiguous or documented formats.
func extractDates(text string) []Mention {
var out []Mention
emit := func(start, end int, t time.Time) {
out = append(out, Mention{
Name: t.Format("2006-01-02"),
Type: domain.EntityDate,
Start: start,
End: end,
Quote: safeSlice(text, start, end),
Method: domain.MethodDeterministic,
Normalised: t.Format("2006-01-02"),
})
}
for _, loc := range isoDatePattern.FindAllStringSubmatchIndex(text, 200) {
if t, err := time.Parse("2006-01-02", text[loc[0]:loc[1]]); err == nil {
emit(loc[0], loc[1], t)
}
}
for _, loc := range dmyPattern.FindAllStringSubmatchIndex(text, 200) {
day := text[loc[2]:loc[3]]
month := text[loc[4]:loc[5]]
year := text[loc[6]:loc[7]]
if t, err := time.Parse("2 January 2006", day+" "+month+" "+year); err == nil {
emit(loc[0], loc[1], t)
}
}
for _, loc := range mdyPattern.FindAllStringSubmatchIndex(text, 200) {
month := text[loc[2]:loc[3]]
day := text[loc[4]:loc[5]]
year := text[loc[6]:loc[7]]
if t, err := time.Parse("2 January 2006", day+" "+month+" "+year); err == nil {
emit(loc[0], loc[1], t)
}
}
for _, loc := range slashDate.FindAllStringSubmatchIndex(text, 200) {
d, m, y := text[loc[2]:loc[3]], text[loc[4]:loc[5]], text[loc[6]:loc[7]]
// Day/month/year. A value above 12 in the first position confirms the
// reading; otherwise the convention is applied as documented.
if t, err := time.Parse("2/1/2006", d+"/"+m+"/"+y); err == nil {
emit(loc[0], loc[1], t)
}
}
return out
}
// plausibleName filters out matches that are almost certainly not names:
// sentence-initial words, all-caps shouting, and absurd lengths.
func plausibleName(name string) bool {
if len(name) < 2 || len(name) > 120 {
return false
}
words := strings.Fields(name)
if len(words) == 0 || len(words) > 8 {
return false
}
// A single very common word that happens to be capitalised is not a name.
if len(words) == 1 && commonWords[strings.ToLower(strings.Trim(words[0], ".,"))] {
return false
}
return true
}
// commonWords are capitalised sentence-openers that would otherwise be matched.
var commonWords = map[string]bool{
"the": true, "this": true, "that": true, "these": true, "those": true,
"a": true, "an": true, "it": true, "we": true, "they": true, "he": true,
"she": true, "i": true, "our": true, "their": true, "his": true, "her": true,
"in": true, "on": true, "at": true, "to": true, "for": true, "and": true,
"but": true, "if": true, "as": true, "by": true, "from": true, "with": true,
"all": true, "any": true, "each": true, "no": true, "not": true, "one": true,
"there": true, "here": true, "when": true, "where": true, "while": true,
"after": true, "before": true, "during": true, "since": true, "until": true,
}
func safeSlice(s string, start, end int) string {
if start < 0 || end > len(s) || start >= end {
return ""
}
return s[start:end]
}
func sortMentions(ms []Mention) {
for i := 1; i < len(ms); i++ {
for j := i; j > 0 && ms[j].Start < ms[j-1].Start; j-- {
ms[j], ms[j-1] = ms[j-1], ms[j]
}
}
}
// dedupeOverlapping removes mentions contained within an earlier, longer
// mention at the same position, so "Example Ltd" is not also recorded as the
// person "Example".
func dedupeOverlapping(ms []Mention) []Mention {
out := ms[:0]
var lastEnd int
var lastType domain.EntityType
for _, m := range ms {
if m.Start < lastEnd && m.Type != lastType {
continue
}
out = append(out, m)
if m.End > lastEnd {
lastEnd = m.End
lastType = m.Type
}
}
return out
}
internal/entities/entities_test.go
+431
View File
@@ -0,0 +1,431 @@
package entities
import (
"strings"
"testing"
"github.com/andydixon/relata/internal/domain"
)
func TestNormaliseFoldsCasePunctuationAndAccents(t *testing.T) {
cases := []struct{ in, want string }{
{"Example Ltd", "example ltd"},
{" EXAMPLE LTD. ", "example ltd"},
{"Example, Ltd.", "example ltd"},
{"Zoë Müller", "zoe muller"},
{"Marks & Spencer", "marks and spencer"},
{"O'Brien", "obrien"},
{"ACME—Corp", "acme corp"},
}
for _, c := range cases {
if got := Normalise(c.in); got != c.want {
t.Errorf("Normalise(%q) = %q, want %q", c.in, got, c.want)
}
}
}
func TestNormaliseIsIdempotent(t *testing.T) {
for _, in := range []string{"Example Ltd.", "Zoë Müller", "A & B"} {
once := Normalise(in)
if twice := Normalise(once); once != twice {
t.Errorf("Normalise is not idempotent for %q: %q then %q", in, once, twice)
}
}
}
// The key must distinguish types: a place and a person sharing a name are not
// the same entity.
func TestKeyIncludesType(t *testing.T) {
person := Key(domain.EntityPerson, "Washington")
place := Key(domain.EntityLocation, "Washington")
if person == place {
t.Fatal("entities of different types collapsed to the same key")
}
}
// This is the central safety property: normalisation must not silently merge
// distinct organisations.
func TestNormalisationDoesNotMergeDistinctCompanies(t *testing.T) {
distinct := [][2]string{
{"Example Ltd", "Example Holdings Ltd"},
{"Example Ltd", "Example Inc"},
{"Northern Rail", "Northern Rail Freight"},
{"Acme Systems", "Acme Systems Europe"},
}
for _, pair := range distinct {
a := Key(domain.EntityOrganisation, pair[0])
b := Key(domain.EntityOrganisation, pair[1])
if a == b {
t.Errorf("%q and %q were merged into one entity key %q", pair[0], pair[1], a)
}
}
}
func TestLooseKeyIsOnlyForAliasProposals(t *testing.T) {
// The loose key deliberately collapses legal forms, which is why it must
// never be used for storage identity.
if LooseKey(domain.EntityOrganisation, "Example Ltd") != LooseKey(domain.EntityOrganisation, "Example Limited") {
t.Fatal("loose key should collapse equivalent legal forms")
}
if Key(domain.EntityOrganisation, "Example Ltd") == Key(domain.EntityOrganisation, "Example Limited") {
t.Fatal("storage key must NOT collapse them without human confirmation")
}
}
func TestAliasSimilarityRanksSensibly(t *testing.T) {
identical := AliasSimilarity("Example Ltd", "example ltd")
if identical != 1 {
t.Fatalf("case-only difference scored %v, want 1", identical)
}
typo := AliasSimilarity("Example Ltd", "Exampel Ltd")
unrelated := AliasSimilarity("Example Ltd", "Northern Rail Freight")
if typo <= unrelated {
t.Fatalf("typo (%v) did not score above an unrelated name (%v)", typo, unrelated)
}
if unrelated > 0.5 {
t.Fatalf("unrelated names scored implausibly high: %v", unrelated)
}
}
func TestAliasSimilarityRecognisesAcronyms(t *testing.T) {
got := AliasSimilarity("BBC", "British Broadcasting Corporation")
if got < 0.8 {
t.Fatalf("acronym match scored only %v", got)
}
}
func TestCanonicaliseIsDeterministic(t *testing.T) {
observed := []string{"example ltd", "Example Ltd", "Example Ltd", "EXAMPLE LTD"}
first := Canonicalise(observed)
if first != "Example Ltd" {
t.Fatalf("canonical form = %q, want the most frequent properly-cased spelling", first)
}
// Re-running with a different input order must give the same answer.
shuffled := []string{"EXAMPLE LTD", "Example Ltd", "example ltd", "Example Ltd"}
if second := Canonicalise(shuffled); second != first {
t.Fatalf("canonicalisation depends on input order: %q vs %q", first, second)
}
}
// --- Deterministic extraction ---------------------------------------------
func findMention(ms []Mention, name string) *Mention {
for i := range ms {
if ms[i].Name == name {
return &ms[i]
}
}
return nil
}
func TestExtractsOrganisationsWithLegalSuffixes(t *testing.T) {
text := "The contract was signed by Example Ltd and Northern Rail Holdings PLC on Tuesday."
ms := ExtractDeterministic(text)
for _, want := range []string{"Example Ltd", "Northern Rail Holdings PLC"} {
m := findMention(ms, want)
if m == nil {
t.Fatalf("did not extract %q from %q", want, text)
}
if m.Type != domain.EntityOrganisation {
t.Errorf("%q typed as %s, want organisation", want, m.Type)
}
if text[m.Start:m.End] != want {
t.Errorf("%q offsets point at %q", want, text[m.Start:m.End])
}
}
}
func TestExtractsPeopleWithHonorifics(t *testing.T) {
text := "Dr Alice Chen approved it, and Prof Raymond Okafor countersigned."
ms := ExtractDeterministic(text)
for _, want := range []string{"Alice Chen", "Raymond Okafor"} {
m := findMention(ms, want)
if m == nil {
t.Fatalf("did not extract %q", want)
}
if m.Type != domain.EntityPerson {
t.Errorf("%q typed as %s, want person", want, m.Type)
}
if text[m.Start:m.End] != want {
t.Errorf("%q offsets point at %q", want, text[m.Start:m.End])
}
}
}
func TestExtractsEmailAddresses(t *testing.T) {
text := "Contact Ops.Team@example.co.uk for details."
ms := ExtractDeterministic(text)
if m := findMention(ms, "ops.team@example.co.uk"); m == nil {
t.Fatalf("email not extracted from %q; got %+v", text, ms)
}
}
func TestNormalisesDatesToISO(t *testing.T) {
cases := map[string]string{
"Due on 2025-03-14 sharp.": "2025-03-14",
"Due on 14 March 2025 sharp.": "2025-03-14",
"Due on 14th March 2025 sharp.": "2025-03-14",
"Due on March 14, 2025 sharp.": "2025-03-14",
"Due on 14/03/2025 sharp.": "2025-03-14",
}
for text, want := range cases {
ms := ExtractDeterministic(text)
m := findMention(ms, want)
if m == nil {
t.Errorf("did not normalise a date in %q to %s (got %+v)", text, want, ms)
continue
}
if m.Type != domain.EntityDate {
t.Errorf("%q typed as %s, want date", text, m.Type)
}
if m.Normalised != want {
t.Errorf("normalised value %q, want %q", m.Normalised, want)
}
}
}
func TestDoesNotExtractCommonWordsAsEntities(t *testing.T) {
text := "The Ltd. This Inc. And Corp."
ms := ExtractDeterministic(text)
for _, m := range ms {
if m.Name == "The Ltd." || m.Name == "This Inc." || m.Name == "And Corp." {
t.Errorf("sentence-opening word extracted as an entity: %q", m.Name)
}
}
}
func TestMentionOffsetsAlwaysAddressTheirQuote(t *testing.T) {
text := "Dr Alice Chen of Example Ltd wrote to ops@example.com on 2025-03-14 about Northern Rail Holdings PLC."
for _, m := range ExtractDeterministic(text) {
if m.Start < 0 || m.End > len(text) || m.Start >= m.End {
t.Fatalf("mention %q has invalid offsets [%d,%d)", m.Name, m.Start, m.End)
}
if m.Quote != text[m.Start:m.End] {
t.Fatalf("mention %q quote %q does not match its offsets %q", m.Name, m.Quote, text[m.Start:m.End])
}
}
}
func TestExtractionIsBounded(t *testing.T) {
var text string
for i := 0; i < 5000; i++ {
text += "Example Ltd and Dr Alice Chen on 2025-03-14. "
}
ms := ExtractDeterministic(text)
if len(ms) > maxDeterministicMentions+600 {
t.Fatalf("extraction is not bounded: %d mentions", len(ms))
}
}
func TestExtractionOnEmptyAndOddInput(t *testing.T) {
for _, in := range []string{"", " ", "\x00\x01", "aaaa", "Ltd", "@@@"} {
if ms := ExtractDeterministic(in); len(ms) > 5 {
t.Errorf("input %q produced %d mentions", in, len(ms))
}
}
}
func FuzzExtractDeterministic(f *testing.F) {
f.Add("Example Ltd on 2025-03-14")
f.Add("Dr A B")
f.Add("a@b.co")
f.Add("14/03/2025")
f.Fuzz(func(t *testing.T, text string) {
for _, m := range ExtractDeterministic(text) {
if m.Start < 0 || m.End > len(text) || m.Start > m.End {
t.Fatalf("mention offsets [%d,%d) out of range for %d bytes", m.Start, m.End, len(text))
}
if m.Quote != text[m.Start:m.End] {
t.Fatal("mention quote does not match its offsets")
}
}
})
}
func FuzzNormalise(f *testing.F) {
f.Add("Example Ltd")
f.Add("Zoë")
f.Add("\x00\xff")
f.Fuzz(func(t *testing.T, s string) {
once := Normalise(s)
if twice := Normalise(once); once != twice {
t.Fatalf("not idempotent: %q -> %q -> %q", s, once, twice)
}
})
}
// --- Deterministic claim extraction ---------------------------------------
// A date must not also be read as a bare number. The number pattern can begin
// at the whitespace before a date and take "202" out of "2025-02-10"; that
// leaked figure was then compared against other documents' quantities and
// produced spurious contradictions.
func TestDatesAreNotAlsoExtractedAsNumbers(t *testing.T) {
cases := []string{
"Prepared by Dr Alice Chen for Example Ltd on 2025-02-10.",
"The report was filed on 14 March 2025.",
"Signed 03/04/2025 by the assessor.",
}
for _, text := range cases {
_, _, nums, dates, _ := AnalyseStatement(text)
if len(dates) == 0 {
t.Errorf("%q: no date detected", text)
}
for _, n := range nums {
// Any fragment of a year would land in this range.
if n >= 19 && n <= 2100 {
t.Errorf("%q: a date fragment leaked into numbers as %v (numbers=%v)", text, n, nums)
}
}
}
}
func TestGenuineFiguresAreStillExtracted(t *testing.T) {
subject, _, nums, _, _ := AnalyseStatement("The total project cost is £42,000 excluding VAT.")
if len(nums) != 1 || nums[0] != 42000 {
t.Fatalf("figure not extracted: %v", nums)
}
if subject != "cost" {
t.Fatalf("subject = %q, want %q", subject, "cost")
}
}
// A subject decides which claims get compared, so a verb or a company suffix
// there manufactures spurious pairs.
func TestSubjectsAreNotVerbsOrLegalSuffixes(t *testing.T) {
// A quantity's subject is the unit noun that follows it; a value's subject
// is the noun that precedes it.
cases := map[string]string{
"Example Ltd will ship 240 units to the Northfield depot.": "units",
"The returns window is 60 days from delivery.": "window",
"Refunds are issued within 14 days.": "days",
"The total project cost is £42,000 excluding VAT.": "cost",
}
for text, want := range cases {
got, _, _, _, _ := AnalyseStatement(text)
if got != want {
t.Errorf("AnalyseStatement(%q) subject = %q, want %q", text, got, want)
}
if badSubjects[got] {
t.Errorf("%q produced a blocked subject %q", text, got)
}
}
}
func TestClaimExtractionRequiresAFigureOrDate(t *testing.T) {
// Pure prose is left to the model stage rather than recorded on weak grounds.
prose := "The team discussed the matter at length and agreed to reflect further."
if got := ExtractDeterministicClaims(prose); len(got) != 0 {
t.Fatalf("prose without a figure or date produced %d claims: %+v", len(got), got)
}
withFigure := "The total project cost is £42,000 excluding VAT."
got := ExtractDeterministicClaims(withFigure)
if len(got) != 1 {
t.Fatalf("expected 1 claim, got %d", len(got))
}
if got[0].Quote != withFigure {
t.Fatalf("claim quote %q does not match the source sentence", got[0].Quote)
}
if got[0].Subject != "cost" {
t.Fatalf("subject = %q, want cost", got[0].Subject)
}
}
func TestClaimOffsetsAddressTheirQuote(t *testing.T) {
text := "Intro sentence here. The total project cost is £42,000. The date is 14 March 2025."
for _, c := range ExtractDeterministicClaims(text) {
if text[c.Start:c.End] != c.Quote {
t.Fatalf("claim offsets [%d,%d) give %q, but the quote is %q",
c.Start, c.End, text[c.Start:c.End], c.Quote)
}
}
}
func TestSentenceSplittingKnowsAbbreviations(t *testing.T) {
// "Ltd." must not end the sentence, or the claim would be lost.
text := "Example Ltd. reported a total cost of £42,000 for the work."
claims := ExtractDeterministicClaims(text)
if len(claims) != 1 {
t.Fatalf("expected 1 claim, got %d: %+v", len(claims), claims)
}
if !strings.Contains(claims[0].Quote, "42,000") {
t.Fatalf("the figure was split away from its sentence: %q", claims[0].Quote)
}
}
func TestNegationIsDetected(t *testing.T) {
_, _, _, _, pol := AnalyseStatement("The order was not delivered on 14 March 2025.")
if pol != -1 {
t.Fatalf("polarity = %d, want -1 for a negated statement", pol)
}
_, _, _, _, pol = AnalyseStatement("The order was delivered on 14 March 2025.")
if pol != 1 {
t.Fatalf("polarity = %d, want 1", pol)
}
}
func FuzzExtractDeterministicClaims(f *testing.F) {
f.Add("The cost is £42,000.")
f.Add("Due 2025-03-14.")
f.Add("Ltd. Inc. Dr. 1.2.3")
f.Fuzz(func(t *testing.T, text string) {
for _, c := range ExtractDeterministicClaims(text) {
if c.Start < 0 || c.End > len(text) || c.Start > c.End {
t.Fatalf("claim offsets [%d,%d) out of range for %d bytes", c.Start, c.End, len(text))
}
if text[c.Start:c.End] != c.Quote {
t.Fatal("claim quote does not match its offsets")
}
}
})
}
// A legal suffix adds a word that breaks the acronym letter count, so the
// comparison must also consider the suffix-stripped forms. Without this, the
// BBC example the documentation gives would never be proposed.
func TestAcronymMatchingIgnoresLegalSuffixes(t *testing.T) {
cases := [][2]string{
{"BBC", "British Broadcasting Corporation Ltd"},
{"BBC", "British Broadcasting Corporation"},
{"NRF", "Northern Rail Freight PLC"},
}
for _, c := range cases {
if got := AliasSimilarity(c[0], c[1]); got < aliasProposalThreshold {
t.Errorf("AliasSimilarity(%q, %q) = %.2f, below the %.2f proposal threshold",
c[0], c[1], got, aliasProposalThreshold)
}
}
}
// Names differing only by legal form are worth a human's attention, but the
// storage keys must still differ so nothing is merged automatically.
func TestLegalFormVariantsAreProposedButNotMerged(t *testing.T) {
if got := AliasSimilarity("Example Ltd", "Example Limited"); got < aliasProposalThreshold {
t.Errorf("Example Ltd / Example Limited scored %.2f, too low to be proposed", got)
}
if Key(domain.EntityOrganisation, "Example Ltd") == Key(domain.EntityOrganisation, "Example Limited") {
t.Fatal("the two names share a storage key, so they were merged without a human decision")
}
}
// The looser matching must not start proposing genuinely different companies.
func TestLooseMatchingStillSeparatesDistinctCompanies(t *testing.T) {
distinct := [][2]string{
{"Example Ltd", "Example Holdings Ltd"},
{"Northern Rail", "Southern Rail"},
{"Acme Systems", "Acme Logistics"},
{"Example Ltd", "Northern Rail Freight"},
}
for _, c := range distinct {
if got := AliasSimilarity(c[0], c[1]); got >= aliasProposalThreshold {
t.Errorf("AliasSimilarity(%q, %q) = %.2f, high enough to propose merging different companies",
c[0], c[1], got)
}
}
}
internal/entities/normalise.go
+311
View File
@@ -0,0 +1,311 @@
// Package entities extracts and resolves identifiable subjects from source
// material.
//
// Resolution is deliberately conservative. Two entities are treated as the same
// only when their normalised keys match exactly. Names that merely look alike
// produce a *candidate alias* awaiting human confirmation — they are never
// merged automatically, because "Example Ltd" and "Example Holdings Ltd" are
// different companies far more often than they are typos of each other.
package entities
import (
"sort"
"strings"
"unicode"
"github.com/andydixon/relata/internal/domain"
)
// Normalise produces the exact-match key for an entity name.
//
// The transformation is intentionally shallow: case folding, whitespace
// collapsing, punctuation and diacritic removal. It does not strip company
// suffixes or expand abbreviations, because doing so would silently merge
// distinct entities — exactly what the design forbids.
func Normalise(name string) string {
var b strings.Builder
b.Grow(len(name))
prevSpace := true // leading space suppressed
for _, r := range strings.ToLower(strings.TrimSpace(name)) {
switch {
case unicode.IsLetter(r) || unicode.IsDigit(r):
b.WriteRune(foldDiacritic(r))
prevSpace = false
case r == '\'' || r == '’' || r == 'ʼ':
// Apostrophes are dropped rather than turned into a break, so
// "O'Brien" and "OBrien" share a key while "O Brien" (two words)
// is a different thing.
case r == '&':
// "&" carries meaning in company names ("Marks & Spencer").
if !prevSpace {
b.WriteByte(' ')
}
b.WriteString("and")
b.WriteByte(' ')
prevSpace = true
case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
if !prevSpace {
b.WriteByte(' ')
prevSpace = true
}
}
}
return strings.TrimSpace(b.String())
}
// Key builds the storage key that determines entity identity. Type is part of
// the key: a place called "Washington" is not the person "Washington".
func Key(t domain.EntityType, name string) string {
return string(t) + ":" + Normalise(name)
}
// diacriticFolds maps accented Latin letters onto their base form so that
// "Zoë" and "Zoe" share a key. It covers Latin-1 and Latin Extended-A rather
// than every script, which is honest about its reach: names in scripts outside
// this table are normalised by case and punctuation only.
var diacriticFolds = func() map[rune]rune {
const from = "àáâãäåāăąèéêëēĕėęěìíîïĩīĭįòóôõöøōŏőùúûüũūŭůűçćĉċčñńņňśŝşšýÿŷžźżðþßæœ"
const to = "aaaaaaaaaeeeeeeeeeiiiiiiiiooooooooouuuuuuuuucccccnnnnsssssyyyzzzdtsao"
f := []rune(from)
t := []rune(to)
m := make(map[rune]rune, len(f))
for i, r := range f {
if i < len(t) {
m[r] = t[i]
}
}
return m
}()
// foldDiacritic returns the base form of an accented letter, or r unchanged.
func foldDiacritic(r rune) rune {
if r < 128 {
return r
}
if base, ok := diacriticFolds[r]; ok {
return base
}
return r
}
// LooseKey is a more aggressive normalisation used ONLY to propose alias
// candidates for human review. It is never used for storage identity.
//
// It strips legal-form suffixes and honorifics, which is precisely why it is
// unsafe as an identity key: "Example Ltd" and "Example Inc" collapse together
// under it, and those may well be different companies.
func LooseKey(t domain.EntityType, name string) string {
n := Normalise(name)
stripped := stripNoiseWords(n)
if stripped == "" {
stripped = n
}
return string(t) + ":" + stripped
}
var legalSuffixes = map[string]bool{
"ltd": true, "limited": true, "plc": true, "llp": true, "llc": true,
"inc": true, "incorporated": true, "corp": true, "corporation": true,
"gmbh": true, "ag": true, "sa": true, "bv": true, "nv": true, "pty": true,
"co": true, "company": true, "group": true, "holdings": false,
}
var honorifics = map[string]bool{
"mr": true, "mrs": true, "ms": true, "miss": true, "dr": true,
"prof": true, "professor": true, "sir": true, "dame": true, "lord": true,
}
// AliasSimilarity scores how alike two names are, in 0..1.
//
// It is used only to decide whether an alias is worth proposing to a human.
// A high score never causes a merge on its own.
func AliasSimilarity(a, b string) float64 {
na, nb := Normalise(a), Normalise(b)
if na == "" || nb == "" {
return 0
}
if na == nb {
return 1
}
// Token overlap (Jaccard) catches reordering and extra words.
ta, tb := tokenSet(na), tokenSet(nb)
inter := 0
for w := range ta {
if tb[w] {
inter++
}
}
union := len(ta) + len(tb) - inter
jaccard := 0.0
if union > 0 {
jaccard = float64(inter) / float64(union)
}
// Character-level similarity catches typos and abbreviations.
edit := 1 - float64(levenshtein(na, nb))/float64(maxInt(len(na), len(nb)))
if edit < 0 {
edit = 0
}
// An acronym match is strong evidence worth a human's attention.
//
// The comparison also uses the suffix-stripped forms, because a legal form
// adds a word that breaks the letter count: "BBC" is an acronym of
// "British Broadcasting Corporation" but not of "British Broadcasting
// Corporation Ltd".
la, lb := stripNoiseWords(na), stripNoiseWords(nb)
if isAcronymOf(na, nb) || isAcronymOf(nb, na) ||
isAcronymOf(la, lb) || isAcronymOf(lb, la) {
return maxFloat(0.85, (jaccard+edit)/2)
}
// Names differing only by a legal form are worth proposing, but they are
// NOT merged: "Example Ltd" and "Example Inc" may be different companies.
if la != "" && la == lb {
return maxFloat(0.85, jaccard)
}
return 0.6*jaccard + 0.4*edit
}
// stripNoiseWords removes TRAILING legal forms and LEADING honorifics from a
// normalised name.
//
// Only the edges are trimmed. "Corporation" is a legal form at the end of
// "Example Corporation" but part of the name in "British Broadcasting
// Corporation Ltd", and stripping every occurrence would lose the word that
// makes the acronym work.
func stripNoiseWords(normalised string) string {
words := strings.Fields(normalised)
// At most one of each. Stripping repeatedly would take "Corporation" off
// "British Broadcasting Corporation Ltd" once "Ltd" had gone, losing the
// word the acronym depends on.
if len(words) > 1 && honorifics[words[0]] {
words = words[1:]
}
if len(words) > 1 && legalSuffixes[words[len(words)-1]] {
words = words[:len(words)-1]
}
return strings.Join(words, " ")
}
func tokenSet(s string) map[string]bool {
m := map[string]bool{}
for _, w := range strings.Fields(s) {
m[w] = true
}
return m
}
// isAcronymOf reports whether short is the initialism of long.
func isAcronymOf(short, long string) bool {
short = strings.ReplaceAll(short, " ", "")
words := strings.Fields(long)
if len(words) < 2 || len(short) != len(words) {
return false
}
for i, w := range words {
if len(w) == 0 || w[0] != short[i] {
return false
}
}
return true
}
// levenshtein computes edit distance with a two-row buffer.
func levenshtein(a, b string) int {
ra, rb := []rune(a), []rune(b)
if len(ra) == 0 {
return len(rb)
}
if len(rb) == 0 {
return len(ra)
}
prev := make([]int, len(rb)+1)
cur := make([]int, len(rb)+1)
for j := range prev {
prev[j] = j
}
for i := 1; i <= len(ra); i++ {
cur[0] = i
for j := 1; j <= len(rb); j++ {
cost := 1
if ra[i-1] == rb[j-1] {
cost = 0
}
cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost)
}
prev, cur = cur, prev
}
return prev[len(rb)]
}
// Canonicalise picks the display name for an entity seen under several spellings.
//
// The most frequent spelling wins; ties are broken by preferring the form with
// more capital letters (usually the properly cased one) and then alphabetically,
// so the choice is deterministic across runs.
func Canonicalise(observed []string) string {
if len(observed) == 0 {
return ""
}
counts := map[string]int{}
for _, o := range observed {
if t := strings.TrimSpace(o); t != "" {
counts[t]++
}
}
if len(counts) == 0 {
return ""
}
names := make([]string, 0, len(counts))
for n := range counts {
names = append(names, n)
}
sort.Slice(names, func(i, j int) bool {
if counts[names[i]] != counts[names[j]] {
return counts[names[i]] > counts[names[j]]
}
ci, cj := capitals(names[i]), capitals(names[j])
if ci != cj {
return ci > cj
}
return names[i] < names[j]
})
return names[0]
}
func capitals(s string) int {
n := 0
for _, r := range s {
if unicode.IsUpper(r) {
n++
}
}
return n
}
func maxInt(a, b int) int {
if a > b {
return a
}
return b
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
func maxFloat(a, b float64) float64 {
if a > b {
return a
}
return b
}
internal/entities/service.go
+441
View File
@@ -0,0 +1,441 @@
package entities
import (
"context"
"strings"
"time"
"github.com/andydixon/relata/internal/analysis"
"github.com/andydixon/relata/internal/config"
"github.com/andydixon/relata/internal/domain"
"github.com/andydixon/relata/internal/ids"
"github.com/andydixon/relata/internal/jobs"
"github.com/andydixon/relata/internal/observability"
)
// Service extracts entities and claims from an item's chunks and stores them.
//
// Extraction runs in two stages, as separate and separately-labelled sources of
// evidence: deterministic rules first, then model-assisted extraction for the
// broader concepts rules cannot reach. Every stored record carries the method
// that produced it, so a user can tell a matched pattern from a model's guess.
type Service struct {
entities domain.EntityRepository
aliases domain.AliasRepository
claims domain.ClaimRepository
analysis *analysis.Service
cfg *config.Config
}
// NewService builds the extraction service.
func NewService(ents domain.EntityRepository, aliases domain.AliasRepository,
claims domain.ClaimRepository, an *analysis.Service, cfg *config.Config) *Service {
return &Service{entities: ents, aliases: aliases, claims: claims, analysis: an, cfg: cfg}
}
// maxModelChunksPerItem bounds how many chunks are sent to the model for
// extraction, so a large document cannot consume unlimited model time.
const maxModelChunksPerItem = 12
// observation accumulates everything seen about one entity within an item.
type observation struct {
names []string
typ domain.EntityType
refs []domain.SourceRef
aliases map[string]bool
methods map[domain.ExtractionMethod]bool
count int
}
// ExtractForItem extracts entities from an item's chunks and stores them.
//
// Returns the number of distinct entities recorded. Model failures are returned
// to the caller, which treats them as degraded quality rather than a pipeline
// failure: the deterministic results are already persisted by then.
func (s *Service) ExtractForItem(ctx context.Context, item *domain.Item, chunks []*domain.Chunk, pool *jobs.Pool) (int, error) {
log := observability.LoggerFrom(ctx)
obs := map[string]*observation{}
record := func(name string, typ domain.EntityType, ref domain.SourceRef, method domain.ExtractionMethod, aliases []string) {
name = strings.TrimSpace(name)
if name == "" || !typ.Valid() {
return
}
key := Key(typ, name)
if Normalise(name) == "" {
return
}
o := obs[key]
if o == nil {
o = &observation{
typ: typ,
aliases: map[string]bool{},
methods: map[domain.ExtractionMethod]bool{},
}
obs[key] = o
}
o.names = append(o.names, name)
o.methods[method] = true
o.count++
// Source references are bounded per entity per item: a name repeated a
// hundred times does not need a hundred stored references.
if len(o.refs) < 8 {
o.refs = append(o.refs, ref)
}
for _, a := range aliases {
if a != "" && !strings.EqualFold(a, name) {
o.aliases[a] = true
}
}
}
// Stage 1: deterministic extraction over every chunk.
for _, c := range chunks {
for _, m := range ExtractDeterministic(c.Text) {
record(m.Name, m.Type, domain.SourceRef{
ItemID: item.ID,
ChunkID: c.ID,
// Offsets are translated from chunk-relative to item-relative,
// so a stored reference addresses the item's extracted text.
Start: c.StartOffset + m.Start,
End: c.StartOffset + m.End,
Quote: m.Quote,
Page: c.Page,
}, domain.MethodDeterministic, nil)
}
}
// Stage 2: model-assisted extraction over a bounded sample of chunks.
var modelErr error
if s.analysis.Enabled() {
limit := len(chunks)
if limit > maxModelChunksPerItem {
limit = maxModelChunksPerItem
}
for _, c := range chunks[:limit] {
if err := ctx.Err(); err != nil {
modelErr = err
break
}
release, err := pool.AcquireAI(ctx)
if err != nil {
modelErr = err
break
}
cands, err := s.analysis.ExtractEntities(ctx, c.Text)
release()
if err != nil {
modelErr = err
log.Debug("model entity extraction failed for a chunk", "error", err.Error())
continue
}
for _, cand := range cands {
ref := domain.SourceRef{ItemID: item.ID, ChunkID: c.ID, Page: c.Page}
// Locate the model's quotation in the chunk so the reference
// points at real text. Unlocatable evidence yields a reference
// to the chunk rather than a fabricated offset.
if cand.Evidence != "" {
if idx := strings.Index(c.Text, cand.Evidence); idx >= 0 {
ref.Start = c.StartOffset + idx
ref.End = ref.Start + len(cand.Evidence)
ref.Quote = cand.Evidence
}
}
if ref.Quote == "" {
if idx := strings.Index(c.Text, cand.Name); idx >= 0 {
ref.Start = c.StartOffset + idx
ref.End = ref.Start + len(cand.Name)
ref.Quote = cand.Name
}
}
record(cand.Name, cand.Type, ref, domain.MethodModel, cand.Aliases)
}
}
} else {
modelErr = analysis.ErrDisabled
}
stored := s.persist(ctx, item, obs)
return stored, modelErr
}
// persist writes the accumulated observations and proposes aliases.
func (s *Service) persist(ctx context.Context, item *domain.Item, obs map[string]*observation) int {
log := observability.LoggerFrom(ctx)
stored := 0
saved := make([]*domain.Entity, 0, len(obs))
for key, o := range obs {
canonical := Canonicalise(o.names)
if canonical == "" {
continue
}
method := domain.MethodModel
if o.methods[domain.MethodDeterministic] {
// A rule matched, which is the stronger provenance.
method = domain.MethodDeterministic
}
aliasList := make([]string, 0, len(o.aliases))
for a := range o.aliases {
aliasList = append(aliasList, a)
}
e := &domain.Entity{
ID: ids.New(),
WorkspaceID: item.WorkspaceID,
CanonicalName: canonical,
NormalisedKey: key,
Type: o.typ,
Aliases: aliasList,
SourceRefs: o.refs,
ItemIDs: []string{item.ID},
Method: method,
Mentions: o.count,
EvidenceScore: evidenceScore(o),
}
if method == domain.MethodModel {
e.ModelVersion = s.analysis.ModelID()
}
got, err := s.entities.UpsertByKey(ctx, e)
if err != nil {
log.Warn("could not store an entity", "entity", canonical, "error", err.Error())
continue
}
stored++
saved = append(saved, got)
}
s.proposeAliases(ctx, item.WorkspaceID, saved)
return stored
}
// evidenceScore reflects how well-attested an entity is. It is not a
// probability: it says "several independent signals point at this", nothing
// about whether the entity is real.
func evidenceScore(o *observation) float64 {
score := 0.3
if o.methods[domain.MethodDeterministic] {
score += 0.4
}
if o.methods[domain.MethodModel] && o.methods[domain.MethodDeterministic] {
// Both stages agreeing is the strongest evidence available here.
score += 0.2
}
switch {
case o.count >= 5:
score += 0.2
case o.count >= 2:
score += 0.1
}
if score > 1 {
score = 1
}
return score
}
// aliasProposalThreshold is the similarity at which a name pair is worth a
// human's attention.
const aliasProposalThreshold = 0.82
// maxAliasComparisons bounds the pairwise comparison so a workspace with many
// entities does not turn alias proposal into an O(n²) sweep.
const maxAliasComparisons = 400
// proposeAliases suggests that two entities may be the same thing.
//
// This is the safety valve for conservative normalisation: "BBC" and "British
// Broadcasting Corporation" get separate keys, and rather than merging them —
// which would be a guess with real consequences — Relata records a candidate
// alias and waits for a person to decide.
func (s *Service) proposeAliases(ctx context.Context, workspaceID string, saved []*domain.Entity) {
if len(saved) == 0 {
return
}
log := observability.LoggerFrom(ctx)
// Compare each newly-seen entity against existing entities that share its
// loose key — the cheap indexed way to find plausible pairs.
comparisons := 0
for _, e := range saved {
if comparisons >= maxAliasComparisons {
return
}
if e.Type == domain.EntityDate || e.Type == domain.EntityConcept {
// Dates are already normalised, and concepts overlap by nature.
continue
}
candidates, _, err := s.entities.List(ctx, workspaceID, domain.EntityFilter{
Types: []domain.EntityType{e.Type},
}, domain.Page{Limit: 200})
if err != nil {
return
}
for _, other := range candidates {
if other.ID == e.ID {
continue
}
comparisons++
if comparisons >= maxAliasComparisons {
break
}
sim := AliasSimilarity(e.CanonicalName, other.CanonicalName)
if sim < aliasProposalThreshold {
continue
}
// Record the proposal in a stable direction so the same pair does
// not generate two mirror-image proposals.
from, to := e, other
if from.ID > to.ID {
from, to = to, from
}
err := s.aliases.Upsert(ctx, &domain.EntityAlias{
ID: ids.New(),
WorkspaceID: workspaceID,
EntityID: from.ID,
Alias: to.CanonicalName,
Normalised: Normalise(to.CanonicalName),
CandidateEntityID: to.ID,
Status: domain.AliasCandidate,
Method: domain.MethodDeterministic,
Similarity: sim,
Reason: "These names are similar. Relata has NOT merged them: confirm only if they refer to the same thing.",
})
if err != nil {
log.Debug("could not record an alias proposal", "error", err.Error())
}
}
}
}
// ---------------------------------------------------------------------------
// Claims
// ---------------------------------------------------------------------------
// ExtractClaimsForItem extracts the statements an item makes.
//
// Deterministic extraction picks out sentences carrying a figure or a date —
// the statements most likely to be compared, agreed with or contradicted.
// Model-assisted extraction adds the rest. Either way, a claim is stored only
// when its quotation can be located in the source text.
func (s *Service) ExtractClaimsForItem(ctx context.Context, item *domain.Item, chunks []*domain.Chunk, pool *jobs.Pool) (int, error) {
log := observability.LoggerFrom(ctx)
now := time.Now().UTC()
var out []*domain.Claim
seen := map[string]bool{}
add := func(c *domain.Claim) {
if c.DedupeKey == "" || seen[c.DedupeKey] {
return
}
seen[c.DedupeKey] = true
out = append(out, c)
}
// Stage 1: deterministic claims.
for _, ch := range chunks {
for _, dc := range ExtractDeterministicClaims(ch.Text) {
add(&domain.Claim{
ID: ids.New(),
WorkspaceID: item.WorkspaceID,
ItemID: item.ID,
ChunkID: ch.ID,
Statement: dc.Statement,
SourceRef: domain.SourceRef{
ItemID: item.ID,
ChunkID: ch.ID,
Start: ch.StartOffset + dc.Start,
End: ch.StartOffset + dc.End,
Quote: dc.Quote,
Page: ch.Page,
},
Nature: domain.ClaimQuoted,
Subject: dc.Subject,
Predicate: dc.Predicate,
Numbers: dc.Numbers,
Dates: dc.Dates,
Polarity: dc.Polarity,
Method: domain.MethodDeterministic,
Confidence: 0.7,
DedupeKey: ids.Deterministic(item.WorkspaceID, item.ID, "det", dc.Quote),
CreatedAt: now,
})
}
}
// Stage 2: model-assisted claims.
var modelErr error
if s.analysis.Enabled() {
limit := len(chunks)
if limit > maxModelChunksPerItem {
limit = maxModelChunksPerItem
}
for _, ch := range chunks[:limit] {
if err := ctx.Err(); err != nil {
modelErr = err
break
}
release, err := pool.AcquireAI(ctx)
if err != nil {
modelErr = err
break
}
cands, err := s.analysis.ExtractClaims(ctx, ch.Text)
release()
if err != nil {
modelErr = err
log.Debug("model claim extraction failed for a chunk", "error", err.Error())
continue
}
for _, cand := range cands {
// The analysis layer has already verified the quotation exists
// and returned where it is.
if cand.QuoteOffset < 0 {
continue
}
subj, pred, nums, dates, pol := AnalyseStatement(cand.Statement)
if cand.Subject != "" {
subj = Normalise(cand.Subject)
}
add(&domain.Claim{
ID: ids.New(),
WorkspaceID: item.WorkspaceID,
ItemID: item.ID,
ChunkID: ch.ID,
Statement: cand.Statement,
SourceRef: domain.SourceRef{
ItemID: item.ID,
ChunkID: ch.ID,
Start: ch.StartOffset + cand.QuoteOffset,
End: ch.StartOffset + cand.QuoteOffset + len(cand.Quote),
Quote: cand.Quote,
Page: ch.Page,
},
Attribution: cand.Attribution,
Nature: cand.Nature,
Subject: subj,
Predicate: pred,
Numbers: nums,
Dates: dates,
Polarity: pol,
Method: domain.MethodModel,
ModelVersion: s.analysis.ModelID(),
Confidence: 0.6,
DedupeKey: ids.Deterministic(item.WorkspaceID, item.ID, "model", cand.Quote),
CreatedAt: now,
})
}
}
} else {
modelErr = analysis.ErrDisabled
}
if err := s.claims.ReplaceForItem(ctx, item.WorkspaceID, item.ID, out); err != nil {
return 0, err
}
return len(out), modelErr
}