312 lines
8.5 KiB
Go
312 lines
8.5 KiB
Go
// Package entities extracts and resolves identifiable subjects from source
|
||
// material.
|
||
//
|
||
// Resolution is deliberately conservative. Two entities are treated as the same
|
||
// only when their normalised keys match exactly. Names that merely look alike
|
||
// produce a *candidate alias* awaiting human confirmation — they are never
|
||
// merged automatically, because "Example Ltd" and "Example Holdings Ltd" are
|
||
// different companies far more often than they are typos of each other.
|
||
package entities
|
||
|
||
import (
|
||
"sort"
|
||
"strings"
|
||
"unicode"
|
||
|
||
"github.com/andydixon/relata/internal/domain"
|
||
)
|
||
|
||
// Normalise produces the exact-match key for an entity name.
|
||
//
|
||
// The transformation is intentionally shallow: case folding, whitespace
|
||
// collapsing, punctuation and diacritic removal. It does not strip company
|
||
// suffixes or expand abbreviations, because doing so would silently merge
|
||
// distinct entities — exactly what the design forbids.
|
||
func Normalise(name string) string {
|
||
var b strings.Builder
|
||
b.Grow(len(name))
|
||
|
||
prevSpace := true // leading space suppressed
|
||
for _, r := range strings.ToLower(strings.TrimSpace(name)) {
|
||
switch {
|
||
case unicode.IsLetter(r) || unicode.IsDigit(r):
|
||
b.WriteRune(foldDiacritic(r))
|
||
prevSpace = false
|
||
case r == '\'' || r == '’' || r == 'ʼ':
|
||
// Apostrophes are dropped rather than turned into a break, so
|
||
// "O'Brien" and "OBrien" share a key while "O Brien" (two words)
|
||
// is a different thing.
|
||
|
||
case r == '&':
|
||
// "&" carries meaning in company names ("Marks & Spencer").
|
||
if !prevSpace {
|
||
b.WriteByte(' ')
|
||
}
|
||
b.WriteString("and")
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
case unicode.IsSpace(r) || unicode.IsPunct(r) || unicode.IsSymbol(r):
|
||
if !prevSpace {
|
||
b.WriteByte(' ')
|
||
prevSpace = true
|
||
}
|
||
}
|
||
}
|
||
return strings.TrimSpace(b.String())
|
||
}
|
||
|
||
// Key builds the storage key that determines entity identity. Type is part of
|
||
// the key: a place called "Washington" is not the person "Washington".
|
||
func Key(t domain.EntityType, name string) string {
|
||
return string(t) + ":" + Normalise(name)
|
||
}
|
||
|
||
// diacriticFolds maps accented Latin letters onto their base form so that
|
||
// "Zoë" and "Zoe" share a key. It covers Latin-1 and Latin Extended-A rather
|
||
// than every script, which is honest about its reach: names in scripts outside
|
||
// this table are normalised by case and punctuation only.
|
||
var diacriticFolds = func() map[rune]rune {
|
||
const from = "àáâãäåāăąèéêëēĕėęěìíîïĩīĭįòóôõöøōŏőùúûüũūŭůűçćĉċčñńņňśŝşšýÿŷžźżðþßæœ"
|
||
const to = "aaaaaaaaaeeeeeeeeeiiiiiiiiooooooooouuuuuuuuucccccnnnnsssssyyyzzzdtsao"
|
||
|
||
f := []rune(from)
|
||
t := []rune(to)
|
||
m := make(map[rune]rune, len(f))
|
||
for i, r := range f {
|
||
if i < len(t) {
|
||
m[r] = t[i]
|
||
}
|
||
}
|
||
return m
|
||
}()
|
||
|
||
// foldDiacritic returns the base form of an accented letter, or r unchanged.
|
||
func foldDiacritic(r rune) rune {
|
||
if r < 128 {
|
||
return r
|
||
}
|
||
if base, ok := diacriticFolds[r]; ok {
|
||
return base
|
||
}
|
||
return r
|
||
}
|
||
|
||
// LooseKey is a more aggressive normalisation used ONLY to propose alias
|
||
// candidates for human review. It is never used for storage identity.
|
||
//
|
||
// It strips legal-form suffixes and honorifics, which is precisely why it is
|
||
// unsafe as an identity key: "Example Ltd" and "Example Inc" collapse together
|
||
// under it, and those may well be different companies.
|
||
func LooseKey(t domain.EntityType, name string) string {
|
||
n := Normalise(name)
|
||
stripped := stripNoiseWords(n)
|
||
if stripped == "" {
|
||
stripped = n
|
||
}
|
||
return string(t) + ":" + stripped
|
||
}
|
||
|
||
var legalSuffixes = map[string]bool{
|
||
"ltd": true, "limited": true, "plc": true, "llp": true, "llc": true,
|
||
"inc": true, "incorporated": true, "corp": true, "corporation": true,
|
||
"gmbh": true, "ag": true, "sa": true, "bv": true, "nv": true, "pty": true,
|
||
"co": true, "company": true, "group": true, "holdings": false,
|
||
}
|
||
|
||
var honorifics = map[string]bool{
|
||
"mr": true, "mrs": true, "ms": true, "miss": true, "dr": true,
|
||
"prof": true, "professor": true, "sir": true, "dame": true, "lord": true,
|
||
}
|
||
|
||
// AliasSimilarity scores how alike two names are, in 0..1.
|
||
//
|
||
// It is used only to decide whether an alias is worth proposing to a human.
|
||
// A high score never causes a merge on its own.
|
||
func AliasSimilarity(a, b string) float64 {
|
||
na, nb := Normalise(a), Normalise(b)
|
||
if na == "" || nb == "" {
|
||
return 0
|
||
}
|
||
if na == nb {
|
||
return 1
|
||
}
|
||
|
||
// Token overlap (Jaccard) catches reordering and extra words.
|
||
ta, tb := tokenSet(na), tokenSet(nb)
|
||
inter := 0
|
||
for w := range ta {
|
||
if tb[w] {
|
||
inter++
|
||
}
|
||
}
|
||
union := len(ta) + len(tb) - inter
|
||
jaccard := 0.0
|
||
if union > 0 {
|
||
jaccard = float64(inter) / float64(union)
|
||
}
|
||
|
||
// Character-level similarity catches typos and abbreviations.
|
||
edit := 1 - float64(levenshtein(na, nb))/float64(maxInt(len(na), len(nb)))
|
||
if edit < 0 {
|
||
edit = 0
|
||
}
|
||
|
||
// An acronym match is strong evidence worth a human's attention.
|
||
//
|
||
// The comparison also uses the suffix-stripped forms, because a legal form
|
||
// adds a word that breaks the letter count: "BBC" is an acronym of
|
||
// "British Broadcasting Corporation" but not of "British Broadcasting
|
||
// Corporation Ltd".
|
||
la, lb := stripNoiseWords(na), stripNoiseWords(nb)
|
||
if isAcronymOf(na, nb) || isAcronymOf(nb, na) ||
|
||
isAcronymOf(la, lb) || isAcronymOf(lb, la) {
|
||
return maxFloat(0.85, (jaccard+edit)/2)
|
||
}
|
||
|
||
// Names differing only by a legal form are worth proposing, but they are
|
||
// NOT merged: "Example Ltd" and "Example Inc" may be different companies.
|
||
if la != "" && la == lb {
|
||
return maxFloat(0.85, jaccard)
|
||
}
|
||
return 0.6*jaccard + 0.4*edit
|
||
}
|
||
|
||
// stripNoiseWords removes TRAILING legal forms and LEADING honorifics from a
|
||
// normalised name.
|
||
//
|
||
// Only the edges are trimmed. "Corporation" is a legal form at the end of
|
||
// "Example Corporation" but part of the name in "British Broadcasting
|
||
// Corporation Ltd", and stripping every occurrence would lose the word that
|
||
// makes the acronym work.
|
||
func stripNoiseWords(normalised string) string {
|
||
words := strings.Fields(normalised)
|
||
|
||
// At most one of each. Stripping repeatedly would take "Corporation" off
|
||
// "British Broadcasting Corporation Ltd" once "Ltd" had gone, losing the
|
||
// word the acronym depends on.
|
||
if len(words) > 1 && honorifics[words[0]] {
|
||
words = words[1:]
|
||
}
|
||
if len(words) > 1 && legalSuffixes[words[len(words)-1]] {
|
||
words = words[:len(words)-1]
|
||
}
|
||
return strings.Join(words, " ")
|
||
}
|
||
|
||
func tokenSet(s string) map[string]bool {
|
||
m := map[string]bool{}
|
||
for _, w := range strings.Fields(s) {
|
||
m[w] = true
|
||
}
|
||
return m
|
||
}
|
||
|
||
// isAcronymOf reports whether short is the initialism of long.
|
||
func isAcronymOf(short, long string) bool {
|
||
short = strings.ReplaceAll(short, " ", "")
|
||
words := strings.Fields(long)
|
||
if len(words) < 2 || len(short) != len(words) {
|
||
return false
|
||
}
|
||
for i, w := range words {
|
||
if len(w) == 0 || w[0] != short[i] {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// levenshtein computes edit distance with a two-row buffer.
|
||
func levenshtein(a, b string) int {
|
||
ra, rb := []rune(a), []rune(b)
|
||
if len(ra) == 0 {
|
||
return len(rb)
|
||
}
|
||
if len(rb) == 0 {
|
||
return len(ra)
|
||
}
|
||
prev := make([]int, len(rb)+1)
|
||
cur := make([]int, len(rb)+1)
|
||
for j := range prev {
|
||
prev[j] = j
|
||
}
|
||
for i := 1; i <= len(ra); i++ {
|
||
cur[0] = i
|
||
for j := 1; j <= len(rb); j++ {
|
||
cost := 1
|
||
if ra[i-1] == rb[j-1] {
|
||
cost = 0
|
||
}
|
||
cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost)
|
||
}
|
||
prev, cur = cur, prev
|
||
}
|
||
return prev[len(rb)]
|
||
}
|
||
|
||
// Canonicalise picks the display name for an entity seen under several spellings.
|
||
//
|
||
// The most frequent spelling wins; ties are broken by preferring the form with
|
||
// more capital letters (usually the properly cased one) and then alphabetically,
|
||
// so the choice is deterministic across runs.
|
||
func Canonicalise(observed []string) string {
|
||
if len(observed) == 0 {
|
||
return ""
|
||
}
|
||
counts := map[string]int{}
|
||
for _, o := range observed {
|
||
if t := strings.TrimSpace(o); t != "" {
|
||
counts[t]++
|
||
}
|
||
}
|
||
if len(counts) == 0 {
|
||
return ""
|
||
}
|
||
names := make([]string, 0, len(counts))
|
||
for n := range counts {
|
||
names = append(names, n)
|
||
}
|
||
sort.Slice(names, func(i, j int) bool {
|
||
if counts[names[i]] != counts[names[j]] {
|
||
return counts[names[i]] > counts[names[j]]
|
||
}
|
||
ci, cj := capitals(names[i]), capitals(names[j])
|
||
if ci != cj {
|
||
return ci > cj
|
||
}
|
||
return names[i] < names[j]
|
||
})
|
||
return names[0]
|
||
}
|
||
|
||
func capitals(s string) int {
|
||
n := 0
|
||
for _, r := range s {
|
||
if unicode.IsUpper(r) {
|
||
n++
|
||
}
|
||
}
|
||
return n
|
||
}
|
||
|
||
func maxInt(a, b int) int {
|
||
if a > b {
|
||
return a
|
||
}
|
||
return b
|
||
}
|
||
|
||
func minInt(a, b int) int {
|
||
if a < b {
|
||
return a
|
||
}
|
||
return b
|
||
}
|
||
|
||
func maxFloat(a, b float64) float64 {
|
||
if a > b {
|
||
return a
|
||
}
|
||
return b
|
||
}
|