v3 candidate
This commit is contained in:
Andy Dixon
@@ -0,0 +1,224 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// fuzzy.go implements context-triggered piecewise hashing (CTPH) compatible
|
||||
// with ssdeep. CTPH produces a short signature of the form
|
||||
// "blocksize:hash1:hash2" that stays similar when the input changes only a
|
||||
// little, so investigators can cluster related samples or match a file
|
||||
// against a known-bad ssdeep hash even after minor edits.
|
||||
//
|
||||
// The signature format and the rolling/sum hashes follow Andrew Tridgell's
|
||||
// spamsum / Jesse Kornblum's ssdeep so the output interoperates with the
|
||||
// standard tooling (e.g. an empty input hashes to "3::").
|
||||
|
||||
const (
|
||||
spamsumLength = 64
|
||||
rollingWindow = 7
|
||||
minBlocksize = 3
|
||||
hashInit = uint32(0x28021967)
|
||||
hashPrime = uint32(0x01000193)
|
||||
)
|
||||
|
||||
const b64alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
|
||||
|
||||
type rollState struct {
|
||||
h1, h2, h3 uint32
|
||||
window [rollingWindow]byte
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *rollState) roll(c byte) uint32 {
|
||||
r.h2 -= r.h1
|
||||
r.h2 += rollingWindow * uint32(c)
|
||||
r.h1 += uint32(c)
|
||||
r.h1 -= uint32(r.window[r.n%rollingWindow])
|
||||
r.window[r.n%rollingWindow] = c
|
||||
r.n++
|
||||
r.h3 = r.h3 << 5
|
||||
r.h3 ^= uint32(c)
|
||||
return r.h1 + r.h2 + r.h3
|
||||
}
|
||||
|
||||
func sumHash(c byte, h uint32) uint32 { return (h * hashPrime) ^ uint32(c) }
|
||||
|
||||
// FuzzyDigest computes the ssdeep-compatible signature for data.
|
||||
func FuzzyDigest(data []byte) string {
|
||||
bs := uint32(minBlocksize)
|
||||
for uint64(bs)*spamsumLength < uint64(len(data)) {
|
||||
bs *= 2
|
||||
}
|
||||
for {
|
||||
sig1, sig2 := ctphSigs(data, bs)
|
||||
if bs > minBlocksize && len(sig1) < spamsumLength/2 {
|
||||
bs /= 2
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%d:%s:%s", bs, sig1, sig2)
|
||||
}
|
||||
}
|
||||
|
||||
func ctphSigs(data []byte, bs uint32) (string, string) {
|
||||
var rs rollState
|
||||
h1, h2 := hashInit, hashInit
|
||||
var sig1, sig2 strings.Builder
|
||||
var lastRoll uint32
|
||||
|
||||
for _, c := range data {
|
||||
h1 = sumHash(c, h1)
|
||||
h2 = sumHash(c, h2)
|
||||
rh := rs.roll(c)
|
||||
lastRoll = rh
|
||||
|
||||
if rh%bs == bs-1 && sig1.Len() < spamsumLength-1 {
|
||||
sig1.WriteByte(b64alphabet[h1%64])
|
||||
h1 = hashInit
|
||||
}
|
||||
if rh%(bs*2) == bs*2-1 && sig2.Len() < spamsumLength/2-1 {
|
||||
sig2.WriteByte(b64alphabet[h2%64])
|
||||
h2 = hashInit
|
||||
}
|
||||
}
|
||||
if lastRoll != 0 {
|
||||
sig1.WriteByte(b64alphabet[h1%64])
|
||||
sig2.WriteByte(b64alphabet[h2%64])
|
||||
}
|
||||
return sig1.String(), sig2.String()
|
||||
}
|
||||
|
||||
// FuzzyHash computes the CTPH signature over a document range.
|
||||
func FuzzyHash(d *document.Document, start, length int64) (map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
buf, err := d.Range(start, length)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sig := FuzzyDigest(buf)
|
||||
bs := strings.SplitN(sig, ":", 2)[0]
|
||||
return map[string]any{
|
||||
"algo": "ssdeep",
|
||||
"signature": sig,
|
||||
"blocksize": bs,
|
||||
"length": length,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FuzzyCompare returns a 0-100 similarity score between two ssdeep
|
||||
// signatures. 0 means unrelated (or incomparable block sizes); 100 means the
|
||||
// signatures are identical.
|
||||
func FuzzyCompare(a, b string) int {
|
||||
bs1, a1, a2, ok1 := parseSig(a)
|
||||
bs2, b1, b2, ok2 := parseSig(b)
|
||||
if !ok1 || !ok2 {
|
||||
return 0
|
||||
}
|
||||
if bs1 != bs2 && bs1 != bs2*2 && bs2 != bs1*2 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case bs1 == bs2:
|
||||
s1 := scoreStrings(a1, b1, bs1)
|
||||
s2 := scoreStrings(a2, b2, bs1*2)
|
||||
if s2 > s1 {
|
||||
return s2
|
||||
}
|
||||
return s1
|
||||
case bs1 == bs2*2:
|
||||
return scoreStrings(a1, b2, bs1)
|
||||
default: // bs2 == bs1*2
|
||||
return scoreStrings(a2, b1, bs2)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSig(s string) (uint32, string, string, bool) {
|
||||
parts := strings.SplitN(s, ":", 3)
|
||||
if len(parts) != 3 {
|
||||
return 0, "", "", false
|
||||
}
|
||||
bs, err := strconv.ParseUint(parts[0], 10, 32)
|
||||
if err != nil || bs == 0 {
|
||||
return 0, "", "", false
|
||||
}
|
||||
return uint32(bs), parts[1], parts[2], true
|
||||
}
|
||||
|
||||
func scoreStrings(s1, s2 string, blocksize uint32) int {
|
||||
if !hasCommonSubstring(s1, s2, rollingWindow) {
|
||||
return 0
|
||||
}
|
||||
if s1 == s2 {
|
||||
return 100
|
||||
}
|
||||
d := editDistance(s1, s2)
|
||||
if len(s1)+len(s2) == 0 {
|
||||
return 0
|
||||
}
|
||||
score := (d * spamsumLength) / (len(s1) + len(s2))
|
||||
score = (100 * score) / spamsumLength
|
||||
score = 100 - score
|
||||
// Small block sizes describe little data, so cap the score they can claim.
|
||||
cap := int(blocksize) / minBlocksize * minInt(len(s1), len(s2))
|
||||
if score > cap {
|
||||
score = cap
|
||||
}
|
||||
if score > 100 {
|
||||
score = 100
|
||||
}
|
||||
if score < 0 {
|
||||
score = 0
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func hasCommonSubstring(s1, s2 string, n int) bool {
|
||||
if len(s1) < n || len(s2) < n {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(s1))
|
||||
for i := 0; i+n <= len(s1); i++ {
|
||||
seen[s1[i:i+n]] = struct{}{}
|
||||
}
|
||||
for i := 0; i+n <= len(s2); i++ {
|
||||
if _, ok := seen[s2[i:i+n]]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// editDistance is classic Levenshtein with unit insert/delete/substitute cost.
|
||||
func editDistance(s1, s2 string) int {
|
||||
m, n := len(s1), len(s2)
|
||||
prev := make([]int, n+1)
|
||||
cur := make([]int, n+1)
|
||||
for j := 0; j <= n; j++ {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= m; i++ {
|
||||
cur[0] = i
|
||||
for j := 1; j <= n; j++ {
|
||||
cost := 1
|
||||
if s1[i-1] == s2[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost)
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
return prev[n]
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user