v3 candidate
This commit is contained in:
Andy Dixon
@@ -0,0 +1,876 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// Signature describes a carveable object. `validate` is run against
|
||||
// the candidate start and returns the object length if it is a real
|
||||
// object, or 0 to reject. `trailer`, if present, must appear before
|
||||
// the object ends (for simple trailer-based terminations).
|
||||
type Signature struct {
|
||||
Kind string
|
||||
Magic []byte
|
||||
Extension string
|
||||
// Anchor is how many bytes into the object the magic appears. It is 0 for
|
||||
// almost every format (magic at offset 0), but a few — notably ISO-BMFF
|
||||
// (MP4/MOV), whose `ftyp` magic follows a 4-byte box size — carry it.
|
||||
Anchor int64
|
||||
Validate func(d *document.Document, start, max int64) (length int64, confidence float64)
|
||||
}
|
||||
|
||||
// carveSignatures is the phase-1 signature set. We stay conservative
|
||||
// and only declare a carve when we can determine length reliably.
|
||||
var carveSignatures = []Signature{
|
||||
{Kind: "png", Magic: []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, Extension: "png", Validate: validatePNG},
|
||||
{Kind: "jpeg", Magic: []byte{0xff, 0xd8, 0xff}, Extension: "jpg", Validate: validateJPEG},
|
||||
{Kind: "gif87a", Magic: []byte("GIF87a"), Extension: "gif", Validate: validateGIF},
|
||||
{Kind: "gif89a", Magic: []byte("GIF89a"), Extension: "gif", Validate: validateGIF},
|
||||
{Kind: "zip", Magic: []byte{'P', 'K', 0x03, 0x04}, Extension: "zip", Validate: validateZIP},
|
||||
{Kind: "pdf", Magic: []byte("%PDF-"), Extension: "pdf", Validate: validatePDF},
|
||||
{Kind: "elf", Magic: []byte{0x7f, 'E', 'L', 'F'}, Extension: "elf", Validate: validateELF},
|
||||
{Kind: "mz", Magic: []byte{'M', 'Z'}, Extension: "exe", Validate: validatePE},
|
||||
{Kind: "gzip", Magic: []byte{0x1f, 0x8b, 0x08}, Extension: "gz", Validate: validateGZIP},
|
||||
{Kind: "bzip2", Magic: []byte{'B', 'Z', 'h'}, Extension: "bz2", Validate: validateBzip2},
|
||||
{Kind: "sevenz", Magic: []byte{'7', 'z', 0xbc, 0xaf, 0x27, 0x1c}, Extension: "7z", Validate: validate7z},
|
||||
{Kind: "rar4", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x00}, Extension: "rar", Validate: validateRAR},
|
||||
{Kind: "rar5", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00}, Extension: "rar", Validate: validateRAR},
|
||||
{Kind: "wav", Magic: []byte("RIFF"), Extension: "wav", Validate: validateRIFF},
|
||||
{Kind: "mach-o-le32", Magic: []byte{0xce, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-le64", Magic: []byte{0xcf, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-be32", Magic: []byte{0xfe, 0xed, 0xfa, 0xce}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-be64", Magic: []byte{0xfe, 0xed, 0xfa, 0xcf}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "sqlite", Magic: []byte("SQLite format 3\x00"), Extension: "sqlite", Validate: validateSQLite},
|
||||
{Kind: "mp3", Magic: []byte("ID3"), Extension: "mp3", Validate: validateID3},
|
||||
{Kind: "ogg", Magic: []byte("OggS"), Extension: "ogg", Validate: validateOgg},
|
||||
{Kind: "matroska", Magic: []byte{0x1a, 0x45, 0xdf, 0xa3}, Extension: "mkv", Validate: validateMatroska},
|
||||
{Kind: "flac", Magic: []byte("fLaC"), Extension: "flac", Validate: validateFLAC},
|
||||
{Kind: "bmp", Magic: []byte{'B', 'M'}, Extension: "bmp", Validate: validateBMP},
|
||||
{Kind: "mp4", Magic: []byte("ftyp"), Anchor: 4, Extension: "mp4", Validate: validateMP4},
|
||||
}
|
||||
|
||||
// Carve scans the document for known signatures and returns a list of
|
||||
// discovered objects with offset, length, kind, and confidence.
|
||||
func Carve(d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
|
||||
return CarveContext(context.Background(), d, start, length, maxHits)
|
||||
}
|
||||
|
||||
func CarveContext(ctx context.Context, d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
if maxHits <= 0 {
|
||||
maxHits = 10_000
|
||||
}
|
||||
results := make([]map[string]any, 0, 64)
|
||||
|
||||
// Build a flat list of (magicBytes, sigIndex) for a simple first-pass scan.
|
||||
// Phase 1 uses per-signature naive scan chunks.
|
||||
for idx, sig := range carveSignatures {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits, err := SearchBytesContext(ctx, d, start, length, sig.Magic, maxHits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, h := range hits {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The object begins Anchor bytes before the magic (0 for most formats).
|
||||
objStart := h.Offset - sig.Anchor
|
||||
if objStart < start {
|
||||
continue
|
||||
}
|
||||
ln, conf := int64(0), 0.4
|
||||
if sig.Validate != nil {
|
||||
ln, conf = sig.Validate(d, objStart, start+length)
|
||||
}
|
||||
if ln <= 0 {
|
||||
// record low-confidence magic hit only for kinds where length is not determinable
|
||||
if sig.Kind == "mz" || sig.Kind == "elf" || sig.Kind == "mach-o-le32" ||
|
||||
sig.Kind == "mach-o-le64" || sig.Kind == "mach-o-be32" || sig.Kind == "mach-o-be64" ||
|
||||
sig.Kind == "sqlite" || sig.Kind == "mp3" || sig.Kind == "matroska" ||
|
||||
sig.Kind == "flac" || sig.Kind == "bzip2" || sig.Kind == "rar4" || sig.Kind == "rar5" {
|
||||
results = append(results, map[string]any{
|
||||
"kind": sig.Kind,
|
||||
"extension": sig.Extension,
|
||||
"offset": objStart,
|
||||
"length": int64(0),
|
||||
"confidence": 0.3,
|
||||
"status": "magic-only",
|
||||
"sigIndex": idx,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
results = append(results, map[string]any{
|
||||
"kind": sig.Kind,
|
||||
"extension": sig.Extension,
|
||||
"offset": objStart,
|
||||
"length": ln,
|
||||
"confidence": conf,
|
||||
"status": "valid",
|
||||
"sigIndex": idx,
|
||||
})
|
||||
if len(results) >= maxHits {
|
||||
return results, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// --- validators ---
|
||||
|
||||
func validatePNG(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Walk chunks until IEND
|
||||
off := start + 8
|
||||
for off < max {
|
||||
hdr, err := d.Range(off, 8)
|
||||
if err != nil || len(hdr) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.BigEndian.Uint32(hdr[0:4]))
|
||||
typ := string(hdr[4:8])
|
||||
// Skip payload + CRC (4 bytes)
|
||||
off += 8 + size + 4
|
||||
if typ == "IEND" {
|
||||
return off - start, 0.95
|
||||
}
|
||||
if off > max {
|
||||
return 0, 0
|
||||
}
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateJPEG(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Walk segments until EOI (0xFFD9)
|
||||
off := start + 2
|
||||
for off < max-1 {
|
||||
buf, err := d.Range(off, 2)
|
||||
if err != nil || len(buf) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
if buf[0] != 0xff {
|
||||
return 0, 0
|
||||
}
|
||||
marker := buf[1]
|
||||
off += 2
|
||||
// No-length markers
|
||||
if marker == 0xd9 { // EOI
|
||||
return off - start, 0.95
|
||||
}
|
||||
if marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker == 0x00 {
|
||||
continue
|
||||
}
|
||||
// Read length
|
||||
lb, err := d.Range(off, 2)
|
||||
if err != nil || len(lb) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
ln := int64(binary.BigEndian.Uint16(lb))
|
||||
off += ln
|
||||
if marker == 0xda { // SOS, scan until next 0xff!=0
|
||||
for off < max {
|
||||
b, err := d.Range(off, 2)
|
||||
if err != nil || len(b) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
if b[0] == 0xff && b[1] != 0x00 && !(b[1] >= 0xd0 && b[1] <= 0xd7) {
|
||||
break
|
||||
}
|
||||
off++
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateGIF(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Minimal: locate trailer 0x3B
|
||||
buf, err := d.Range(start, min64(max-start, 1<<20))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.IndexByte(buf[13:], 0x3b)
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
return int64(idx + 13 + 1), 0.7
|
||||
}
|
||||
|
||||
func validateZIP(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Search forward for End of Central Directory record 0x06054b50.
|
||||
const sig = "\x50\x4b\x05\x06"
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.LastIndex(buf, []byte(sig))
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
// EOCD record is at least 22 bytes; read comment length.
|
||||
if idx+22 > len(buf) {
|
||||
return 0, 0
|
||||
}
|
||||
commentLen := int(binary.LittleEndian.Uint16(buf[idx+20 : idx+22]))
|
||||
end := int64(idx + 22 + commentLen)
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
func validatePDF(d *document.Document, start, max int64) (int64, float64) {
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.LastIndex(buf, []byte("%%EOF"))
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
return int64(idx + 5), 0.85
|
||||
}
|
||||
|
||||
func validateELF(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 64)
|
||||
if err != nil || len(hdr) < 52 {
|
||||
return 0, 0
|
||||
}
|
||||
if hdr[4] != 1 && hdr[4] != 2 {
|
||||
return 0, 0
|
||||
}
|
||||
is64 := hdr[4] == 2
|
||||
little := hdr[5] == 1
|
||||
var end int64
|
||||
if is64 {
|
||||
if len(hdr) < 64 {
|
||||
return 0, 0
|
||||
}
|
||||
// e_shoff at 40, e_shentsize at 58, e_shnum at 60
|
||||
off := binary.LittleEndian.Uint64(hdr[40:48])
|
||||
entSz := uint64(binary.LittleEndian.Uint16(hdr[58:60]))
|
||||
num := uint64(binary.LittleEndian.Uint16(hdr[60:62]))
|
||||
if !little {
|
||||
off = binary.BigEndian.Uint64(hdr[40:48])
|
||||
entSz = uint64(binary.BigEndian.Uint16(hdr[58:60]))
|
||||
num = uint64(binary.BigEndian.Uint16(hdr[60:62]))
|
||||
}
|
||||
end = int64(off + entSz*num)
|
||||
} else {
|
||||
off := binary.LittleEndian.Uint32(hdr[32:36])
|
||||
entSz := uint32(binary.LittleEndian.Uint16(hdr[46:48]))
|
||||
num := uint32(binary.LittleEndian.Uint16(hdr[48:50]))
|
||||
if !little {
|
||||
off = binary.BigEndian.Uint32(hdr[32:36])
|
||||
entSz = uint32(binary.BigEndian.Uint16(hdr[46:48]))
|
||||
num = uint32(binary.BigEndian.Uint16(hdr[48:50]))
|
||||
}
|
||||
end = int64(off) + int64(entSz)*int64(num)
|
||||
}
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
func validatePE(d *document.Document, start, max int64) (int64, float64) {
|
||||
// e_lfanew at offset 0x3C points to PE header
|
||||
hdr, err := d.Range(start, 64)
|
||||
if err != nil || len(hdr) < 64 {
|
||||
return 0, 0
|
||||
}
|
||||
peOff := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40]))
|
||||
if peOff < 0x40 || start+peOff+4 > max {
|
||||
return 0, 0
|
||||
}
|
||||
sig, err := d.Range(start+peOff, 4)
|
||||
if err != nil || string(sig) != "PE\x00\x00" {
|
||||
return 0, 0.3
|
||||
}
|
||||
coff, err := d.Range(start+peOff+4, 20)
|
||||
if err != nil || len(coff) < 20 {
|
||||
return 0, 0.4
|
||||
}
|
||||
numSections := int(binary.LittleEndian.Uint16(coff[2:4]))
|
||||
sizeOpt := int64(binary.LittleEndian.Uint16(coff[16:18]))
|
||||
if numSections <= 0 || numSections > 96 || sizeOpt < 0 || sizeOpt > 4096 {
|
||||
return 0, 0.5
|
||||
}
|
||||
sectOff := start + peOff + 24 + sizeOpt
|
||||
end := peOff + 24 + sizeOpt + int64(numSections*40)
|
||||
for i := 0; i < numSections; i++ {
|
||||
sh, err := d.Range(sectOff+int64(i*40), 40)
|
||||
if err != nil || len(sh) < 40 {
|
||||
return 0, 0.5
|
||||
}
|
||||
rawSize := int64(binary.LittleEndian.Uint32(sh[16:20]))
|
||||
rawPtr := int64(binary.LittleEndian.Uint32(sh[20:24]))
|
||||
if rawSize < 0 || rawPtr < 0 {
|
||||
return 0, 0.5
|
||||
}
|
||||
if rawSize > 0 && rawPtr+rawSize > end {
|
||||
end = rawPtr + rawSize
|
||||
}
|
||||
}
|
||||
if end <= peOff || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
func validateGZIP(d *document.Document, start, max int64) (int64, float64) {
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil || len(buf) < 18 {
|
||||
return 0, 0
|
||||
}
|
||||
br := bytes.NewReader(buf)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
zr.Multistream(false)
|
||||
if _, err := io.Copy(io.Discard, io.LimitReader(zr, 512<<20)); err != nil {
|
||||
_ = zr.Close()
|
||||
return 0, 0
|
||||
}
|
||||
if err := zr.Close(); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
consumed := int64(len(buf)) - int64(br.Len())
|
||||
if consumed <= 10 || start+consumed > max {
|
||||
return 0, 0
|
||||
}
|
||||
return consumed, 0.9
|
||||
}
|
||||
|
||||
// validateBzip2 locates the bit-aligned end-of-stream marker (0x177245385090,
|
||||
// the "sqrt(pi)" magic) and returns the stream length including the trailing
|
||||
// 32-bit combined CRC. bzip2 carries no length in its header, so this is the
|
||||
// only reliable way to bound the object.
|
||||
func validateBzip2(d *document.Document, start, max int64) (int64, float64) {
|
||||
if start+4 > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
hdr, err := d.Range(start, 4)
|
||||
if err != nil || len(hdr) < 4 || hdr[0] != 'B' || hdr[1] != 'Z' || hdr[2] != 'h' || hdr[3] < '1' || hdr[3] > '9' {
|
||||
return 0, 0
|
||||
}
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil || len(buf) < 14 {
|
||||
return 0, 0.4
|
||||
}
|
||||
const eos uint64 = 0x177245385090
|
||||
const mask48 uint64 = (1 << 48) - 1
|
||||
var w uint64
|
||||
for i := 0; i < len(buf); i++ {
|
||||
w = (w << 8) | uint64(buf[i])
|
||||
// Check all eight bit alignments ending at or before this byte.
|
||||
for s := 0; s < 8; s++ {
|
||||
if (w>>uint(s))&mask48 == eos {
|
||||
endBit := (int64(i)+1)*8 - int64(s) + 32 // include stream CRC
|
||||
endByte := (endBit + 7) / 8
|
||||
if endByte > 14 && start+endByte <= max {
|
||||
return endByte, 0.85
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validate7z(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
nextHdrOff := binary.LittleEndian.Uint64(hdr[12:20])
|
||||
nextHdrSz := binary.LittleEndian.Uint64(hdr[20:28])
|
||||
end := int64(32 + nextHdrOff + nextHdrSz)
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
// validateRAR walks the block headers of a RAR4 or RAR5 archive to the
|
||||
// end-of-archive marker. The two formats differ structurally, so the version
|
||||
// is detected from the signature byte at offset 6.
|
||||
func validateRAR(d *document.Document, start, max int64) (int64, float64) {
|
||||
sig, err := d.Range(start, 8)
|
||||
if err != nil || len(sig) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
if sig[6] == 0x01 {
|
||||
return validateRAR5(d, start, max)
|
||||
}
|
||||
return validateRAR4(d, start, max)
|
||||
}
|
||||
|
||||
func validateRAR4(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start + 7 // 7-byte signature
|
||||
for n := 0; n < 100000; n++ {
|
||||
h, err := d.Range(pos, 11)
|
||||
if err != nil || len(h) < 7 {
|
||||
return 0, 0.4
|
||||
}
|
||||
flags := binary.LittleEndian.Uint16(h[3:5])
|
||||
headSize := int64(binary.LittleEndian.Uint16(h[5:7]))
|
||||
if headSize < 7 {
|
||||
return 0, 0.4
|
||||
}
|
||||
blockSize := headSize
|
||||
if flags&0x8000 != 0 { // ADD_SIZE present
|
||||
if len(h) < 11 {
|
||||
return 0, 0.4
|
||||
}
|
||||
blockSize += int64(binary.LittleEndian.Uint32(h[7:11]))
|
||||
}
|
||||
pos += blockSize
|
||||
if h[2] == 0x7b { // terminator block
|
||||
if pos > start && pos <= max {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validateRAR5(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start + 8 // 8-byte signature
|
||||
for n := 0; n < 100000; n++ {
|
||||
// Each block: CRC32(4) + vint(header_size) + header_size bytes [+ data].
|
||||
h, err := d.Range(pos, 4+11)
|
||||
if err != nil || len(h) < 5 {
|
||||
return 0, 0.4
|
||||
}
|
||||
headerSize, vlen := rar5Vint(h, 4)
|
||||
if vlen == 0 || headerSize == 0 {
|
||||
return 0, 0.4
|
||||
}
|
||||
hdrStart := pos + 4 + int64(vlen)
|
||||
body, err := d.Range(hdrStart, int64(headerSize))
|
||||
if err != nil || int64(len(body)) < int64(headerSize) {
|
||||
return 0, 0.4
|
||||
}
|
||||
htype, t1 := rar5Vint(body, 0)
|
||||
hflags, t2 := rar5Vint(body, t1)
|
||||
p := t1 + t2
|
||||
var dataSize uint64
|
||||
if hflags&0x0001 != 0 { // extra area present
|
||||
_, te := rar5Vint(body, p)
|
||||
p += te
|
||||
}
|
||||
if hflags&0x0002 != 0 { // data area present
|
||||
dataSize, _ = rar5Vint(body, p)
|
||||
}
|
||||
pos = hdrStart + int64(headerSize) + int64(dataSize)
|
||||
if htype == 5 { // end-of-archive header
|
||||
if pos > start && pos <= max {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
// rar5Vint decodes a RAR5 little-endian base-128 variable integer.
|
||||
func rar5Vint(buf []byte, pos int) (uint64, int) {
|
||||
var v uint64
|
||||
shift := uint(0)
|
||||
for n := 0; pos+n < len(buf) && n < 10; n++ {
|
||||
b := buf[pos+n]
|
||||
v |= uint64(b&0x7f) << shift
|
||||
if b&0x80 == 0 {
|
||||
return v, n + 1
|
||||
}
|
||||
shift += 7
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateRIFF(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 12)
|
||||
if err != nil || len(hdr) < 12 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.LittleEndian.Uint32(hdr[4:8]))
|
||||
return size + 8, 0.9
|
||||
}
|
||||
|
||||
func validateMachO(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 28 {
|
||||
return 0, 0
|
||||
}
|
||||
var (
|
||||
bo binary.ByteOrder
|
||||
is64 bool
|
||||
hdrSize int64 = 28
|
||||
)
|
||||
switch binary.BigEndian.Uint32(hdr[0:4]) {
|
||||
case 0xfeedface:
|
||||
bo = binary.BigEndian
|
||||
case 0xfeedfacf:
|
||||
bo = binary.BigEndian
|
||||
is64 = true
|
||||
hdrSize = 32
|
||||
case 0xcefaedfe:
|
||||
bo = binary.LittleEndian
|
||||
case 0xcffaedfe:
|
||||
bo = binary.LittleEndian
|
||||
is64 = true
|
||||
hdrSize = 32
|
||||
default:
|
||||
return 0, 0
|
||||
}
|
||||
if is64 && len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
ncmds := bo.Uint32(hdr[16:20])
|
||||
sizeofcmds := bo.Uint32(hdr[20:24])
|
||||
if ncmds == 0 || ncmds > 10000 || sizeofcmds > 128<<20 {
|
||||
return 0, 0
|
||||
}
|
||||
cmdBytes, err := d.Range(start+hdrSize, int64(sizeofcmds))
|
||||
if err != nil || len(cmdBytes) < int(sizeofcmds) {
|
||||
return 0, 0
|
||||
}
|
||||
end := hdrSize + int64(sizeofcmds)
|
||||
off := 0
|
||||
for i := uint32(0); i < ncmds; i++ {
|
||||
if off+8 > len(cmdBytes) {
|
||||
return 0, 0
|
||||
}
|
||||
cmd := bo.Uint32(cmdBytes[off : off+4])
|
||||
cmdSize := int(bo.Uint32(cmdBytes[off+4 : off+8]))
|
||||
if cmdSize < 8 || off+cmdSize > len(cmdBytes) {
|
||||
return 0, 0
|
||||
}
|
||||
switch cmd {
|
||||
case 0x1: // LC_SEGMENT
|
||||
if cmdSize >= 56 {
|
||||
fileOff := int64(bo.Uint32(cmdBytes[off+32 : off+36]))
|
||||
fileSize := int64(bo.Uint32(cmdBytes[off+36 : off+40]))
|
||||
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
|
||||
end = fileOff + fileSize
|
||||
}
|
||||
}
|
||||
case 0x19: // LC_SEGMENT_64
|
||||
if cmdSize >= 72 {
|
||||
fileOff := int64(bo.Uint64(cmdBytes[off+40 : off+48]))
|
||||
fileSize := int64(bo.Uint64(cmdBytes[off+48 : off+56]))
|
||||
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
|
||||
end = fileOff + fileSize
|
||||
}
|
||||
}
|
||||
}
|
||||
off += cmdSize
|
||||
}
|
||||
if end <= hdrSize || start+end > max {
|
||||
return 0, 0
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
func validateID3(d *document.Document, start, max int64) (int64, float64) {
|
||||
// ID3v2 header: "ID3" + ver(2) + flags(1) + size(4, synchsafe).
|
||||
hdr, err := d.Range(start, 10)
|
||||
if err != nil || len(hdr) < 10 {
|
||||
return 0, 0.3
|
||||
}
|
||||
if hdr[3] == 0xff || hdr[4] == 0xff {
|
||||
return 0, 0.3
|
||||
}
|
||||
// Synchsafe 28-bit size (7 bits per byte).
|
||||
sz := (int64(hdr[6]&0x7f) << 21) | (int64(hdr[7]&0x7f) << 14) |
|
||||
(int64(hdr[8]&0x7f) << 7) | int64(hdr[9]&0x7f)
|
||||
if sz < 0 || sz > 32*1024*1024 {
|
||||
return 0, 0.4
|
||||
}
|
||||
// Skip the ID3v2 tag (header + synchsafe body, plus a 10-byte footer if the
|
||||
// footer-present flag is set), then walk MPEG audio frames to the end.
|
||||
audioStart := start + 10 + sz
|
||||
if hdr[5]&0x10 != 0 {
|
||||
audioStart += 10
|
||||
}
|
||||
if end := walkMP3Frames(d, audioStart, max); end > audioStart {
|
||||
return end - start, 0.85
|
||||
}
|
||||
return 0, 0.7
|
||||
}
|
||||
|
||||
// walkMP3Frames follows MPEG-1/2/2.5 Layer I/II/III frame headers from start
|
||||
// until a byte that is not a valid frame sync, returning the offset just past
|
||||
// the last frame. APE/Lyrics3 trailers are not stripped — the audio extent is
|
||||
// what matters for carving.
|
||||
func walkMP3Frames(d *document.Document, start, max int64) int64 {
|
||||
pos := start
|
||||
frames := 0
|
||||
for pos+4 <= max && frames < 5_000_000 {
|
||||
h, err := d.Range(pos, 4)
|
||||
if err != nil || len(h) < 4 {
|
||||
break
|
||||
}
|
||||
ln := mp3FrameLen(h)
|
||||
if ln <= 0 {
|
||||
break
|
||||
}
|
||||
pos += ln
|
||||
frames++
|
||||
}
|
||||
if frames == 0 {
|
||||
return 0
|
||||
}
|
||||
if pos > max {
|
||||
return max
|
||||
}
|
||||
return pos
|
||||
}
|
||||
|
||||
var mp3BitrateV1 = [16]int{0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0}
|
||||
var mp3BitrateV2 = [16]int{0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0}
|
||||
var mp3SampleRate = map[int][3]int{ // [version]{rate index 0,1,2}
|
||||
3: {44100, 48000, 32000}, // MPEG1
|
||||
2: {22050, 24000, 16000}, // MPEG2
|
||||
0: {11025, 12000, 8000}, // MPEG2.5
|
||||
}
|
||||
|
||||
// mp3FrameLen returns the byte length of the MPEG audio frame whose 4-byte
|
||||
// header is h, or 0 if h is not a valid frame header.
|
||||
func mp3FrameLen(h []byte) int64 {
|
||||
if h[0] != 0xff || h[1]&0xe0 != 0xe0 {
|
||||
return 0
|
||||
}
|
||||
version := int(h[1] >> 3 & 0x03) // 3=MPEG1, 2=MPEG2, 0=MPEG2.5
|
||||
layer := int(h[1] >> 1 & 0x03) // 3=I, 2=II, 1=III
|
||||
if version == 1 || layer == 0 {
|
||||
return 0 // reserved
|
||||
}
|
||||
brIndex := int(h[2] >> 4 & 0x0f)
|
||||
srIndex := int(h[2] >> 2 & 0x03)
|
||||
padding := int(h[2] >> 1 & 0x01)
|
||||
if brIndex == 0 || brIndex == 15 || srIndex == 3 {
|
||||
return 0
|
||||
}
|
||||
var bitrate int
|
||||
if version == 3 {
|
||||
bitrate = mp3BitrateV1[brIndex]
|
||||
} else {
|
||||
bitrate = mp3BitrateV2[brIndex]
|
||||
}
|
||||
rates, ok := mp3SampleRate[version]
|
||||
if !ok || bitrate == 0 {
|
||||
return 0
|
||||
}
|
||||
sampleRate := rates[srIndex]
|
||||
if sampleRate == 0 {
|
||||
return 0
|
||||
}
|
||||
bps := bitrate * 1000
|
||||
if layer == 3 { // Layer I
|
||||
return int64(((12 * bps / sampleRate) + padding) * 4)
|
||||
}
|
||||
// Layer II & III. MPEG2/2.5 Layer III uses 72 samples-per-byte factor.
|
||||
coef := 144
|
||||
if version != 3 && layer == 1 {
|
||||
coef = 72
|
||||
}
|
||||
return int64(coef*bps/sampleRate + padding)
|
||||
}
|
||||
|
||||
func validateOgg(d *document.Document, start, max int64) (int64, float64) {
|
||||
off := start
|
||||
first := true
|
||||
for off+27 <= max {
|
||||
hdr, err := d.Range(off, 27)
|
||||
if err != nil || len(hdr) < 27 || string(hdr[:4]) != "OggS" || hdr[4] != 0 {
|
||||
return 0, 0
|
||||
}
|
||||
headerType := hdr[5]
|
||||
if first && headerType&0x02 == 0 {
|
||||
return 0, 0.4
|
||||
}
|
||||
segCount := int(hdr[26])
|
||||
lacing, err := d.Range(off+27, int64(segCount))
|
||||
if err != nil || len(lacing) < segCount {
|
||||
return 0, 0
|
||||
}
|
||||
payload := int64(0)
|
||||
for _, v := range lacing {
|
||||
payload += int64(v)
|
||||
}
|
||||
pageLen := int64(27 + segCount)
|
||||
if payload < 0 || off+pageLen+payload > max {
|
||||
return 0, 0
|
||||
}
|
||||
off += pageLen + payload
|
||||
if headerType&0x04 != 0 {
|
||||
return off - start, 0.9
|
||||
}
|
||||
first = false
|
||||
}
|
||||
return 0, 0.6
|
||||
}
|
||||
|
||||
// validateMatroska reads the EBML header element and the following Segment
|
||||
// element's size to bound a Matroska/WebM stream. A Segment with "unknown size"
|
||||
// (all size bits set, as live-streamed files use) cannot be bounded this way.
|
||||
func validateMatroska(d *document.Document, start, max int64) (int64, float64) {
|
||||
buf, err := d.Range(start, min64(max-start, 4096))
|
||||
if err != nil || len(buf) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
// EBML header: ID 0x1A45DFA3, then a vint size.
|
||||
if !(buf[0] == 0x1a && buf[1] == 0x45 && buf[2] == 0xdf && buf[3] == 0xa3) {
|
||||
return 0, 0
|
||||
}
|
||||
hsz, n, unknown := ebmlSize(buf, 4)
|
||||
if n == 0 || unknown {
|
||||
return 0, 0.6
|
||||
}
|
||||
pos := 4 + n + int(hsz) // start of the Segment element
|
||||
if pos+4 > len(buf) {
|
||||
return 0, 0.6
|
||||
}
|
||||
// Segment: ID 0x18538067, then a vint size.
|
||||
if !(buf[pos] == 0x18 && buf[pos+1] == 0x53 && buf[pos+2] == 0x80 && buf[pos+3] == 0x67) {
|
||||
return 0, 0.6
|
||||
}
|
||||
segSz, sn, segUnknown := ebmlSize(buf, pos+4)
|
||||
if sn == 0 || segUnknown {
|
||||
return 0, 0.6 // live/streamed segment without a known size
|
||||
}
|
||||
end := int64(pos+4+sn) + segSz
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.6
|
||||
}
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
// ebmlSize decodes an EBML variable-length integer (data size form). The length
|
||||
// is the number of leading zero bits in the first byte plus one; the value is
|
||||
// the remaining bits. unknown is true when every value bit is set.
|
||||
func ebmlSize(buf []byte, pos int) (val int64, n int, unknown bool) {
|
||||
if pos >= len(buf) || buf[pos] == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
first := buf[pos]
|
||||
mask := byte(0x80)
|
||||
length := 1
|
||||
for first&mask == 0 {
|
||||
mask >>= 1
|
||||
length++
|
||||
}
|
||||
if length > 8 || pos+length > len(buf) {
|
||||
return 0, 0, false
|
||||
}
|
||||
low := mask - 1
|
||||
v := int64(first & low)
|
||||
allOnes := (first & low) == low
|
||||
for i := 1; i < length; i++ {
|
||||
b := buf[pos+i]
|
||||
v = (v << 8) | int64(b)
|
||||
if b != 0xff {
|
||||
allOnes = false
|
||||
}
|
||||
}
|
||||
return v, length, allOnes
|
||||
}
|
||||
|
||||
func validateFLAC(d *document.Document, start, max int64) (int64, float64) {
|
||||
return 0, 0.8
|
||||
}
|
||||
|
||||
// validateBMP reads the BITMAPFILEHEADER size field (bfSize), sanity-checking
|
||||
// the embedded pixel-data offset against the declared length.
|
||||
func validateBMP(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 14)
|
||||
if err != nil || len(hdr) < 14 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.LittleEndian.Uint32(hdr[2:6]))
|
||||
dataOff := int64(binary.LittleEndian.Uint32(hdr[10:14]))
|
||||
if size < 26 || dataOff < 26 || dataOff > size || start+size > max {
|
||||
return 0, 0
|
||||
}
|
||||
return size, 0.8
|
||||
}
|
||||
|
||||
// validateMP4 walks the top-level boxes of an ISO base-media file (MP4/MOV/M4A)
|
||||
// from the file start (the ftyp box). 64-bit box sizes are honoured.
|
||||
func validateMP4(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start
|
||||
boxes := 0
|
||||
for pos+8 <= max && boxes < 100000 {
|
||||
h, err := d.Range(pos, 16)
|
||||
if err != nil || len(h) < 8 {
|
||||
break
|
||||
}
|
||||
size := int64(binary.BigEndian.Uint32(h[0:4]))
|
||||
switch {
|
||||
case size == 1: // 64-bit largesize follows the type
|
||||
if len(h) < 16 {
|
||||
return 0, 0.4
|
||||
}
|
||||
size = int64(binary.BigEndian.Uint64(h[8:16]))
|
||||
case size == 0: // box extends to end of file
|
||||
return max - start, 0.8
|
||||
}
|
||||
if size < 8 {
|
||||
break
|
||||
}
|
||||
pos += size
|
||||
boxes++
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
if pos > start && pos <= max && boxes > 0 {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validateSQLite(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
pageSize := int64(binary.BigEndian.Uint16(hdr[16:18]))
|
||||
if pageSize == 1 {
|
||||
pageSize = 65536
|
||||
}
|
||||
pages := int64(binary.BigEndian.Uint32(hdr[28:32]))
|
||||
if pages == 0 || pageSize == 0 {
|
||||
return 0, 0.5
|
||||
}
|
||||
return pageSize * pages, 0.9
|
||||
}
|
||||
|
||||
func min64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// DiffRegion describes a contiguous run of differing bytes.
|
||||
type DiffRegion struct {
|
||||
Offset int64 `json:"offset"`
|
||||
Length int64 `json:"length"`
|
||||
// kinds: "edit" (both sides have bytes), "add" (only B), "remove" (only A)
|
||||
Kind string `json:"kind"`
|
||||
}
|
||||
|
||||
type DiffBlock struct {
|
||||
Offset int64 `json:"offset"`
|
||||
Length int64 `json:"length"`
|
||||
ChangedBytes int64 `json:"changedBytes"`
|
||||
EditBytes int64 `json:"editBytes"`
|
||||
AddBytes int64 `json:"addBytes"`
|
||||
RemoveBytes int64 `json:"removeBytes"`
|
||||
Similarity float64 `json:"similarity"`
|
||||
}
|
||||
|
||||
// Diff compares two documents by byte. It handles documents of
|
||||
// different sizes by treating the tail of the longer as add/remove.
|
||||
// Chunks at both sides. Returns a compact list of regions.
|
||||
func Diff(a, b *document.Document, start, length int64) (map[string]any, error) {
|
||||
return DiffContext(context.Background(), a, b, start, length)
|
||||
}
|
||||
|
||||
func DiffContext(ctx context.Context, a, b *document.Document, start, length int64) (map[string]any, error) {
|
||||
aSize := a.Size()
|
||||
bSize := b.Size()
|
||||
maxSize := aSize
|
||||
if bSize > maxSize {
|
||||
maxSize = bSize
|
||||
}
|
||||
if length < 0 {
|
||||
length = maxSize - start
|
||||
}
|
||||
end := start + length
|
||||
if end > maxSize {
|
||||
end = maxSize
|
||||
}
|
||||
|
||||
const chunk = 1 << 20
|
||||
regions := make([]DiffRegion, 0, 64)
|
||||
|
||||
var inRun bool
|
||||
var runStart int64
|
||||
var runKind string
|
||||
|
||||
flush := func(off int64) {
|
||||
if inRun {
|
||||
regions = append(regions, DiffRegion{Offset: runStart, Length: off - runStart, Kind: runKind})
|
||||
inRun = false
|
||||
}
|
||||
}
|
||||
|
||||
for off := start; off < end; {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
winEnd := off + chunk
|
||||
if winEnd > end {
|
||||
winEnd = end
|
||||
}
|
||||
// Read both sides, tolerating short reads for the shorter file.
|
||||
var ab, bb []byte
|
||||
if off < aSize {
|
||||
n := winEnd - off
|
||||
if off+n > aSize {
|
||||
n = aSize - off
|
||||
}
|
||||
var err error
|
||||
ab, err = a.Range(off, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if off < bSize {
|
||||
n := winEnd - off
|
||||
if off+n > bSize {
|
||||
n = bSize - off
|
||||
}
|
||||
var err error
|
||||
bb, err = b.Range(off, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for i := int64(0); i < winEnd-off; i++ {
|
||||
if i%4096 == 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
curOff := off + i
|
||||
var av, bv byte
|
||||
var av_ok, bv_ok bool
|
||||
if curOff < aSize && int64(len(ab)) > i {
|
||||
av = ab[i]
|
||||
av_ok = true
|
||||
}
|
||||
if curOff < bSize && int64(len(bb)) > i {
|
||||
bv = bb[i]
|
||||
bv_ok = true
|
||||
}
|
||||
kind := ""
|
||||
if av_ok && bv_ok {
|
||||
if av != bv {
|
||||
kind = "edit"
|
||||
}
|
||||
} else if av_ok && !bv_ok {
|
||||
kind = "remove"
|
||||
} else if !av_ok && bv_ok {
|
||||
kind = "add"
|
||||
}
|
||||
if kind == "" {
|
||||
flush(curOff)
|
||||
continue
|
||||
}
|
||||
if !inRun {
|
||||
inRun = true
|
||||
runStart = curOff
|
||||
runKind = kind
|
||||
} else if kind != runKind {
|
||||
flush(curOff)
|
||||
inRun = true
|
||||
runStart = curOff
|
||||
runKind = kind
|
||||
}
|
||||
}
|
||||
off = winEnd
|
||||
}
|
||||
flush(end)
|
||||
|
||||
// Convert to JS-friendly output + summary
|
||||
out := make([]map[string]any, len(regions))
|
||||
editBytes := int64(0)
|
||||
addBytes := int64(0)
|
||||
removeBytes := int64(0)
|
||||
for i, r := range regions {
|
||||
out[i] = map[string]any{
|
||||
"offset": r.Offset,
|
||||
"length": r.Length,
|
||||
"kind": r.Kind,
|
||||
}
|
||||
switch r.Kind {
|
||||
case "edit":
|
||||
editBytes += r.Length
|
||||
case "add":
|
||||
addBytes += r.Length
|
||||
case "remove":
|
||||
removeBytes += r.Length
|
||||
}
|
||||
}
|
||||
const blockSize int64 = 4096
|
||||
blocks := diffBlocksFromRegions(regions, start, end, blockSize)
|
||||
blockOut := make([]map[string]any, len(blocks))
|
||||
for i, b := range blocks {
|
||||
blockOut[i] = map[string]any{
|
||||
"offset": b.Offset,
|
||||
"length": b.Length,
|
||||
"changedBytes": b.ChangedBytes,
|
||||
"editBytes": b.EditBytes,
|
||||
"addBytes": b.AddBytes,
|
||||
"removeBytes": b.RemoveBytes,
|
||||
"similarity": b.Similarity,
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"regions": out,
|
||||
"regionCount": len(regions),
|
||||
"blocks": blockOut,
|
||||
"blockCount": len(blocks),
|
||||
"blockSize": blockSize,
|
||||
"editBytes": editBytes,
|
||||
"addBytes": addBytes,
|
||||
"removeBytes": removeBytes,
|
||||
"aSize": aSize,
|
||||
"bSize": bSize,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func diffBlocksFromRegions(regions []DiffRegion, start, end, blockSize int64) []DiffBlock {
|
||||
if blockSize <= 0 || end <= start || len(regions) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := []DiffBlock{}
|
||||
for alignedStart := start - start%blockSize; alignedStart < end; alignedStart += blockSize {
|
||||
blockStart := alignedStart
|
||||
blockEnd := alignedStart + blockSize
|
||||
if blockEnd > end {
|
||||
blockEnd = end
|
||||
}
|
||||
if blockEnd <= start {
|
||||
continue
|
||||
}
|
||||
if blockStart < start {
|
||||
blockStart = start
|
||||
}
|
||||
b := DiffBlock{Offset: blockStart, Length: blockEnd - blockStart}
|
||||
for _, r := range regions {
|
||||
rEnd := r.Offset + r.Length
|
||||
if rEnd <= blockStart {
|
||||
continue
|
||||
}
|
||||
if r.Offset >= blockEnd {
|
||||
break
|
||||
}
|
||||
ovStart := r.Offset
|
||||
if ovStart < blockStart {
|
||||
ovStart = blockStart
|
||||
}
|
||||
ovEnd := rEnd
|
||||
if ovEnd > blockEnd {
|
||||
ovEnd = blockEnd
|
||||
}
|
||||
changed := ovEnd - ovStart
|
||||
if changed <= 0 {
|
||||
continue
|
||||
}
|
||||
b.ChangedBytes += changed
|
||||
switch r.Kind {
|
||||
case "edit":
|
||||
b.EditBytes += changed
|
||||
case "add":
|
||||
b.AddBytes += changed
|
||||
case "remove":
|
||||
b.RemoveBytes += changed
|
||||
}
|
||||
}
|
||||
if b.ChangedBytes == 0 {
|
||||
continue
|
||||
}
|
||||
b.Similarity = 1 - float64(b.ChangedBytes)/float64(b.Length)
|
||||
out = append(out, b)
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// Entropy computes Shannon entropy per sliding window over the range.
|
||||
//
|
||||
// The returned map has:
|
||||
//
|
||||
// windowSize: int
|
||||
// step: int
|
||||
// samples: []float64 — entropy per window (0..8)
|
||||
// offsets: []int64 — start offset of each window
|
||||
// mean/min/max: summary stats
|
||||
// highCount: number of windows with entropy >= 7.5 (candidate compressed/encrypted)
|
||||
func Entropy(d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
|
||||
return EntropyContext(context.Background(), d, start, length, windowSize, step)
|
||||
}
|
||||
|
||||
func EntropyContext(ctx context.Context, d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
|
||||
if windowSize <= 0 {
|
||||
windowSize = 4096
|
||||
}
|
||||
if step <= 0 {
|
||||
step = windowSize
|
||||
}
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
if length <= 0 {
|
||||
return map[string]any{
|
||||
"windowSize": windowSize, "step": step,
|
||||
"samples": []float64{}, "offsets": []int64{},
|
||||
"mean": 0.0, "min": 0.0, "max": 0.0, "highCount": 0,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// We run in a streaming fashion: advance through chunks, refill
|
||||
// a rolling window, and emit a sample whenever the cursor hits
|
||||
// a step boundary.
|
||||
samples := make([]float64, 0, length/step+1)
|
||||
offsets := make([]int64, 0, length/step+1)
|
||||
|
||||
// For bounded memory we read the document range chunk by chunk
|
||||
// but we still need a window of size `windowSize`. Windows can be
|
||||
// as small as 256 or as large as 1MB; we cap to length.
|
||||
if windowSize > length {
|
||||
windowSize = length
|
||||
}
|
||||
|
||||
// Simple approach: iterate in step-sized hops, reading window
|
||||
// bytes at each hop. This scales O(n * (w/step)) but window and
|
||||
// step are usually the same for UI.
|
||||
mean := 0.0
|
||||
minE := math.Inf(1)
|
||||
maxE := math.Inf(-1)
|
||||
high := 0
|
||||
|
||||
for off := start; off < start+length; off += step {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
winLen := windowSize
|
||||
if off+winLen > start+length {
|
||||
winLen = start + length - off
|
||||
}
|
||||
buf, err := d.Range(off, winLen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
e := shannon(buf)
|
||||
samples = append(samples, e)
|
||||
offsets = append(offsets, off)
|
||||
mean += e
|
||||
if e < minE {
|
||||
minE = e
|
||||
}
|
||||
if e > maxE {
|
||||
maxE = e
|
||||
}
|
||||
if e >= 7.5 {
|
||||
high++
|
||||
}
|
||||
}
|
||||
if len(samples) > 0 {
|
||||
mean /= float64(len(samples))
|
||||
} else {
|
||||
minE, maxE = 0, 0
|
||||
}
|
||||
|
||||
return map[string]any{
|
||||
"windowSize": windowSize,
|
||||
"step": step,
|
||||
"samples": samples,
|
||||
"offsets": offsets,
|
||||
"mean": mean,
|
||||
"min": minE,
|
||||
"max": maxE,
|
||||
"highCount": high,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func shannon(buf []byte) float64 {
|
||||
if len(buf) == 0 {
|
||||
return 0
|
||||
}
|
||||
var counts [256]int
|
||||
for _, b := range buf {
|
||||
counts[b]++
|
||||
}
|
||||
n := float64(len(buf))
|
||||
e := 0.0
|
||||
for _, c := range counts {
|
||||
if c == 0 {
|
||||
continue
|
||||
}
|
||||
p := float64(c) / n
|
||||
e -= p * math.Log2(p)
|
||||
}
|
||||
return e
|
||||
}
|
||||
@@ -0,0 +1,224 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// fuzzy.go implements context-triggered piecewise hashing (CTPH) compatible
|
||||
// with ssdeep. CTPH produces a short signature of the form
|
||||
// "blocksize:hash1:hash2" that stays similar when the input changes only a
|
||||
// little, so investigators can cluster related samples or match a file
|
||||
// against a known-bad ssdeep hash even after minor edits.
|
||||
//
|
||||
// The signature format and the rolling/sum hashes follow Andrew Tridgell's
|
||||
// spamsum / Jesse Kornblum's ssdeep so the output interoperates with the
|
||||
// standard tooling (e.g. an empty input hashes to "3::").
|
||||
|
||||
const (
|
||||
spamsumLength = 64
|
||||
rollingWindow = 7
|
||||
minBlocksize = 3
|
||||
hashInit = uint32(0x28021967)
|
||||
hashPrime = uint32(0x01000193)
|
||||
)
|
||||
|
||||
const b64alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
|
||||
|
||||
type rollState struct {
|
||||
h1, h2, h3 uint32
|
||||
window [rollingWindow]byte
|
||||
n int
|
||||
}
|
||||
|
||||
func (r *rollState) roll(c byte) uint32 {
|
||||
r.h2 -= r.h1
|
||||
r.h2 += rollingWindow * uint32(c)
|
||||
r.h1 += uint32(c)
|
||||
r.h1 -= uint32(r.window[r.n%rollingWindow])
|
||||
r.window[r.n%rollingWindow] = c
|
||||
r.n++
|
||||
r.h3 = r.h3 << 5
|
||||
r.h3 ^= uint32(c)
|
||||
return r.h1 + r.h2 + r.h3
|
||||
}
|
||||
|
||||
func sumHash(c byte, h uint32) uint32 { return (h * hashPrime) ^ uint32(c) }
|
||||
|
||||
// FuzzyDigest computes the ssdeep-compatible signature for data.
|
||||
func FuzzyDigest(data []byte) string {
|
||||
bs := uint32(minBlocksize)
|
||||
for uint64(bs)*spamsumLength < uint64(len(data)) {
|
||||
bs *= 2
|
||||
}
|
||||
for {
|
||||
sig1, sig2 := ctphSigs(data, bs)
|
||||
if bs > minBlocksize && len(sig1) < spamsumLength/2 {
|
||||
bs /= 2
|
||||
continue
|
||||
}
|
||||
return fmt.Sprintf("%d:%s:%s", bs, sig1, sig2)
|
||||
}
|
||||
}
|
||||
|
||||
func ctphSigs(data []byte, bs uint32) (string, string) {
|
||||
var rs rollState
|
||||
h1, h2 := hashInit, hashInit
|
||||
var sig1, sig2 strings.Builder
|
||||
var lastRoll uint32
|
||||
|
||||
for _, c := range data {
|
||||
h1 = sumHash(c, h1)
|
||||
h2 = sumHash(c, h2)
|
||||
rh := rs.roll(c)
|
||||
lastRoll = rh
|
||||
|
||||
if rh%bs == bs-1 && sig1.Len() < spamsumLength-1 {
|
||||
sig1.WriteByte(b64alphabet[h1%64])
|
||||
h1 = hashInit
|
||||
}
|
||||
if rh%(bs*2) == bs*2-1 && sig2.Len() < spamsumLength/2-1 {
|
||||
sig2.WriteByte(b64alphabet[h2%64])
|
||||
h2 = hashInit
|
||||
}
|
||||
}
|
||||
if lastRoll != 0 {
|
||||
sig1.WriteByte(b64alphabet[h1%64])
|
||||
sig2.WriteByte(b64alphabet[h2%64])
|
||||
}
|
||||
return sig1.String(), sig2.String()
|
||||
}
|
||||
|
||||
// FuzzyHash computes the CTPH signature over a document range.
|
||||
func FuzzyHash(d *document.Document, start, length int64) (map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
buf, err := d.Range(start, length)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sig := FuzzyDigest(buf)
|
||||
bs := strings.SplitN(sig, ":", 2)[0]
|
||||
return map[string]any{
|
||||
"algo": "ssdeep",
|
||||
"signature": sig,
|
||||
"blocksize": bs,
|
||||
"length": length,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// FuzzyCompare returns a 0-100 similarity score between two ssdeep
|
||||
// signatures. 0 means unrelated (or incomparable block sizes); 100 means the
|
||||
// signatures are identical.
|
||||
func FuzzyCompare(a, b string) int {
|
||||
bs1, a1, a2, ok1 := parseSig(a)
|
||||
bs2, b1, b2, ok2 := parseSig(b)
|
||||
if !ok1 || !ok2 {
|
||||
return 0
|
||||
}
|
||||
if bs1 != bs2 && bs1 != bs2*2 && bs2 != bs1*2 {
|
||||
return 0
|
||||
}
|
||||
switch {
|
||||
case bs1 == bs2:
|
||||
s1 := scoreStrings(a1, b1, bs1)
|
||||
s2 := scoreStrings(a2, b2, bs1*2)
|
||||
if s2 > s1 {
|
||||
return s2
|
||||
}
|
||||
return s1
|
||||
case bs1 == bs2*2:
|
||||
return scoreStrings(a1, b2, bs1)
|
||||
default: // bs2 == bs1*2
|
||||
return scoreStrings(a2, b1, bs2)
|
||||
}
|
||||
}
|
||||
|
||||
func parseSig(s string) (uint32, string, string, bool) {
|
||||
parts := strings.SplitN(s, ":", 3)
|
||||
if len(parts) != 3 {
|
||||
return 0, "", "", false
|
||||
}
|
||||
bs, err := strconv.ParseUint(parts[0], 10, 32)
|
||||
if err != nil || bs == 0 {
|
||||
return 0, "", "", false
|
||||
}
|
||||
return uint32(bs), parts[1], parts[2], true
|
||||
}
|
||||
|
||||
func scoreStrings(s1, s2 string, blocksize uint32) int {
|
||||
if !hasCommonSubstring(s1, s2, rollingWindow) {
|
||||
return 0
|
||||
}
|
||||
if s1 == s2 {
|
||||
return 100
|
||||
}
|
||||
d := editDistance(s1, s2)
|
||||
if len(s1)+len(s2) == 0 {
|
||||
return 0
|
||||
}
|
||||
score := (d * spamsumLength) / (len(s1) + len(s2))
|
||||
score = (100 * score) / spamsumLength
|
||||
score = 100 - score
|
||||
// Small block sizes describe little data, so cap the score they can claim.
|
||||
cap := int(blocksize) / minBlocksize * minInt(len(s1), len(s2))
|
||||
if score > cap {
|
||||
score = cap
|
||||
}
|
||||
if score > 100 {
|
||||
score = 100
|
||||
}
|
||||
if score < 0 {
|
||||
score = 0
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func hasCommonSubstring(s1, s2 string, n int) bool {
|
||||
if len(s1) < n || len(s2) < n {
|
||||
return false
|
||||
}
|
||||
seen := make(map[string]struct{}, len(s1))
|
||||
for i := 0; i+n <= len(s1); i++ {
|
||||
seen[s1[i:i+n]] = struct{}{}
|
||||
}
|
||||
for i := 0; i+n <= len(s2); i++ {
|
||||
if _, ok := seen[s2[i:i+n]]; ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// editDistance is classic Levenshtein with unit insert/delete/substitute cost.
|
||||
func editDistance(s1, s2 string) int {
|
||||
m, n := len(s1), len(s2)
|
||||
prev := make([]int, n+1)
|
||||
cur := make([]int, n+1)
|
||||
for j := 0; j <= n; j++ {
|
||||
prev[j] = j
|
||||
}
|
||||
for i := 1; i <= m; i++ {
|
||||
cur[0] = i
|
||||
for j := 1; j <= n; j++ {
|
||||
cost := 1
|
||||
if s1[i-1] == s2[j-1] {
|
||||
cost = 0
|
||||
}
|
||||
cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost)
|
||||
}
|
||||
prev, cur = cur, prev
|
||||
}
|
||||
return prev[n]
|
||||
}
|
||||
|
||||
func minInt(a, b int) int {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/hex"
|
||||
"hash"
|
||||
"hash/crc32"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// HashSet contains CRC32/MD5/SHA-1/SHA-256/SHA-512 for a range.
|
||||
type HashSet struct {
|
||||
Length int64 `json:"length"`
|
||||
CRC32 string `json:"crc32"`
|
||||
MD5 string `json:"md5"`
|
||||
SHA1 string `json:"sha1"`
|
||||
SHA256 string `json:"sha256"`
|
||||
SHA512 string `json:"sha512"`
|
||||
}
|
||||
|
||||
// Hashes computes all supported hashes over the document range.
|
||||
// The computation runs chunk-by-chunk so large files do not force a
|
||||
// full-buffer duplication.
|
||||
func Hashes(d *document.Document, start, length int64) (map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
|
||||
crc := crc32.NewIEEE()
|
||||
mh := md5.New()
|
||||
s1 := sha1.New()
|
||||
s256 := sha256.New()
|
||||
s512 := sha512.New()
|
||||
|
||||
writers := []hash.Hash{mh, s1, s256, s512}
|
||||
|
||||
total := int64(0)
|
||||
err := d.ForEachChunk(start, length, func(_ int64, chunk []byte) error {
|
||||
crc.Write(chunk)
|
||||
for _, w := range writers {
|
||||
w.Write(chunk)
|
||||
}
|
||||
total += int64(len(chunk))
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"length": total,
|
||||
"crc32": toHex(crc.Sum(nil)),
|
||||
"md5": toHex(mh.Sum(nil)),
|
||||
"sha1": toHex(s1.Sum(nil)),
|
||||
"sha256": toHex(s256.Sum(nil)),
|
||||
"sha512": toHex(s512.Sum(nil)),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func toHex(b []byte) string { return hex.EncodeToString(b) }
|
||||
@@ -0,0 +1,162 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"math"
|
||||
"time"
|
||||
"unicode/utf16"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// Inspect interprets bytes starting at `buf[0]` as many candidate
|
||||
// types. It is resilient: types that don't have enough bytes simply
|
||||
// return empty strings.
|
||||
func Inspect(buf []byte, offset int64) map[string]any {
|
||||
out := map[string]any{
|
||||
"offset": offset,
|
||||
"length": len(buf),
|
||||
}
|
||||
|
||||
// 8-bit
|
||||
if len(buf) >= 1 {
|
||||
out["u8"] = int(buf[0])
|
||||
out["i8"] = int(int8(buf[0]))
|
||||
out["char"] = printable(buf[0])
|
||||
out["bin8"] = fmt.Sprintf("%08b", buf[0])
|
||||
}
|
||||
|
||||
// 16-bit
|
||||
if len(buf) >= 2 {
|
||||
le16 := binary.LittleEndian.Uint16(buf)
|
||||
be16 := binary.BigEndian.Uint16(buf)
|
||||
out["u16_le"] = int(le16)
|
||||
out["u16_be"] = int(be16)
|
||||
out["i16_le"] = int(int16(le16))
|
||||
out["i16_be"] = int(int16(be16))
|
||||
}
|
||||
|
||||
// 32-bit
|
||||
if len(buf) >= 4 {
|
||||
le32 := binary.LittleEndian.Uint32(buf)
|
||||
be32 := binary.BigEndian.Uint32(buf)
|
||||
out["u32_le"] = uint(le32)
|
||||
out["u32_be"] = uint(be32)
|
||||
out["i32_le"] = int(int32(le32))
|
||||
out["i32_be"] = int(int32(be32))
|
||||
out["f32_le"] = float64(math.Float32frombits(le32))
|
||||
out["f32_be"] = float64(math.Float32frombits(be32))
|
||||
}
|
||||
|
||||
// 64-bit
|
||||
if len(buf) >= 8 {
|
||||
le64 := binary.LittleEndian.Uint64(buf)
|
||||
be64 := binary.BigEndian.Uint64(buf)
|
||||
out["u64_le"] = float64(le64)
|
||||
out["u64_be"] = float64(be64)
|
||||
out["i64_le"] = float64(int64(le64))
|
||||
out["i64_be"] = float64(int64(be64))
|
||||
out["f64_le"] = math.Float64frombits(le64)
|
||||
out["f64_be"] = math.Float64frombits(be64)
|
||||
}
|
||||
|
||||
// Timestamp guesses
|
||||
if len(buf) >= 4 {
|
||||
le := binary.LittleEndian.Uint32(buf)
|
||||
out["unix_le"] = formatUnix(int64(le))
|
||||
}
|
||||
if len(buf) >= 8 {
|
||||
le := binary.LittleEndian.Uint64(buf)
|
||||
out["filetime_le"] = formatFiletime(le)
|
||||
out["webkit_le"] = formatWebkit(le)
|
||||
out["oletime_le"] = formatOLEDate(math.Float64frombits(le))
|
||||
}
|
||||
|
||||
// ASCII preview (up to 32 bytes)
|
||||
n := len(buf)
|
||||
if n > 32 {
|
||||
n = 32
|
||||
}
|
||||
ascii := make([]byte, n)
|
||||
for i := 0; i < n; i++ {
|
||||
if buf[i] >= 0x20 && buf[i] <= 0x7e {
|
||||
ascii[i] = buf[i]
|
||||
} else {
|
||||
ascii[i] = '.'
|
||||
}
|
||||
}
|
||||
out["ascii"] = string(ascii)
|
||||
|
||||
// UTF-8 attempt
|
||||
if utf8.Valid(buf[:n]) {
|
||||
out["utf8"] = string(buf[:n])
|
||||
}
|
||||
|
||||
// UTF-16 LE/BE preview
|
||||
if len(buf) >= 2 {
|
||||
lu := n / 2
|
||||
if lu > 0 {
|
||||
u := make([]uint16, lu)
|
||||
for i := 0; i < lu; i++ {
|
||||
u[i] = uint16(buf[i*2]) | uint16(buf[i*2+1])<<8
|
||||
}
|
||||
out["utf16le"] = string(utf16.Decode(u))
|
||||
for i := 0; i < lu; i++ {
|
||||
u[i] = uint16(buf[i*2])<<8 | uint16(buf[i*2+1])
|
||||
}
|
||||
out["utf16be"] = string(utf16.Decode(u))
|
||||
}
|
||||
}
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
func printable(b byte) string {
|
||||
if b >= 0x20 && b <= 0x7e {
|
||||
return string([]byte{b})
|
||||
}
|
||||
return "."
|
||||
}
|
||||
|
||||
func formatUnix(sec int64) string {
|
||||
if sec < 0 || sec > 4102444800 { // past 2100
|
||||
return ""
|
||||
}
|
||||
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// Windows FILETIME epoch 1601-01-01 in 100ns ticks.
|
||||
const filetimeEpochDelta = 11644473600
|
||||
|
||||
func formatFiletime(v uint64) string {
|
||||
if v == 0 {
|
||||
return ""
|
||||
}
|
||||
sec := int64(v/10_000_000) - filetimeEpochDelta
|
||||
if sec < 0 || sec > 4102444800 {
|
||||
return ""
|
||||
}
|
||||
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// WebKit/Chrome epoch 1601-01-01 in microseconds.
|
||||
func formatWebkit(v uint64) string {
|
||||
if v == 0 {
|
||||
return ""
|
||||
}
|
||||
sec := int64(v/1_000_000) - filetimeEpochDelta
|
||||
if sec < 0 || sec > 4102444800 {
|
||||
return ""
|
||||
}
|
||||
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
|
||||
}
|
||||
|
||||
// OLE Automation date: days since 1899-12-30, as float64.
|
||||
func formatOLEDate(days float64) string {
|
||||
if days < 2 || days > 75000 {
|
||||
return ""
|
||||
}
|
||||
base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
|
||||
sec := int64(days * 86400)
|
||||
return base.Add(time.Duration(sec) * time.Second).Format(time.RFC3339)
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// ioc regexes. They run against printable ASCII strings extracted
|
||||
// from the document, which is the standard approach for IOC hunting
|
||||
// and avoids matching noise in raw binary data.
|
||||
var iocRegexes = map[string]*regexp.Regexp{
|
||||
"url": regexp.MustCompile(`(?i)\b(?:https?|ftp)://[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`),
|
||||
"domain": regexp.MustCompile(`(?i)\b(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.)+(?:com|net|org|io|co|uk|de|fr|ru|cn|info|xyz|app|dev|sh|me|gov|mil|edu|biz)\b`),
|
||||
"email": regexp.MustCompile(`(?i)\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b`),
|
||||
"ipv4": regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`),
|
||||
"ipv6": regexp.MustCompile(`\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b`),
|
||||
"guid": regexp.MustCompile(`\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`),
|
||||
"md5": regexp.MustCompile(`\b[a-fA-F0-9]{32}\b`),
|
||||
"sha1": regexp.MustCompile(`\b[a-fA-F0-9]{40}\b`),
|
||||
"sha256": regexp.MustCompile(`\b[a-fA-F0-9]{64}\b`),
|
||||
"winpath": regexp.MustCompile(`[A-Za-z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*`),
|
||||
"unc": regexp.MustCompile(`\\\\[A-Za-z0-9._\-]+(?:\\[^\\/:*?"<>|\r\n]+)+`),
|
||||
"unixpath": regexp.MustCompile(`(?:/(?:etc|var|usr|opt|home|tmp|root|srv|bin|sbin|lib|lib64)(?:/[A-Za-z0-9._\-]+)+)`),
|
||||
"regkey": regexp.MustCompile(`(?i)\b(?:HKLM|HKCU|HKCR|HKU|HKCC|HKEY_[A-Z_]+)\\[^\s"<>]+`),
|
||||
"onion": regexp.MustCompile(`(?i)\b[a-z2-7]{16,56}\.onion\b`),
|
||||
"btc": regexp.MustCompile(`\b(?:[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-z0-9]{20,90})\b`),
|
||||
}
|
||||
|
||||
// ExtractIOCs runs the IOC regexes over ASCII strings first and then
|
||||
// UTF-16 LE/BE strings. Returns a map of category -> slice of hits
|
||||
// with offsets into the original file.
|
||||
func ExtractIOCs(d *document.Document, start, length int64, minLen int) (map[string]any, error) {
|
||||
return ExtractIOCsContext(context.Background(), d, start, length, minLen)
|
||||
}
|
||||
|
||||
func ExtractIOCsContext(ctx context.Context, d *document.Document, start, length int64, minLen int) (map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
if minLen <= 0 {
|
||||
minLen = 5
|
||||
}
|
||||
|
||||
strings, err := ExtractStringsContext(ctx, d, start, length, minLen, []string{"ascii", "utf16le", "utf16be"}, 200_000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result := map[string]any{}
|
||||
totals := map[string]int{}
|
||||
for category, re := range iocRegexes {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits := make([]map[string]any, 0)
|
||||
for _, s := range strings {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
str, _ := s["value"].(string)
|
||||
base, _ := s["offset"].(int64)
|
||||
for _, loc := range re.FindAllStringIndex(str, -1) {
|
||||
hits = append(hits, map[string]any{
|
||||
"category": category,
|
||||
"offset": base + int64(loc[0]),
|
||||
"length": loc[1] - loc[0],
|
||||
"value": str[loc[0]:loc[1]],
|
||||
})
|
||||
}
|
||||
}
|
||||
result[category] = hits
|
||||
totals[category] = len(hits)
|
||||
}
|
||||
// flat list for UI convenience
|
||||
all := []map[string]any{}
|
||||
for _, v := range result {
|
||||
if s, ok := v.([]map[string]any); ok {
|
||||
all = append(all, s...)
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"byCategory": result,
|
||||
"all": all,
|
||||
"totals": totals,
|
||||
"totalStrings": len(strings),
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,367 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/ecdsa"
|
||||
"crypto/ed25519"
|
||||
"crypto/rsa"
|
||||
"crypto/x509"
|
||||
"encoding/pem"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// KeyArtefact is a single key-ish object we've found in the raw file.
|
||||
// Length is the span we recognised; Note carries any decoding detail.
|
||||
type KeyArtefact struct {
|
||||
Kind string `json:"kind"`
|
||||
Format string `json:"format"` // "pem" | "der" | "ssh" | "p12"
|
||||
Offset int64 `json:"offset"`
|
||||
Length int64 `json:"length"`
|
||||
Summary string `json:"summary"`
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// ScanKeys walks the whole document looking for cryptographic artefacts:
|
||||
// - PEM blocks with common key-related labels
|
||||
// - OpenSSH keys (authorized_keys lines and the "openssh-key-v1" private-key envelope)
|
||||
// - PKCS#12 magic (1.2.840.113549.1.12.1 appears in headers)
|
||||
// - Loose DER SEQUENCEs that parse as RSA/EC/PKCS#8 keys
|
||||
//
|
||||
// We cap the scan at 64 MiB. Users with larger blobs can select a range
|
||||
// and call this against that subset via a future "keys.scanRange" bridge.
|
||||
func ScanKeys(d *document.Document) ([]KeyArtefact, error) {
|
||||
size := d.Size()
|
||||
if size > 64*1024*1024 {
|
||||
return nil, fmt.Errorf("keys.scan: file > 64 MiB; use selection")
|
||||
}
|
||||
buf, err := d.Range(0, size)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []KeyArtefact{}
|
||||
out = append(out, scanPEMKeys(buf)...)
|
||||
out = append(out, scanOpenSSHPubKeys(buf)...)
|
||||
out = append(out, scanOpenSSHPrivKeys(buf)...)
|
||||
out = append(out, scanPKCS12(buf)...)
|
||||
// Loose DER pass — for raw PKCS#8 / PKCS#1 blobs that aren't wrapped in PEM.
|
||||
out = append(out, scanLooseDERKeys(buf)...)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Known PEM labels that indicate key-ish content.
|
||||
var pemKeyLabels = map[string]string{
|
||||
"RSA PRIVATE KEY": "rsa-private",
|
||||
"RSA PUBLIC KEY": "rsa-public",
|
||||
"EC PRIVATE KEY": "ec-private",
|
||||
"EC PUBLIC KEY": "ec-public",
|
||||
"DSA PRIVATE KEY": "dsa-private",
|
||||
"PRIVATE KEY": "pkcs8-private",
|
||||
"ENCRYPTED PRIVATE KEY": "pkcs8-encrypted",
|
||||
"PUBLIC KEY": "pkix-public",
|
||||
"OPENSSH PRIVATE KEY": "openssh-private",
|
||||
"SSH2 PUBLIC KEY": "ssh2-public",
|
||||
"PGP PRIVATE KEY BLOCK": "pgp-private",
|
||||
"PGP PUBLIC KEY BLOCK": "pgp-public",
|
||||
"CERTIFICATE REQUEST": "csr",
|
||||
"NEW CERTIFICATE REQUEST": "csr",
|
||||
"DH PARAMETERS": "dh-params",
|
||||
"PARAMETERS": "params",
|
||||
}
|
||||
|
||||
func scanPEMKeys(buf []byte) []KeyArtefact {
|
||||
out := []KeyArtefact{}
|
||||
rest := buf
|
||||
origLen := int64(len(buf))
|
||||
for {
|
||||
block, next := pem.Decode(rest)
|
||||
if block == nil {
|
||||
break
|
||||
}
|
||||
beginIdx := bytes.Index(rest, []byte("-----BEGIN"))
|
||||
if beginIdx < 0 {
|
||||
break
|
||||
}
|
||||
start := origLen - int64(len(rest)) + int64(beginIdx)
|
||||
end := origLen - int64(len(next))
|
||||
kind, match := pemKeyLabels[block.Type]
|
||||
if !match {
|
||||
rest = next
|
||||
continue
|
||||
}
|
||||
art := KeyArtefact{
|
||||
Kind: kind, Format: "pem",
|
||||
Offset: start, Length: end - start,
|
||||
}
|
||||
art.Summary = describePEMBlock(block)
|
||||
out = append(out, art)
|
||||
rest = next
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func describePEMBlock(b *pem.Block) string {
|
||||
name := b.Type
|
||||
switch b.Type {
|
||||
case "RSA PRIVATE KEY":
|
||||
if k, err := x509.ParsePKCS1PrivateKey(b.Bytes); err == nil {
|
||||
return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen())
|
||||
}
|
||||
case "RSA PUBLIC KEY":
|
||||
if k, err := x509.ParsePKCS1PublicKey(b.Bytes); err == nil {
|
||||
return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen())
|
||||
}
|
||||
case "EC PRIVATE KEY":
|
||||
if k, err := x509.ParseECPrivateKey(b.Bytes); err == nil {
|
||||
return fmt.Sprintf("%s — curve %s", name, k.Curve.Params().Name)
|
||||
}
|
||||
case "PRIVATE KEY":
|
||||
if k, err := x509.ParsePKCS8PrivateKey(b.Bytes); err == nil {
|
||||
return fmt.Sprintf("%s — %s", name, describePrivateKey(k))
|
||||
}
|
||||
case "PUBLIC KEY":
|
||||
if k, err := x509.ParsePKIXPublicKey(b.Bytes); err == nil {
|
||||
return fmt.Sprintf("%s — %s", name, describePublicKey(k))
|
||||
}
|
||||
case "OPENSSH PRIVATE KEY":
|
||||
return fmt.Sprintf("%s (%d B payload)", name, len(b.Bytes))
|
||||
case "ENCRYPTED PRIVATE KEY":
|
||||
return fmt.Sprintf("%s — PKCS#8 encrypted (%d B)", name, len(b.Bytes))
|
||||
}
|
||||
return fmt.Sprintf("%s (%d B)", name, len(b.Bytes))
|
||||
}
|
||||
|
||||
func describePrivateKey(k any) string {
|
||||
switch key := k.(type) {
|
||||
case *rsa.PrivateKey:
|
||||
return fmt.Sprintf("RSA %d-bit", key.N.BitLen())
|
||||
case *ecdsa.PrivateKey:
|
||||
return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name)
|
||||
case ed25519.PrivateKey:
|
||||
return "Ed25519"
|
||||
}
|
||||
return fmt.Sprintf("%T", k)
|
||||
}
|
||||
|
||||
func describePublicKey(k any) string {
|
||||
switch key := k.(type) {
|
||||
case *rsa.PublicKey:
|
||||
return fmt.Sprintf("RSA %d-bit", key.N.BitLen())
|
||||
case *ecdsa.PublicKey:
|
||||
return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name)
|
||||
case ed25519.PublicKey:
|
||||
return "Ed25519"
|
||||
}
|
||||
return fmt.Sprintf("%T", k)
|
||||
}
|
||||
|
||||
// OpenSSH authorized_keys / id_*.pub format is one line per key:
|
||||
// "<algo> <base64> [comment]\n". We search for well-known algos and
|
||||
// delimit by the next newline. False positives on text logs are possible,
|
||||
// so we validate by base64-decoding.
|
||||
var sshAlgos = []string{
|
||||
"ssh-rsa", "ssh-dss", "ssh-ed25519",
|
||||
"ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521",
|
||||
"sk-ssh-ed25519@openssh.com", "sk-ecdsa-sha2-nistp256@openssh.com",
|
||||
}
|
||||
|
||||
func scanOpenSSHPubKeys(buf []byte) []KeyArtefact {
|
||||
out := []KeyArtefact{}
|
||||
for _, algo := range sshAlgos {
|
||||
needle := []byte(algo + " ")
|
||||
off := 0
|
||||
for {
|
||||
idx := bytes.Index(buf[off:], needle)
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
absStart := off + idx
|
||||
// Must be at start of line or after whitespace.
|
||||
if absStart > 0 {
|
||||
prev := buf[absStart-1]
|
||||
if prev != '\n' && prev != '\r' && prev != ' ' && prev != '\t' {
|
||||
off = absStart + 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
// End of line.
|
||||
endOfLine := absStart + len(needle)
|
||||
for endOfLine < len(buf) && buf[endOfLine] != '\n' && buf[endOfLine] != '\r' {
|
||||
endOfLine++
|
||||
}
|
||||
line := buf[absStart:endOfLine]
|
||||
// Expect at least one space-separated payload that decodes.
|
||||
parts := bytes.SplitN(line, []byte(" "), 3)
|
||||
if len(parts) < 2 || len(parts[1]) < 16 || !isBase64Like(parts[1]) {
|
||||
off = absStart + 1
|
||||
continue
|
||||
}
|
||||
art := KeyArtefact{
|
||||
Kind: "openssh-public", Format: "ssh",
|
||||
Offset: int64(absStart), Length: int64(len(line)),
|
||||
Summary: fmt.Sprintf("%s (%d-byte payload b64)", algo, len(parts[1])),
|
||||
}
|
||||
if len(parts) == 3 && len(parts[2]) > 0 {
|
||||
art.Note = "comment: " + string(parts[2])
|
||||
}
|
||||
out = append(out, art)
|
||||
off = endOfLine
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func scanOpenSSHPrivKeys(buf []byte) []KeyArtefact {
|
||||
// Already caught by PEM scan (label "OPENSSH PRIVATE KEY"). We also
|
||||
// look for the magic string inside the PEM body — "openssh-key-v1\0".
|
||||
out := []KeyArtefact{}
|
||||
needle := []byte("openssh-key-v1\x00")
|
||||
off := 0
|
||||
for {
|
||||
idx := bytes.Index(buf[off:], needle)
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
absStart := off + idx
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "openssh-private-magic", Format: "ssh",
|
||||
Offset: int64(absStart), Length: int64(len(needle)),
|
||||
Summary: "OpenSSH v1 private-key magic",
|
||||
Note: "magic only; no envelope length inferred",
|
||||
})
|
||||
off = absStart + len(needle)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// PKCS#12 (.p12 / .pfx) files begin with an ASN.1 SEQUENCE; the second
|
||||
// element is an OID for the bag type. Its encoded form is reasonably
|
||||
// distinctive: "\x06\x0b\x2a\x86\x48\x86\xf7\x0d\x01\x07\x01" (data).
|
||||
// The simplest reliable magic for our purposes is the OID for
|
||||
// pkcs-12 (1.2.840.113549.1.12.1): "\x06\x0a\x2a\x86\x48\x86\xf7\x0d\x01\x0c\x01".
|
||||
func scanPKCS12(buf []byte) []KeyArtefact {
|
||||
out := []KeyArtefact{}
|
||||
needle := []byte{0x06, 0x0a, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x0c, 0x01}
|
||||
off := 0
|
||||
for {
|
||||
idx := bytes.Index(buf[off:], needle)
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
absStart := off + idx
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "pkcs12-oid", Format: "p12",
|
||||
Offset: int64(absStart), Length: int64(len(needle)),
|
||||
Summary: "pkcs-12 OID (1.2.840.113549.1.12.1)",
|
||||
Note: "bag-type marker; may indicate a .p12 or .pfx container",
|
||||
})
|
||||
off = absStart + len(needle)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// scanLooseDERKeys walks DER SEQUENCE headers and tries to parse each
|
||||
// candidate as a known key format. False negatives are fine; we only
|
||||
// emit an artefact on a positive parse.
|
||||
func scanLooseDERKeys(buf []byte) []KeyArtefact {
|
||||
out := []KeyArtefact{}
|
||||
// Use the same SEQUENCE 0x30 0x82 xx yy (long-form, 2 length bytes)
|
||||
// pattern as the cert scanner. Minimum useful key length ~200 bytes.
|
||||
//
|
||||
// Each candidate runs up to six x509 parse attempts over a blob up to
|
||||
// 64 KiB, so an unfiltered scan of a buffer dense with "30 82" pairs is
|
||||
// O(n) expensive parses and could freeze the UI. A cheap structural
|
||||
// pre-filter (a DER key body begins with a SEQUENCE 0x30 or an INTEGER
|
||||
// 0x02) rejects almost all false candidates without parsing, and a hard
|
||||
// cap bounds adversarial input that defeats the pre-filter.
|
||||
const maxDERAttempts = 1 << 16
|
||||
derAttempts := 0
|
||||
for i := 0; i+4 < len(buf); i++ {
|
||||
if buf[i] != 0x30 || buf[i+1] != 0x82 {
|
||||
continue
|
||||
}
|
||||
if c := buf[i+4]; c != 0x30 && c != 0x02 {
|
||||
continue
|
||||
}
|
||||
total := int(buf[i+2])<<8 | int(buf[i+3])
|
||||
if total < 80 || i+4+total > len(buf) {
|
||||
continue
|
||||
}
|
||||
if derAttempts++; derAttempts > maxDERAttempts {
|
||||
break
|
||||
}
|
||||
blob := buf[i : i+4+total]
|
||||
// Skip anything that looks like a certificate (already caught by certs panel).
|
||||
if _, err := x509.ParseCertificate(blob); err == nil {
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
if _, err := x509.ParsePKCS1PrivateKey(blob); err == nil {
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "rsa-private", Format: "der",
|
||||
Offset: int64(i), Length: int64(4 + total),
|
||||
Summary: "RSA private key (PKCS#1 DER)",
|
||||
})
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
if _, err := x509.ParsePKCS1PublicKey(blob); err == nil {
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "rsa-public", Format: "der",
|
||||
Offset: int64(i), Length: int64(4 + total),
|
||||
Summary: "RSA public key (PKCS#1 DER)",
|
||||
})
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
if k, err := x509.ParseECPrivateKey(blob); err == nil {
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "ec-private", Format: "der",
|
||||
Offset: int64(i), Length: int64(4 + total),
|
||||
Summary: "EC private key (SEC1 DER) curve=" + k.Curve.Params().Name,
|
||||
})
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
if k, err := x509.ParsePKCS8PrivateKey(blob); err == nil {
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "pkcs8-private", Format: "der",
|
||||
Offset: int64(i), Length: int64(4 + total),
|
||||
Summary: "PKCS#8 private key — " + describePrivateKey(k),
|
||||
})
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
if k, err := x509.ParsePKIXPublicKey(blob); err == nil {
|
||||
out = append(out, KeyArtefact{
|
||||
Kind: "pkix-public", Format: "der",
|
||||
Offset: int64(i), Length: int64(4 + total),
|
||||
Summary: "PKIX public key — " + describePublicKey(k),
|
||||
})
|
||||
i += 4 + total - 1
|
||||
continue
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isBase64Like(b []byte) bool {
|
||||
if len(b) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, c := range b {
|
||||
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
|
||||
c == '+' || c == '/' || c == '=' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Stringer helper used by the bridge; avoids leaking internal types to JS.
|
||||
func (a KeyArtefact) String() string {
|
||||
return strings.TrimSpace(fmt.Sprintf("%s %s @0x%x %s", a.Kind, a.Format, a.Offset, a.Summary))
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
type MultiSearchQuery struct {
|
||||
Mode string
|
||||
Query string
|
||||
}
|
||||
|
||||
func ParseMultiSearchSpec(spec string) ([]MultiSearchQuery, error) {
|
||||
spec = strings.ReplaceAll(spec, ";", "\n")
|
||||
out := []MultiSearchQuery{}
|
||||
for _, line := range strings.Split(spec, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
parts := strings.Fields(line)
|
||||
if len(parts) < 2 {
|
||||
return nil, errors.New("multi-search lines require: text|hex|regex <query>")
|
||||
}
|
||||
mode := strings.ToLower(parts[0])
|
||||
query := strings.TrimSpace(strings.TrimPrefix(line, parts[0]))
|
||||
switch mode {
|
||||
case "text", "hex", "regex":
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown multi-search mode %q", mode)
|
||||
}
|
||||
if query == "" {
|
||||
return nil, errors.New("empty multi-search query")
|
||||
}
|
||||
out = append(out, MultiSearchQuery{Mode: mode, Query: query})
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, errors.New("no multi-search queries")
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func MultiSearch(d *document.Document, start, length int64, spec string, maxPerQuery int) (map[string]any, error) {
|
||||
queries, err := ParseMultiSearchSpec(spec)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if maxPerQuery <= 0 {
|
||||
maxPerQuery = 1000
|
||||
}
|
||||
results := make([]map[string]any, 0, len(queries))
|
||||
all := []map[string]any{}
|
||||
for i, q := range queries {
|
||||
var hits []SearchHit
|
||||
switch q.Mode {
|
||||
case "text":
|
||||
hits, err = SearchBytes(d, start, length, []byte(q.Query), maxPerQuery)
|
||||
case "hex":
|
||||
var needle []byte
|
||||
needle, err = parseMultiSearchHex(q.Query)
|
||||
if err == nil {
|
||||
hits, err = SearchBytes(d, start, length, needle, maxPerQuery)
|
||||
}
|
||||
case "regex":
|
||||
if _, err = regexp.Compile(q.Query); err == nil {
|
||||
hits, err = SearchRegex(d, start, length, q.Query, maxPerQuery)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query %d: %w", i+1, err)
|
||||
}
|
||||
maps := HitsToMaps(hits)
|
||||
name := fmt.Sprintf("%s:%s", q.Mode, q.Query)
|
||||
for _, h := range maps {
|
||||
h["mode"] = q.Mode
|
||||
h["query"] = q.Query
|
||||
h["name"] = name
|
||||
all = append(all, h)
|
||||
}
|
||||
results = append(results, map[string]any{
|
||||
"mode": q.Mode,
|
||||
"query": q.Query,
|
||||
"count": len(hits),
|
||||
"hits": maps,
|
||||
})
|
||||
}
|
||||
return map[string]any{
|
||||
"queries": results,
|
||||
"all": all,
|
||||
"total": len(all),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseMultiSearchHex(s string) ([]byte, error) {
|
||||
s = strings.Map(func(r rune) rune {
|
||||
switch r {
|
||||
case ' ', '\t', '\n', '\r', ':', '-':
|
||||
return -1
|
||||
default:
|
||||
return r
|
||||
}
|
||||
}, strings.TrimSpace(s))
|
||||
s = strings.TrimPrefix(strings.ToLower(s), "0x")
|
||||
if len(s)%2 != 0 {
|
||||
s = "0" + s
|
||||
}
|
||||
return hex.DecodeString(s)
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// Overview returns a compact first-pass report intended for both UI
|
||||
// surfaces. It favours fast, bounded summaries over exhaustive output.
|
||||
func Overview(d *document.Document) (map[string]any, error) {
|
||||
size := d.Size()
|
||||
hashes, err := Hashes(d, 0, -1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entropy, err := Entropy(d, 0, min64(size, 8<<20), 4096, 0)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stringsOut, err := ExtractStrings(d, 0, min64(size, 16<<20), 4, []string{"ascii", "utf16le", "utf16be"}, 5000)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
iocs, err := ExtractIOCs(d, 0, min64(size, 16<<20), 5)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
carves, err := Carve(d, 0, min64(size, 64<<20), 128)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return map[string]any{
|
||||
"name": d.Name(),
|
||||
"size": size,
|
||||
"hashes": hashes,
|
||||
"entropy": overviewEntropy(entropy),
|
||||
"strings": overviewStrings(stringsOut),
|
||||
"iocs": iocs["totals"],
|
||||
"carve": overviewCarve(carves),
|
||||
"limits": map[string]any{
|
||||
"entropyBytes": min64(size, 8<<20),
|
||||
"stringsBytes": min64(size, 16<<20),
|
||||
"iocBytes": min64(size, 16<<20),
|
||||
"carveBytes": min64(size, 64<<20),
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
func overviewEntropy(in map[string]any) map[string]any {
|
||||
return map[string]any{
|
||||
"windowSize": in["windowSize"],
|
||||
"step": in["step"],
|
||||
"mean": in["mean"],
|
||||
"min": in["min"],
|
||||
"max": in["max"],
|
||||
"highCount": in["highCount"],
|
||||
}
|
||||
}
|
||||
|
||||
func overviewStrings(items []map[string]any) map[string]any {
|
||||
counts := map[string]int{}
|
||||
for _, item := range items {
|
||||
if enc, ok := item["encoding"].(string); ok {
|
||||
counts[enc]++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"total": len(items),
|
||||
"byEncoding": counts,
|
||||
}
|
||||
}
|
||||
|
||||
func overviewCarve(items []map[string]any) map[string]any {
|
||||
counts := map[string]int{}
|
||||
valid := 0
|
||||
for _, item := range items {
|
||||
if kind, ok := item["kind"].(string); ok {
|
||||
counts[kind]++
|
||||
}
|
||||
if status, ok := item["status"].(string); ok && status == "valid" {
|
||||
valid++
|
||||
}
|
||||
}
|
||||
return map[string]any{
|
||||
"total": len(items),
|
||||
"valid": valid,
|
||||
"byKind": counts,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,648 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// Rule represents a lightweight YARA-like rule. A rule matches when
|
||||
// Condition evaluates to true for the document. Phase 1 supports:
|
||||
// - "all of them" (every string pattern must appear)
|
||||
// - "any of them" (at least one pattern must appear)
|
||||
// - "<count> of them" (at least N patterns)
|
||||
// - named identifiers referencing specific strings with $x or !$x
|
||||
type Rule struct {
|
||||
Name string
|
||||
Tags []string
|
||||
Strings []RuleString
|
||||
Condition string
|
||||
}
|
||||
|
||||
// RuleString is one string directive inside a rule.
|
||||
type RuleString struct {
|
||||
ID string
|
||||
Kind string // "ascii", "hex", "regex"
|
||||
Value string // original source text
|
||||
// compiled:
|
||||
raw []byte
|
||||
regex *regexp.Regexp
|
||||
nocase bool
|
||||
ascii bool
|
||||
wide bool
|
||||
fullword bool
|
||||
}
|
||||
|
||||
// ParseRules parses a simplified YARA-like source:
|
||||
//
|
||||
// rule Name : tag1 tag2 {
|
||||
// strings:
|
||||
// $a = "hello" nocase
|
||||
// $b = { 4d 5a ?? ?? }
|
||||
// $c = /foo[0-9]+/
|
||||
// condition:
|
||||
// any of them
|
||||
// }
|
||||
//
|
||||
// Hex patterns support `??` wildcards by being converted to a regex.
|
||||
func ParseRules(src string) ([]Rule, error) {
|
||||
rules := []Rule{}
|
||||
s := strings.TrimSpace(src)
|
||||
if s == "" {
|
||||
return rules, nil
|
||||
}
|
||||
// Split on "rule " keywords from start of line.
|
||||
entries := splitRules(s)
|
||||
for _, e := range entries {
|
||||
r, err := parseRule(e)
|
||||
if err != nil {
|
||||
return rules, err
|
||||
}
|
||||
rules = append(rules, r)
|
||||
}
|
||||
return rules, nil
|
||||
}
|
||||
|
||||
func splitRules(src string) []string {
|
||||
out := []string{}
|
||||
i := 0
|
||||
for i < len(src) {
|
||||
idx := strings.Index(src[i:], "rule ")
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
j := i + idx
|
||||
// find matching close brace (naïve but workable for phase 1)
|
||||
depth := 0
|
||||
start := j
|
||||
k := j
|
||||
for k < len(src) {
|
||||
c := src[k]
|
||||
if c == '{' {
|
||||
depth++
|
||||
} else if c == '}' {
|
||||
depth--
|
||||
if depth == 0 {
|
||||
out = append(out, src[start:k+1])
|
||||
k++
|
||||
break
|
||||
}
|
||||
}
|
||||
k++
|
||||
}
|
||||
i = k
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var ruleHeaderRe = regexp.MustCompile(`(?s)^\s*rule\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s*:\s*([^{]*))?\s*\{(.*)\}\s*$`)
|
||||
|
||||
func parseRule(src string) (Rule, error) {
|
||||
m := ruleHeaderRe.FindStringSubmatch(src)
|
||||
if m == nil {
|
||||
return Rule{}, errors.New("malformed rule")
|
||||
}
|
||||
r := Rule{Name: m[1]}
|
||||
if tags := strings.TrimSpace(m[2]); tags != "" {
|
||||
r.Tags = strings.Fields(tags)
|
||||
}
|
||||
body := m[3]
|
||||
if i := strings.Index(body, "strings:"); i >= 0 {
|
||||
sEnd := strings.Index(body, "condition:")
|
||||
sect := body[i+len("strings:") : sEnd]
|
||||
for _, line := range strings.Split(sect, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "//") {
|
||||
continue
|
||||
}
|
||||
rs, err := parseRuleString(line)
|
||||
if err != nil {
|
||||
return r, err
|
||||
}
|
||||
r.Strings = append(r.Strings, rs)
|
||||
}
|
||||
}
|
||||
if i := strings.Index(body, "condition:"); i >= 0 {
|
||||
r.Condition = strings.TrimSpace(body[i+len("condition:"):])
|
||||
} else {
|
||||
return r, errors.New("rule missing condition")
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
var ruleStringRe = regexp.MustCompile(`^\s*(\$[A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.+?)\s*$`)
|
||||
|
||||
func parseRuleString(line string) (RuleString, error) {
|
||||
m := ruleStringRe.FindStringSubmatch(line)
|
||||
if m == nil {
|
||||
return RuleString{}, errors.New("bad string line: " + line)
|
||||
}
|
||||
id := m[1]
|
||||
rest := strings.TrimSpace(m[2])
|
||||
value, modifiers, err := splitRuleValueModifiers(rest)
|
||||
if err != nil {
|
||||
return RuleString{}, err
|
||||
}
|
||||
rs := RuleString{ID: id, Value: value}
|
||||
for _, mod := range modifiers {
|
||||
switch strings.ToLower(mod) {
|
||||
case "nocase":
|
||||
rs.nocase = true
|
||||
case "ascii":
|
||||
rs.ascii = true
|
||||
case "wide":
|
||||
rs.wide = true
|
||||
case "fullword":
|
||||
rs.fullword = true
|
||||
default:
|
||||
return rs, errors.New("unknown string modifier: " + mod)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) && len(value) >= 2:
|
||||
rs.Kind = "ascii"
|
||||
rs.raw = []byte(value[1 : len(value)-1])
|
||||
if !rs.ascii && !rs.wide {
|
||||
rs.ascii = true
|
||||
}
|
||||
case strings.HasPrefix(value, "{") && strings.HasSuffix(value, "}"):
|
||||
rs.Kind = "hex"
|
||||
pattern := regexFromHexPattern(value[1 : len(value)-1])
|
||||
re, err := regexp.Compile("(?s)" + pattern)
|
||||
if err != nil {
|
||||
return rs, err
|
||||
}
|
||||
rs.regex = re
|
||||
case strings.HasPrefix(value, "/") && strings.HasSuffix(value, "/"):
|
||||
rs.Kind = "regex"
|
||||
pat := value[1 : len(value)-1]
|
||||
if rs.nocase {
|
||||
pat = "(?i)" + pat
|
||||
}
|
||||
re, err := regexp.Compile(pat)
|
||||
if err != nil {
|
||||
return rs, err
|
||||
}
|
||||
rs.regex = re
|
||||
default:
|
||||
return rs, errors.New("unknown string form: " + value)
|
||||
}
|
||||
return rs, nil
|
||||
}
|
||||
|
||||
func splitRuleValueModifiers(rest string) (string, []string, error) {
|
||||
rest = strings.TrimSpace(rest)
|
||||
if rest == "" {
|
||||
return "", nil, errors.New("empty string value")
|
||||
}
|
||||
var end int
|
||||
switch rest[0] {
|
||||
case '"':
|
||||
escaped := false
|
||||
end = -1
|
||||
for i := 1; i < len(rest); i++ {
|
||||
c := rest[i]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if c == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if c == '"' {
|
||||
end = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if end < 0 {
|
||||
return "", nil, errors.New("unterminated string literal")
|
||||
}
|
||||
case '{':
|
||||
idx := strings.IndexByte(rest, '}')
|
||||
if idx < 0 {
|
||||
return "", nil, errors.New("unterminated hex string")
|
||||
}
|
||||
end = idx + 1
|
||||
case '/':
|
||||
escaped := false
|
||||
end = -1
|
||||
for i := 1; i < len(rest); i++ {
|
||||
c := rest[i]
|
||||
if escaped {
|
||||
escaped = false
|
||||
continue
|
||||
}
|
||||
if c == '\\' {
|
||||
escaped = true
|
||||
continue
|
||||
}
|
||||
if c == '/' {
|
||||
end = i + 1
|
||||
break
|
||||
}
|
||||
}
|
||||
if end < 0 {
|
||||
return "", nil, errors.New("unterminated regex")
|
||||
}
|
||||
default:
|
||||
return "", nil, errors.New("unknown string form: " + rest)
|
||||
}
|
||||
return strings.TrimSpace(rest[:end]), strings.Fields(strings.TrimSpace(rest[end:])), nil
|
||||
}
|
||||
|
||||
func regexFromHexPattern(hexPat string) string {
|
||||
var sb strings.Builder
|
||||
hexPat = strings.ReplaceAll(hexPat, " ", "")
|
||||
for i := 0; i < len(hexPat); {
|
||||
if i+1 >= len(hexPat) {
|
||||
break
|
||||
}
|
||||
a, b := hexPat[i], hexPat[i+1]
|
||||
if a == '?' && b == '?' {
|
||||
sb.WriteString(`[\x00-\xff]`)
|
||||
} else {
|
||||
decoded, err := hex.DecodeString(string([]byte{a, b}))
|
||||
if err != nil {
|
||||
// bail to literal to avoid panic
|
||||
sb.WriteString(regexp.QuoteMeta(string([]byte{a, b})))
|
||||
} else {
|
||||
sb.WriteString(regexp.QuoteMeta(string(decoded)))
|
||||
}
|
||||
}
|
||||
i += 2
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// RuleMatch is a per-rule result.
|
||||
type RuleMatch struct {
|
||||
Name string
|
||||
Tags []string
|
||||
Matched bool
|
||||
Hits map[string][]SearchHit
|
||||
}
|
||||
|
||||
// RunRules scans rules against the document range and returns matches.
|
||||
func RunRules(d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) {
|
||||
return RunRulesContext(context.Background(), d, start, length, rules, maxPerString)
|
||||
}
|
||||
|
||||
func RunRulesContext(ctx context.Context, d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) {
|
||||
if maxPerString <= 0 {
|
||||
maxPerString = 1000
|
||||
}
|
||||
out := make([]map[string]any, 0, len(rules))
|
||||
for _, r := range rules {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits := map[string][]SearchHit{}
|
||||
for _, s := range r.Strings {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch s.Kind {
|
||||
case "ascii":
|
||||
found, err := searchRuleTextContext(ctx, d, start, length, s, maxPerString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits[s.ID] = found
|
||||
case "hex", "regex":
|
||||
found, err := SearchRegexContext(ctx, d, start, length, s.regex.String(), maxPerString)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.fullword {
|
||||
found = filterFullWordHits(d, found, false)
|
||||
}
|
||||
hits[s.ID] = found
|
||||
}
|
||||
}
|
||||
matched := evalCondition(r, hits)
|
||||
var hitFlat []map[string]any
|
||||
for id, hs := range hits {
|
||||
for _, h := range hs {
|
||||
hitFlat = append(hitFlat, map[string]any{
|
||||
"id": id,
|
||||
"offset": h.Offset,
|
||||
"length": h.Length,
|
||||
"sample": h.Sample,
|
||||
})
|
||||
}
|
||||
}
|
||||
out = append(out, map[string]any{
|
||||
"name": r.Name,
|
||||
"tags": toAnySlice(r.Tags),
|
||||
"matched": matched,
|
||||
"hits": hitFlat,
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func searchRuleText(d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) {
|
||||
return searchRuleTextContext(context.Background(), d, start, length, s, max)
|
||||
}
|
||||
|
||||
func searchRuleTextContext(ctx context.Context, d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) {
|
||||
out := []SearchHit{}
|
||||
if s.ascii {
|
||||
var (
|
||||
found []SearchHit
|
||||
err error
|
||||
)
|
||||
if s.nocase {
|
||||
found, err = SearchRegexContext(ctx, d, start, length, "(?i)"+regexp.QuoteMeta(string(s.raw)), max)
|
||||
} else {
|
||||
found, err = SearchBytesContext(ctx, d, start, length, s.raw, max)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.fullword {
|
||||
found = filterFullWordHits(d, found, false)
|
||||
}
|
||||
out = append(out, found...)
|
||||
}
|
||||
if s.wide {
|
||||
wide := utf16LEBytes(s.raw)
|
||||
var (
|
||||
found []SearchHit
|
||||
err error
|
||||
)
|
||||
if s.nocase {
|
||||
pat := wideNoCasePattern(s.raw)
|
||||
found, err = SearchRegexContext(ctx, d, start, length, pat, max)
|
||||
} else {
|
||||
found, err = SearchBytesContext(ctx, d, start, length, wide, max)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.fullword {
|
||||
found = filterFullWordHits(d, found, true)
|
||||
}
|
||||
out = append(out, found...)
|
||||
}
|
||||
if len(out) > max {
|
||||
out = out[:max]
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func utf16LEBytes(in []byte) []byte {
|
||||
out := make([]byte, 0, len(in)*2)
|
||||
for _, b := range in {
|
||||
out = append(out, b, 0)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func wideNoCasePattern(in []byte) string {
|
||||
var sb strings.Builder
|
||||
sb.WriteString("(?i)")
|
||||
for _, b := range in {
|
||||
sb.WriteString(regexp.QuoteMeta(string([]byte{b, 0})))
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
func filterFullWordHits(d *document.Document, hits []SearchHit, wide bool) []SearchHit {
|
||||
out := hits[:0]
|
||||
for _, h := range hits {
|
||||
if isFullWordHit(d, h, wide) {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func isFullWordHit(d *document.Document, h SearchHit, wide bool) bool {
|
||||
if wide {
|
||||
beforeOff := h.Offset - 2
|
||||
afterOff := h.Offset + int64(h.Length)
|
||||
beforeWord := false
|
||||
afterWord := false
|
||||
if beforeOff >= 0 {
|
||||
if b, err := d.Range(beforeOff, 2); err == nil && len(b) == 2 && b[1] == 0 {
|
||||
beforeWord = isWordByte(b[0])
|
||||
}
|
||||
}
|
||||
if afterOff+1 < d.Size() {
|
||||
if b, err := d.Range(afterOff, 2); err == nil && len(b) == 2 && b[1] == 0 {
|
||||
afterWord = isWordByte(b[0])
|
||||
}
|
||||
}
|
||||
return !beforeWord && !afterWord
|
||||
}
|
||||
beforeWord := false
|
||||
afterWord := false
|
||||
if h.Offset > 0 {
|
||||
if b, err := d.At(h.Offset - 1); err == nil {
|
||||
beforeWord = isWordByte(b)
|
||||
}
|
||||
}
|
||||
if h.Offset+int64(h.Length) < d.Size() {
|
||||
if b, err := d.At(h.Offset + int64(h.Length)); err == nil {
|
||||
afterWord = isWordByte(b)
|
||||
}
|
||||
}
|
||||
return !beforeWord && !afterWord
|
||||
}
|
||||
|
||||
func isWordByte(b byte) bool {
|
||||
return (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9') || b == '_'
|
||||
}
|
||||
|
||||
func toAnySlice(in []string) []any {
|
||||
out := make([]any, len(in))
|
||||
for i, s := range in {
|
||||
out[i] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func evalCondition(r Rule, hits map[string][]SearchHit) bool {
|
||||
cond := strings.TrimSpace(r.Condition)
|
||||
condLower := strings.ToLower(cond)
|
||||
present := 0
|
||||
for _, s := range r.Strings {
|
||||
if len(hits[s.ID]) > 0 {
|
||||
present++
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case condLower == "any of them":
|
||||
return present > 0
|
||||
case condLower == "all of them":
|
||||
return present == len(r.Strings)
|
||||
case strings.HasSuffix(condLower, "of them"):
|
||||
// "<n> of them"
|
||||
parts := strings.Fields(condLower)
|
||||
if len(parts) >= 3 {
|
||||
n, ok := parseRuleCount(parts[0])
|
||||
return ok && present >= n
|
||||
}
|
||||
}
|
||||
if ok, matched := evalRuleSetCondition(cond, hits); ok {
|
||||
return matched
|
||||
}
|
||||
p := conditionParser{tokens: tokenizeCondition(cond), hits: hits}
|
||||
return p.parse()
|
||||
}
|
||||
|
||||
func evalRuleSetCondition(cond string, hits map[string][]SearchHit) (bool, bool) {
|
||||
tokens := tokenizeCondition(cond)
|
||||
if len(tokens) < 4 || strings.ToLower(tokens[1]) != "of" || tokens[2] != "(" || tokens[len(tokens)-1] != ")" {
|
||||
return false, false
|
||||
}
|
||||
ids := []string{}
|
||||
for _, tok := range tokens[3 : len(tokens)-1] {
|
||||
if tok == "," {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(tok, "$") {
|
||||
ids = append(ids, tok)
|
||||
}
|
||||
}
|
||||
if len(ids) == 0 {
|
||||
return true, false
|
||||
}
|
||||
count := 0
|
||||
for _, id := range ids {
|
||||
if len(hits[id]) > 0 {
|
||||
count++
|
||||
}
|
||||
}
|
||||
switch strings.ToLower(tokens[0]) {
|
||||
case "any":
|
||||
return true, count > 0
|
||||
case "all":
|
||||
return true, count == len(ids)
|
||||
default:
|
||||
n, ok := parseRuleCount(strings.ToLower(tokens[0]))
|
||||
return true, ok && count >= n
|
||||
}
|
||||
}
|
||||
|
||||
func parseRuleCount(s string) (int, bool) {
|
||||
switch s {
|
||||
case "1", "one":
|
||||
return 1, true
|
||||
case "2", "two":
|
||||
return 2, true
|
||||
case "3", "three":
|
||||
return 3, true
|
||||
default:
|
||||
n, err := strconv.Atoi(s)
|
||||
return n, err == nil
|
||||
}
|
||||
}
|
||||
|
||||
type conditionParser struct {
|
||||
tokens []string
|
||||
pos int
|
||||
hits map[string][]SearchHit
|
||||
}
|
||||
|
||||
func (p *conditionParser) parse() bool {
|
||||
v := p.parseOr()
|
||||
return v
|
||||
}
|
||||
|
||||
func (p *conditionParser) parseOr() bool {
|
||||
v := p.parseAnd()
|
||||
for strings.EqualFold(p.peek(), "or") {
|
||||
p.next()
|
||||
v = p.parseAnd() || v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (p *conditionParser) parseAnd() bool {
|
||||
v := p.parseNot()
|
||||
for strings.EqualFold(p.peek(), "and") {
|
||||
p.next()
|
||||
v = p.parseNot() && v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (p *conditionParser) parseNot() bool {
|
||||
if strings.EqualFold(p.peek(), "not") || p.peek() == "!" {
|
||||
p.next()
|
||||
return !p.parseNot()
|
||||
}
|
||||
return p.parsePrimary()
|
||||
}
|
||||
|
||||
func (p *conditionParser) parsePrimary() bool {
|
||||
tok := p.next()
|
||||
switch {
|
||||
case tok == "":
|
||||
return false
|
||||
case tok == "(":
|
||||
v := p.parseOr()
|
||||
if p.peek() == ")" {
|
||||
p.next()
|
||||
}
|
||||
return v
|
||||
case strings.EqualFold(tok, "true"):
|
||||
return true
|
||||
case strings.EqualFold(tok, "false"):
|
||||
return false
|
||||
case strings.HasPrefix(tok, "$"):
|
||||
return len(p.hits[tok]) > 0
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (p *conditionParser) peek() string {
|
||||
if p.pos >= len(p.tokens) {
|
||||
return ""
|
||||
}
|
||||
return p.tokens[p.pos]
|
||||
}
|
||||
|
||||
func (p *conditionParser) next() string {
|
||||
if p.pos >= len(p.tokens) {
|
||||
return ""
|
||||
}
|
||||
tok := p.tokens[p.pos]
|
||||
p.pos++
|
||||
return tok
|
||||
}
|
||||
|
||||
func tokenizeCondition(cond string) []string {
|
||||
out := []string{}
|
||||
for i := 0; i < len(cond); {
|
||||
c := cond[i]
|
||||
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if strings.ContainsRune("(),!", rune(c)) {
|
||||
out = append(out, string(c))
|
||||
i++
|
||||
continue
|
||||
}
|
||||
start := i
|
||||
for i < len(cond) {
|
||||
c = cond[i]
|
||||
if c == ' ' || c == '\t' || c == '\n' || c == '\r' || strings.ContainsRune("(),!", rune(c)) {
|
||||
break
|
||||
}
|
||||
i++
|
||||
}
|
||||
out = append(out, cond[start:i])
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,189 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"regexp"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// SearchHit describes one search result.
|
||||
type SearchHit struct {
|
||||
Offset int64 `json:"offset"`
|
||||
Length int `json:"length"`
|
||||
Sample string `json:"sample"` // short hex preview
|
||||
}
|
||||
|
||||
// SearchBytes finds `needle` in the document range and returns hit
|
||||
// offsets. Scans chunk by chunk with an overlap equal to len(needle)-1
|
||||
// so cross-chunk matches are not missed.
|
||||
func SearchBytes(d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) {
|
||||
return SearchBytesContext(context.Background(), d, start, length, needle, max)
|
||||
}
|
||||
|
||||
func SearchBytesContext(ctx context.Context, d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) {
|
||||
if len(needle) == 0 {
|
||||
return nil, errors.New("empty needle")
|
||||
}
|
||||
if max <= 0 {
|
||||
max = 10_000
|
||||
}
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
|
||||
hits := make([]SearchHit, 0, 64)
|
||||
overlap := int64(len(needle) - 1)
|
||||
|
||||
// We chunk in windows of chunkSize + overlap.
|
||||
const chunkSize = 1 << 20 // 1MB
|
||||
off := start
|
||||
endExclusive := start + length
|
||||
|
||||
for off < endExclusive && len(hits) < max {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
winEnd := off + chunkSize
|
||||
if winEnd > endExclusive {
|
||||
winEnd = endExclusive
|
||||
}
|
||||
readEnd := winEnd + overlap
|
||||
if readEnd > endExclusive {
|
||||
readEnd = endExclusive
|
||||
}
|
||||
buf, err := d.Range(off, readEnd-off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Limit search to bytes whose start lies in [0, winEnd-off)
|
||||
searchLimit := winEnd - off
|
||||
scan := 0
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
idx := bytes.Index(buf[scan:], needle)
|
||||
if idx < 0 {
|
||||
break
|
||||
}
|
||||
abs := int64(scan+idx) + off
|
||||
relStart := abs - off
|
||||
if relStart >= searchLimit {
|
||||
break
|
||||
}
|
||||
hits = append(hits, SearchHit{
|
||||
Offset: abs,
|
||||
Length: len(needle),
|
||||
Sample: hexPreview(buf[scan+idx:], len(needle)),
|
||||
})
|
||||
if len(hits) >= max {
|
||||
break
|
||||
}
|
||||
scan = scan + idx + 1
|
||||
if scan >= len(buf) {
|
||||
break
|
||||
}
|
||||
}
|
||||
off = winEnd
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// SearchRegex runs a Go regexp across the document range, matching
|
||||
// byte-wise. It decodes matches chunk by chunk with overlap, bounded
|
||||
// by a generous max hit count.
|
||||
func SearchRegex(d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) {
|
||||
return SearchRegexContext(context.Background(), d, start, length, pattern, max)
|
||||
}
|
||||
|
||||
func SearchRegexContext(ctx context.Context, d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if max <= 0 {
|
||||
max = 10_000
|
||||
}
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
|
||||
// For regex, a conservative overlap is 4 KiB so matches spanning
|
||||
// chunk boundaries are still caught. Very long matches may still
|
||||
// be missed in extreme cases.
|
||||
const chunkSize = 1 << 20
|
||||
const overlap = 4096
|
||||
|
||||
hits := make([]SearchHit, 0, 64)
|
||||
off := start
|
||||
endExclusive := start + length
|
||||
|
||||
for off < endExclusive && len(hits) < max {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
winEnd := off + chunkSize
|
||||
if winEnd > endExclusive {
|
||||
winEnd = endExclusive
|
||||
}
|
||||
readEnd := winEnd + overlap
|
||||
if readEnd > endExclusive {
|
||||
readEnd = endExclusive
|
||||
}
|
||||
buf, err := d.Range(off, readEnd-off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
searchLimit := winEnd - off
|
||||
locs := re.FindAllIndex(buf, max-len(hits))
|
||||
for _, loc := range locs {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
relStart := int64(loc[0])
|
||||
if relStart >= searchLimit {
|
||||
continue
|
||||
}
|
||||
abs := relStart + off
|
||||
hits = append(hits, SearchHit{
|
||||
Offset: abs,
|
||||
Length: loc[1] - loc[0],
|
||||
Sample: hexPreview(buf[loc[0]:loc[1]], loc[1]-loc[0]),
|
||||
})
|
||||
}
|
||||
off = winEnd
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
func hexPreview(b []byte, n int) string {
|
||||
if n > 32 {
|
||||
n = 32
|
||||
}
|
||||
if n > len(b) {
|
||||
n = len(b)
|
||||
}
|
||||
const hexchars = "0123456789abcdef"
|
||||
out := make([]byte, n*2)
|
||||
for i := 0; i < n; i++ {
|
||||
out[i*2] = hexchars[b[i]>>4]
|
||||
out[i*2+1] = hexchars[b[i]&0x0f]
|
||||
}
|
||||
return string(out)
|
||||
}
|
||||
|
||||
// HitsToMaps converts hits for bridge return.
|
||||
func HitsToMaps(hits []SearchHit) []map[string]any {
|
||||
out := make([]map[string]any, len(hits))
|
||||
for i, h := range hits {
|
||||
out[i] = map[string]any{
|
||||
"offset": h.Offset,
|
||||
"length": h.Length,
|
||||
"sample": h.Sample,
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// StringHit represents one extracted string.
|
||||
type StringHit struct {
|
||||
Offset int64 `json:"offset"`
|
||||
Length int `json:"length"`
|
||||
Encoding string `json:"encoding"` // ascii | utf16le | utf16be
|
||||
Value string `json:"value"`
|
||||
}
|
||||
|
||||
// ExtractStrings finds printable runs in the specified range.
|
||||
// Supports ASCII and UTF-16 LE/BE. minLen is in characters.
|
||||
func ExtractStrings(d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) {
|
||||
return ExtractStringsContext(context.Background(), d, start, length, minLen, encodings, max)
|
||||
}
|
||||
|
||||
func ExtractStringsContext(ctx context.Context, d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) {
|
||||
if minLen <= 0 {
|
||||
minLen = 4
|
||||
}
|
||||
if max <= 0 {
|
||||
max = 50_000
|
||||
}
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
|
||||
doASCII, doLE, doBE := false, false, false
|
||||
if len(encodings) == 0 {
|
||||
doASCII, doLE, doBE = true, true, true
|
||||
} else {
|
||||
for _, e := range encodings {
|
||||
switch e {
|
||||
case "ascii":
|
||||
doASCII = true
|
||||
case "utf16le":
|
||||
doLE = true
|
||||
case "utf16be":
|
||||
doBE = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
out := make([]map[string]any, 0, 256)
|
||||
|
||||
// ASCII
|
||||
if doASCII {
|
||||
if err := appendASCII(ctx, d, start, length, minLen, &out, max); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
// UTF-16
|
||||
if len(out) < max && doLE {
|
||||
if err := appendUTF16(ctx, d, start, length, minLen, true, &out, max); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if len(out) < max && doBE {
|
||||
if err := appendUTF16(ctx, d, start, length, minLen, false, &out, max); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func appendASCII(ctx context.Context, d *document.Document, start, length int64, minLen int, out *[]map[string]any, max int) error {
|
||||
// Chunked scan with a carry for runs crossing chunk boundaries.
|
||||
var runStart int64 = -1
|
||||
var runBuf []byte
|
||||
emit := func() {
|
||||
if runStart >= 0 && len(runBuf) >= minLen {
|
||||
*out = append(*out, map[string]any{
|
||||
"offset": runStart,
|
||||
"length": len(runBuf),
|
||||
"encoding": "ascii",
|
||||
"value": string(runBuf),
|
||||
})
|
||||
}
|
||||
runStart = -1
|
||||
runBuf = runBuf[:0]
|
||||
}
|
||||
|
||||
if err := d.ForEachChunk(start, length, func(off int64, chunk []byte) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
for i, b := range chunk {
|
||||
if i%4096 == 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if isPrintableASCII(b) {
|
||||
if runStart < 0 {
|
||||
runStart = off + int64(i)
|
||||
runBuf = make([]byte, 0, 32)
|
||||
}
|
||||
runBuf = append(runBuf, b)
|
||||
} else {
|
||||
if len(*out) >= max {
|
||||
return nil
|
||||
}
|
||||
emit()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
emit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendUTF16(ctx context.Context, d *document.Document, start, length int64, minLen int, le bool, out *[]map[string]any, max int) error {
|
||||
// Step by 2. Produce one string for each contiguous run of printable code units.
|
||||
if length < 2 {
|
||||
return nil
|
||||
}
|
||||
// Read chunks aligned to 2 bytes so pairs don't split.
|
||||
const chunkSize = 1 << 20
|
||||
end := start + length
|
||||
var runStart int64 = -1
|
||||
var runBuf []rune
|
||||
|
||||
enc := "utf16le"
|
||||
if !le {
|
||||
enc = "utf16be"
|
||||
}
|
||||
|
||||
emit := func() {
|
||||
if runStart >= 0 && len(runBuf) >= minLen {
|
||||
*out = append(*out, map[string]any{
|
||||
"offset": runStart,
|
||||
"length": len(runBuf) * 2,
|
||||
"encoding": enc,
|
||||
"value": string(runBuf),
|
||||
})
|
||||
}
|
||||
runStart = -1
|
||||
runBuf = runBuf[:0]
|
||||
}
|
||||
|
||||
for off := start; off+1 < end; {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
winEnd := off + chunkSize
|
||||
if winEnd > end {
|
||||
winEnd = end
|
||||
}
|
||||
if (winEnd-off)%2 != 0 {
|
||||
winEnd-- // keep pair alignment
|
||||
}
|
||||
buf, err := d.Range(off, winEnd-off)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := 0; i+1 < len(buf); i += 2 {
|
||||
if i%4096 == 0 {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
var cu uint16
|
||||
if le {
|
||||
cu = uint16(buf[i]) | uint16(buf[i+1])<<8
|
||||
} else {
|
||||
cu = uint16(buf[i])<<8 | uint16(buf[i+1])
|
||||
}
|
||||
if cu >= 0x20 && cu <= 0x7e {
|
||||
if runStart < 0 {
|
||||
runStart = off + int64(i)
|
||||
runBuf = make([]rune, 0, 32)
|
||||
}
|
||||
runBuf = append(runBuf, rune(cu))
|
||||
} else {
|
||||
if len(*out) >= max {
|
||||
emit()
|
||||
return nil
|
||||
}
|
||||
emit()
|
||||
}
|
||||
}
|
||||
off = winEnd
|
||||
}
|
||||
emit()
|
||||
return nil
|
||||
}
|
||||
|
||||
func isPrintableASCII(b byte) bool {
|
||||
return b == '\t' || (b >= 0x20 && b <= 0x7e)
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// ScanTimestamps walks the range looking for plausible timestamps in
|
||||
// common encodings. This is a heuristic: it reports any 4/8-byte
|
||||
// window whose decoded time falls in [1990-01-01, 2050-12-31].
|
||||
func ScanTimestamps(d *document.Document, start, length int64, maxHits int) (map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
if maxHits <= 0 {
|
||||
maxHits = 20_000
|
||||
}
|
||||
|
||||
minT := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
|
||||
maxT := time.Date(2050, 12, 31, 0, 0, 0, 0, time.UTC).Unix()
|
||||
ok := func(sec int64) bool { return sec >= minT && sec <= maxT }
|
||||
|
||||
hits := make([]map[string]any, 0, 256)
|
||||
counts := map[string]int{}
|
||||
|
||||
const chunk = 1 << 20
|
||||
end := start + length
|
||||
overlap := int64(7) // enough for 8-byte types
|
||||
for off := start; off < end && len(hits) < maxHits; {
|
||||
winEnd := off + chunk
|
||||
if winEnd > end {
|
||||
winEnd = end
|
||||
}
|
||||
readEnd := winEnd + overlap
|
||||
if readEnd > end {
|
||||
readEnd = end
|
||||
}
|
||||
buf, err := d.Range(off, readEnd-off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit := int(winEnd - off)
|
||||
for i := 0; i < limit && len(hits) < maxHits; i += 4 {
|
||||
if i+4 <= len(buf) {
|
||||
vLE := int64(int32(binary.LittleEndian.Uint32(buf[i:])))
|
||||
if ok(vLE) {
|
||||
hits = append(hits, hit("unix_le", off+int64(i), 4, time.Unix(vLE, 0).UTC()))
|
||||
counts["unix_le"]++
|
||||
continue
|
||||
}
|
||||
vBE := int64(int32(binary.BigEndian.Uint32(buf[i:])))
|
||||
if ok(vBE) {
|
||||
hits = append(hits, hit("unix_be", off+int64(i), 4, time.Unix(vBE, 0).UTC()))
|
||||
counts["unix_be"]++
|
||||
continue
|
||||
}
|
||||
}
|
||||
if i+8 <= len(buf) {
|
||||
v := binary.LittleEndian.Uint64(buf[i:])
|
||||
// Windows FILETIME
|
||||
sec := int64(v/10_000_000) - filetimeEpochDelta
|
||||
if ok(sec) {
|
||||
hits = append(hits, hit("filetime_le", off+int64(i), 8, time.Unix(sec, 0).UTC()))
|
||||
counts["filetime_le"]++
|
||||
continue
|
||||
}
|
||||
// WebKit (microseconds since 1601)
|
||||
sec = int64(v/1_000_000) - filetimeEpochDelta
|
||||
if ok(sec) {
|
||||
hits = append(hits, hit("webkit_le", off+int64(i), 8, time.Unix(sec, 0).UTC()))
|
||||
counts["webkit_le"]++
|
||||
continue
|
||||
}
|
||||
// OLE automation date
|
||||
fdays := math.Float64frombits(v)
|
||||
if fdays > 2 && fdays < 75000 {
|
||||
base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
|
||||
t := base.Add(time.Duration(fdays*86400) * time.Second)
|
||||
if t.Unix() >= minT && t.Unix() <= maxT {
|
||||
hits = append(hits, hit("ole_le", off+int64(i), 8, t))
|
||||
counts["ole_le"]++
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
off = winEnd
|
||||
}
|
||||
return map[string]any{
|
||||
"hits": hits,
|
||||
"counts": counts,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func hit(kind string, off int64, size int, t time.Time) map[string]any {
|
||||
return map[string]any{
|
||||
"kind": kind,
|
||||
"offset": off,
|
||||
"length": size,
|
||||
"iso": t.Format(time.RFC3339),
|
||||
"unix": t.Unix(),
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user