v3 candidate
This commit is contained in:
Andy Dixon
@@ -0,0 +1,876 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
|
||||
"github.com/andydixon/nexus/internal/leviathan/document"
|
||||
)
|
||||
|
||||
// Signature describes a carveable object. `validate` is run against
|
||||
// the candidate start and returns the object length if it is a real
|
||||
// object, or 0 to reject. `trailer`, if present, must appear before
|
||||
// the object ends (for simple trailer-based terminations).
|
||||
type Signature struct {
|
||||
Kind string
|
||||
Magic []byte
|
||||
Extension string
|
||||
// Anchor is how many bytes into the object the magic appears. It is 0 for
|
||||
// almost every format (magic at offset 0), but a few — notably ISO-BMFF
|
||||
// (MP4/MOV), whose `ftyp` magic follows a 4-byte box size — carry it.
|
||||
Anchor int64
|
||||
Validate func(d *document.Document, start, max int64) (length int64, confidence float64)
|
||||
}
|
||||
|
||||
// carveSignatures is the phase-1 signature set. We stay conservative
|
||||
// and only declare a carve when we can determine length reliably.
|
||||
var carveSignatures = []Signature{
|
||||
{Kind: "png", Magic: []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, Extension: "png", Validate: validatePNG},
|
||||
{Kind: "jpeg", Magic: []byte{0xff, 0xd8, 0xff}, Extension: "jpg", Validate: validateJPEG},
|
||||
{Kind: "gif87a", Magic: []byte("GIF87a"), Extension: "gif", Validate: validateGIF},
|
||||
{Kind: "gif89a", Magic: []byte("GIF89a"), Extension: "gif", Validate: validateGIF},
|
||||
{Kind: "zip", Magic: []byte{'P', 'K', 0x03, 0x04}, Extension: "zip", Validate: validateZIP},
|
||||
{Kind: "pdf", Magic: []byte("%PDF-"), Extension: "pdf", Validate: validatePDF},
|
||||
{Kind: "elf", Magic: []byte{0x7f, 'E', 'L', 'F'}, Extension: "elf", Validate: validateELF},
|
||||
{Kind: "mz", Magic: []byte{'M', 'Z'}, Extension: "exe", Validate: validatePE},
|
||||
{Kind: "gzip", Magic: []byte{0x1f, 0x8b, 0x08}, Extension: "gz", Validate: validateGZIP},
|
||||
{Kind: "bzip2", Magic: []byte{'B', 'Z', 'h'}, Extension: "bz2", Validate: validateBzip2},
|
||||
{Kind: "sevenz", Magic: []byte{'7', 'z', 0xbc, 0xaf, 0x27, 0x1c}, Extension: "7z", Validate: validate7z},
|
||||
{Kind: "rar4", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x00}, Extension: "rar", Validate: validateRAR},
|
||||
{Kind: "rar5", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00}, Extension: "rar", Validate: validateRAR},
|
||||
{Kind: "wav", Magic: []byte("RIFF"), Extension: "wav", Validate: validateRIFF},
|
||||
{Kind: "mach-o-le32", Magic: []byte{0xce, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-le64", Magic: []byte{0xcf, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-be32", Magic: []byte{0xfe, 0xed, 0xfa, 0xce}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "mach-o-be64", Magic: []byte{0xfe, 0xed, 0xfa, 0xcf}, Extension: "macho", Validate: validateMachO},
|
||||
{Kind: "sqlite", Magic: []byte("SQLite format 3\x00"), Extension: "sqlite", Validate: validateSQLite},
|
||||
{Kind: "mp3", Magic: []byte("ID3"), Extension: "mp3", Validate: validateID3},
|
||||
{Kind: "ogg", Magic: []byte("OggS"), Extension: "ogg", Validate: validateOgg},
|
||||
{Kind: "matroska", Magic: []byte{0x1a, 0x45, 0xdf, 0xa3}, Extension: "mkv", Validate: validateMatroska},
|
||||
{Kind: "flac", Magic: []byte("fLaC"), Extension: "flac", Validate: validateFLAC},
|
||||
{Kind: "bmp", Magic: []byte{'B', 'M'}, Extension: "bmp", Validate: validateBMP},
|
||||
{Kind: "mp4", Magic: []byte("ftyp"), Anchor: 4, Extension: "mp4", Validate: validateMP4},
|
||||
}
|
||||
|
||||
// Carve scans the document for known signatures and returns a list of
|
||||
// discovered objects with offset, length, kind, and confidence.
|
||||
func Carve(d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
|
||||
return CarveContext(context.Background(), d, start, length, maxHits)
|
||||
}
|
||||
|
||||
func CarveContext(ctx context.Context, d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
|
||||
if length < 0 {
|
||||
length = d.Size() - start
|
||||
}
|
||||
if maxHits <= 0 {
|
||||
maxHits = 10_000
|
||||
}
|
||||
results := make([]map[string]any, 0, 64)
|
||||
|
||||
// Build a flat list of (magicBytes, sigIndex) for a simple first-pass scan.
|
||||
// Phase 1 uses per-signature naive scan chunks.
|
||||
for idx, sig := range carveSignatures {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
hits, err := SearchBytesContext(ctx, d, start, length, sig.Magic, maxHits)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, h := range hits {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// The object begins Anchor bytes before the magic (0 for most formats).
|
||||
objStart := h.Offset - sig.Anchor
|
||||
if objStart < start {
|
||||
continue
|
||||
}
|
||||
ln, conf := int64(0), 0.4
|
||||
if sig.Validate != nil {
|
||||
ln, conf = sig.Validate(d, objStart, start+length)
|
||||
}
|
||||
if ln <= 0 {
|
||||
// record low-confidence magic hit only for kinds where length is not determinable
|
||||
if sig.Kind == "mz" || sig.Kind == "elf" || sig.Kind == "mach-o-le32" ||
|
||||
sig.Kind == "mach-o-le64" || sig.Kind == "mach-o-be32" || sig.Kind == "mach-o-be64" ||
|
||||
sig.Kind == "sqlite" || sig.Kind == "mp3" || sig.Kind == "matroska" ||
|
||||
sig.Kind == "flac" || sig.Kind == "bzip2" || sig.Kind == "rar4" || sig.Kind == "rar5" {
|
||||
results = append(results, map[string]any{
|
||||
"kind": sig.Kind,
|
||||
"extension": sig.Extension,
|
||||
"offset": objStart,
|
||||
"length": int64(0),
|
||||
"confidence": 0.3,
|
||||
"status": "magic-only",
|
||||
"sigIndex": idx,
|
||||
})
|
||||
}
|
||||
continue
|
||||
}
|
||||
results = append(results, map[string]any{
|
||||
"kind": sig.Kind,
|
||||
"extension": sig.Extension,
|
||||
"offset": objStart,
|
||||
"length": ln,
|
||||
"confidence": conf,
|
||||
"status": "valid",
|
||||
"sigIndex": idx,
|
||||
})
|
||||
if len(results) >= maxHits {
|
||||
return results, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// --- validators ---
|
||||
|
||||
func validatePNG(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Walk chunks until IEND
|
||||
off := start + 8
|
||||
for off < max {
|
||||
hdr, err := d.Range(off, 8)
|
||||
if err != nil || len(hdr) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.BigEndian.Uint32(hdr[0:4]))
|
||||
typ := string(hdr[4:8])
|
||||
// Skip payload + CRC (4 bytes)
|
||||
off += 8 + size + 4
|
||||
if typ == "IEND" {
|
||||
return off - start, 0.95
|
||||
}
|
||||
if off > max {
|
||||
return 0, 0
|
||||
}
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateJPEG(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Walk segments until EOI (0xFFD9)
|
||||
off := start + 2
|
||||
for off < max-1 {
|
||||
buf, err := d.Range(off, 2)
|
||||
if err != nil || len(buf) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
if buf[0] != 0xff {
|
||||
return 0, 0
|
||||
}
|
||||
marker := buf[1]
|
||||
off += 2
|
||||
// No-length markers
|
||||
if marker == 0xd9 { // EOI
|
||||
return off - start, 0.95
|
||||
}
|
||||
if marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker == 0x00 {
|
||||
continue
|
||||
}
|
||||
// Read length
|
||||
lb, err := d.Range(off, 2)
|
||||
if err != nil || len(lb) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
ln := int64(binary.BigEndian.Uint16(lb))
|
||||
off += ln
|
||||
if marker == 0xda { // SOS, scan until next 0xff!=0
|
||||
for off < max {
|
||||
b, err := d.Range(off, 2)
|
||||
if err != nil || len(b) < 2 {
|
||||
return 0, 0
|
||||
}
|
||||
if b[0] == 0xff && b[1] != 0x00 && !(b[1] >= 0xd0 && b[1] <= 0xd7) {
|
||||
break
|
||||
}
|
||||
off++
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateGIF(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Minimal: locate trailer 0x3B
|
||||
buf, err := d.Range(start, min64(max-start, 1<<20))
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.IndexByte(buf[13:], 0x3b)
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
return int64(idx + 13 + 1), 0.7
|
||||
}
|
||||
|
||||
func validateZIP(d *document.Document, start, max int64) (int64, float64) {
|
||||
// Search forward for End of Central Directory record 0x06054b50.
|
||||
const sig = "\x50\x4b\x05\x06"
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.LastIndex(buf, []byte(sig))
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
// EOCD record is at least 22 bytes; read comment length.
|
||||
if idx+22 > len(buf) {
|
||||
return 0, 0
|
||||
}
|
||||
commentLen := int(binary.LittleEndian.Uint16(buf[idx+20 : idx+22]))
|
||||
end := int64(idx + 22 + commentLen)
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
func validatePDF(d *document.Document, start, max int64) (int64, float64) {
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
idx := bytes.LastIndex(buf, []byte("%%EOF"))
|
||||
if idx < 0 {
|
||||
return 0, 0
|
||||
}
|
||||
return int64(idx + 5), 0.85
|
||||
}
|
||||
|
||||
func validateELF(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 64)
|
||||
if err != nil || len(hdr) < 52 {
|
||||
return 0, 0
|
||||
}
|
||||
if hdr[4] != 1 && hdr[4] != 2 {
|
||||
return 0, 0
|
||||
}
|
||||
is64 := hdr[4] == 2
|
||||
little := hdr[5] == 1
|
||||
var end int64
|
||||
if is64 {
|
||||
if len(hdr) < 64 {
|
||||
return 0, 0
|
||||
}
|
||||
// e_shoff at 40, e_shentsize at 58, e_shnum at 60
|
||||
off := binary.LittleEndian.Uint64(hdr[40:48])
|
||||
entSz := uint64(binary.LittleEndian.Uint16(hdr[58:60]))
|
||||
num := uint64(binary.LittleEndian.Uint16(hdr[60:62]))
|
||||
if !little {
|
||||
off = binary.BigEndian.Uint64(hdr[40:48])
|
||||
entSz = uint64(binary.BigEndian.Uint16(hdr[58:60]))
|
||||
num = uint64(binary.BigEndian.Uint16(hdr[60:62]))
|
||||
}
|
||||
end = int64(off + entSz*num)
|
||||
} else {
|
||||
off := binary.LittleEndian.Uint32(hdr[32:36])
|
||||
entSz := uint32(binary.LittleEndian.Uint16(hdr[46:48]))
|
||||
num := uint32(binary.LittleEndian.Uint16(hdr[48:50]))
|
||||
if !little {
|
||||
off = binary.BigEndian.Uint32(hdr[32:36])
|
||||
entSz = uint32(binary.BigEndian.Uint16(hdr[46:48]))
|
||||
num = uint32(binary.BigEndian.Uint16(hdr[48:50]))
|
||||
}
|
||||
end = int64(off) + int64(entSz)*int64(num)
|
||||
}
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
func validatePE(d *document.Document, start, max int64) (int64, float64) {
|
||||
// e_lfanew at offset 0x3C points to PE header
|
||||
hdr, err := d.Range(start, 64)
|
||||
if err != nil || len(hdr) < 64 {
|
||||
return 0, 0
|
||||
}
|
||||
peOff := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40]))
|
||||
if peOff < 0x40 || start+peOff+4 > max {
|
||||
return 0, 0
|
||||
}
|
||||
sig, err := d.Range(start+peOff, 4)
|
||||
if err != nil || string(sig) != "PE\x00\x00" {
|
||||
return 0, 0.3
|
||||
}
|
||||
coff, err := d.Range(start+peOff+4, 20)
|
||||
if err != nil || len(coff) < 20 {
|
||||
return 0, 0.4
|
||||
}
|
||||
numSections := int(binary.LittleEndian.Uint16(coff[2:4]))
|
||||
sizeOpt := int64(binary.LittleEndian.Uint16(coff[16:18]))
|
||||
if numSections <= 0 || numSections > 96 || sizeOpt < 0 || sizeOpt > 4096 {
|
||||
return 0, 0.5
|
||||
}
|
||||
sectOff := start + peOff + 24 + sizeOpt
|
||||
end := peOff + 24 + sizeOpt + int64(numSections*40)
|
||||
for i := 0; i < numSections; i++ {
|
||||
sh, err := d.Range(sectOff+int64(i*40), 40)
|
||||
if err != nil || len(sh) < 40 {
|
||||
return 0, 0.5
|
||||
}
|
||||
rawSize := int64(binary.LittleEndian.Uint32(sh[16:20]))
|
||||
rawPtr := int64(binary.LittleEndian.Uint32(sh[20:24]))
|
||||
if rawSize < 0 || rawPtr < 0 {
|
||||
return 0, 0.5
|
||||
}
|
||||
if rawSize > 0 && rawPtr+rawSize > end {
|
||||
end = rawPtr + rawSize
|
||||
}
|
||||
}
|
||||
if end <= peOff || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
func validateGZIP(d *document.Document, start, max int64) (int64, float64) {
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil || len(buf) < 18 {
|
||||
return 0, 0
|
||||
}
|
||||
br := bytes.NewReader(buf)
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
zr.Multistream(false)
|
||||
if _, err := io.Copy(io.Discard, io.LimitReader(zr, 512<<20)); err != nil {
|
||||
_ = zr.Close()
|
||||
return 0, 0
|
||||
}
|
||||
if err := zr.Close(); err != nil {
|
||||
return 0, 0
|
||||
}
|
||||
consumed := int64(len(buf)) - int64(br.Len())
|
||||
if consumed <= 10 || start+consumed > max {
|
||||
return 0, 0
|
||||
}
|
||||
return consumed, 0.9
|
||||
}
|
||||
|
||||
// validateBzip2 locates the bit-aligned end-of-stream marker (0x177245385090,
|
||||
// the "sqrt(pi)" magic) and returns the stream length including the trailing
|
||||
// 32-bit combined CRC. bzip2 carries no length in its header, so this is the
|
||||
// only reliable way to bound the object.
|
||||
func validateBzip2(d *document.Document, start, max int64) (int64, float64) {
|
||||
if start+4 > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
hdr, err := d.Range(start, 4)
|
||||
if err != nil || len(hdr) < 4 || hdr[0] != 'B' || hdr[1] != 'Z' || hdr[2] != 'h' || hdr[3] < '1' || hdr[3] > '9' {
|
||||
return 0, 0
|
||||
}
|
||||
limit := min64(max-start, 64<<20)
|
||||
buf, err := d.Range(start, limit)
|
||||
if err != nil || len(buf) < 14 {
|
||||
return 0, 0.4
|
||||
}
|
||||
const eos uint64 = 0x177245385090
|
||||
const mask48 uint64 = (1 << 48) - 1
|
||||
var w uint64
|
||||
for i := 0; i < len(buf); i++ {
|
||||
w = (w << 8) | uint64(buf[i])
|
||||
// Check all eight bit alignments ending at or before this byte.
|
||||
for s := 0; s < 8; s++ {
|
||||
if (w>>uint(s))&mask48 == eos {
|
||||
endBit := (int64(i)+1)*8 - int64(s) + 32 // include stream CRC
|
||||
endByte := (endBit + 7) / 8
|
||||
if endByte > 14 && start+endByte <= max {
|
||||
return endByte, 0.85
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validate7z(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
nextHdrOff := binary.LittleEndian.Uint64(hdr[12:20])
|
||||
nextHdrSz := binary.LittleEndian.Uint64(hdr[20:28])
|
||||
end := int64(32 + nextHdrOff + nextHdrSz)
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.5
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
// validateRAR walks the block headers of a RAR4 or RAR5 archive to the
|
||||
// end-of-archive marker. The two formats differ structurally, so the version
|
||||
// is detected from the signature byte at offset 6.
|
||||
func validateRAR(d *document.Document, start, max int64) (int64, float64) {
|
||||
sig, err := d.Range(start, 8)
|
||||
if err != nil || len(sig) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
if sig[6] == 0x01 {
|
||||
return validateRAR5(d, start, max)
|
||||
}
|
||||
return validateRAR4(d, start, max)
|
||||
}
|
||||
|
||||
func validateRAR4(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start + 7 // 7-byte signature
|
||||
for n := 0; n < 100000; n++ {
|
||||
h, err := d.Range(pos, 11)
|
||||
if err != nil || len(h) < 7 {
|
||||
return 0, 0.4
|
||||
}
|
||||
flags := binary.LittleEndian.Uint16(h[3:5])
|
||||
headSize := int64(binary.LittleEndian.Uint16(h[5:7]))
|
||||
if headSize < 7 {
|
||||
return 0, 0.4
|
||||
}
|
||||
blockSize := headSize
|
||||
if flags&0x8000 != 0 { // ADD_SIZE present
|
||||
if len(h) < 11 {
|
||||
return 0, 0.4
|
||||
}
|
||||
blockSize += int64(binary.LittleEndian.Uint32(h[7:11]))
|
||||
}
|
||||
pos += blockSize
|
||||
if h[2] == 0x7b { // terminator block
|
||||
if pos > start && pos <= max {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validateRAR5(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start + 8 // 8-byte signature
|
||||
for n := 0; n < 100000; n++ {
|
||||
// Each block: CRC32(4) + vint(header_size) + header_size bytes [+ data].
|
||||
h, err := d.Range(pos, 4+11)
|
||||
if err != nil || len(h) < 5 {
|
||||
return 0, 0.4
|
||||
}
|
||||
headerSize, vlen := rar5Vint(h, 4)
|
||||
if vlen == 0 || headerSize == 0 {
|
||||
return 0, 0.4
|
||||
}
|
||||
hdrStart := pos + 4 + int64(vlen)
|
||||
body, err := d.Range(hdrStart, int64(headerSize))
|
||||
if err != nil || int64(len(body)) < int64(headerSize) {
|
||||
return 0, 0.4
|
||||
}
|
||||
htype, t1 := rar5Vint(body, 0)
|
||||
hflags, t2 := rar5Vint(body, t1)
|
||||
p := t1 + t2
|
||||
var dataSize uint64
|
||||
if hflags&0x0001 != 0 { // extra area present
|
||||
_, te := rar5Vint(body, p)
|
||||
p += te
|
||||
}
|
||||
if hflags&0x0002 != 0 { // data area present
|
||||
dataSize, _ = rar5Vint(body, p)
|
||||
}
|
||||
pos = hdrStart + int64(headerSize) + int64(dataSize)
|
||||
if htype == 5 { // end-of-archive header
|
||||
if pos > start && pos <= max {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
// rar5Vint decodes a RAR5 little-endian base-128 variable integer.
|
||||
func rar5Vint(buf []byte, pos int) (uint64, int) {
|
||||
var v uint64
|
||||
shift := uint(0)
|
||||
for n := 0; pos+n < len(buf) && n < 10; n++ {
|
||||
b := buf[pos+n]
|
||||
v |= uint64(b&0x7f) << shift
|
||||
if b&0x80 == 0 {
|
||||
return v, n + 1
|
||||
}
|
||||
shift += 7
|
||||
}
|
||||
return 0, 0
|
||||
}
|
||||
|
||||
func validateRIFF(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 12)
|
||||
if err != nil || len(hdr) < 12 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.LittleEndian.Uint32(hdr[4:8]))
|
||||
return size + 8, 0.9
|
||||
}
|
||||
|
||||
func validateMachO(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 28 {
|
||||
return 0, 0
|
||||
}
|
||||
var (
|
||||
bo binary.ByteOrder
|
||||
is64 bool
|
||||
hdrSize int64 = 28
|
||||
)
|
||||
switch binary.BigEndian.Uint32(hdr[0:4]) {
|
||||
case 0xfeedface:
|
||||
bo = binary.BigEndian
|
||||
case 0xfeedfacf:
|
||||
bo = binary.BigEndian
|
||||
is64 = true
|
||||
hdrSize = 32
|
||||
case 0xcefaedfe:
|
||||
bo = binary.LittleEndian
|
||||
case 0xcffaedfe:
|
||||
bo = binary.LittleEndian
|
||||
is64 = true
|
||||
hdrSize = 32
|
||||
default:
|
||||
return 0, 0
|
||||
}
|
||||
if is64 && len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
ncmds := bo.Uint32(hdr[16:20])
|
||||
sizeofcmds := bo.Uint32(hdr[20:24])
|
||||
if ncmds == 0 || ncmds > 10000 || sizeofcmds > 128<<20 {
|
||||
return 0, 0
|
||||
}
|
||||
cmdBytes, err := d.Range(start+hdrSize, int64(sizeofcmds))
|
||||
if err != nil || len(cmdBytes) < int(sizeofcmds) {
|
||||
return 0, 0
|
||||
}
|
||||
end := hdrSize + int64(sizeofcmds)
|
||||
off := 0
|
||||
for i := uint32(0); i < ncmds; i++ {
|
||||
if off+8 > len(cmdBytes) {
|
||||
return 0, 0
|
||||
}
|
||||
cmd := bo.Uint32(cmdBytes[off : off+4])
|
||||
cmdSize := int(bo.Uint32(cmdBytes[off+4 : off+8]))
|
||||
if cmdSize < 8 || off+cmdSize > len(cmdBytes) {
|
||||
return 0, 0
|
||||
}
|
||||
switch cmd {
|
||||
case 0x1: // LC_SEGMENT
|
||||
if cmdSize >= 56 {
|
||||
fileOff := int64(bo.Uint32(cmdBytes[off+32 : off+36]))
|
||||
fileSize := int64(bo.Uint32(cmdBytes[off+36 : off+40]))
|
||||
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
|
||||
end = fileOff + fileSize
|
||||
}
|
||||
}
|
||||
case 0x19: // LC_SEGMENT_64
|
||||
if cmdSize >= 72 {
|
||||
fileOff := int64(bo.Uint64(cmdBytes[off+40 : off+48]))
|
||||
fileSize := int64(bo.Uint64(cmdBytes[off+48 : off+56]))
|
||||
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
|
||||
end = fileOff + fileSize
|
||||
}
|
||||
}
|
||||
}
|
||||
off += cmdSize
|
||||
}
|
||||
if end <= hdrSize || start+end > max {
|
||||
return 0, 0
|
||||
}
|
||||
return end, 0.85
|
||||
}
|
||||
|
||||
func validateID3(d *document.Document, start, max int64) (int64, float64) {
|
||||
// ID3v2 header: "ID3" + ver(2) + flags(1) + size(4, synchsafe).
|
||||
hdr, err := d.Range(start, 10)
|
||||
if err != nil || len(hdr) < 10 {
|
||||
return 0, 0.3
|
||||
}
|
||||
if hdr[3] == 0xff || hdr[4] == 0xff {
|
||||
return 0, 0.3
|
||||
}
|
||||
// Synchsafe 28-bit size (7 bits per byte).
|
||||
sz := (int64(hdr[6]&0x7f) << 21) | (int64(hdr[7]&0x7f) << 14) |
|
||||
(int64(hdr[8]&0x7f) << 7) | int64(hdr[9]&0x7f)
|
||||
if sz < 0 || sz > 32*1024*1024 {
|
||||
return 0, 0.4
|
||||
}
|
||||
// Skip the ID3v2 tag (header + synchsafe body, plus a 10-byte footer if the
|
||||
// footer-present flag is set), then walk MPEG audio frames to the end.
|
||||
audioStart := start + 10 + sz
|
||||
if hdr[5]&0x10 != 0 {
|
||||
audioStart += 10
|
||||
}
|
||||
if end := walkMP3Frames(d, audioStart, max); end > audioStart {
|
||||
return end - start, 0.85
|
||||
}
|
||||
return 0, 0.7
|
||||
}
|
||||
|
||||
// walkMP3Frames follows MPEG-1/2/2.5 Layer I/II/III frame headers from start
|
||||
// until a byte that is not a valid frame sync, returning the offset just past
|
||||
// the last frame. APE/Lyrics3 trailers are not stripped — the audio extent is
|
||||
// what matters for carving.
|
||||
func walkMP3Frames(d *document.Document, start, max int64) int64 {
|
||||
pos := start
|
||||
frames := 0
|
||||
for pos+4 <= max && frames < 5_000_000 {
|
||||
h, err := d.Range(pos, 4)
|
||||
if err != nil || len(h) < 4 {
|
||||
break
|
||||
}
|
||||
ln := mp3FrameLen(h)
|
||||
if ln <= 0 {
|
||||
break
|
||||
}
|
||||
pos += ln
|
||||
frames++
|
||||
}
|
||||
if frames == 0 {
|
||||
return 0
|
||||
}
|
||||
if pos > max {
|
||||
return max
|
||||
}
|
||||
return pos
|
||||
}
|
||||
|
||||
var mp3BitrateV1 = [16]int{0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0}
|
||||
var mp3BitrateV2 = [16]int{0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0}
|
||||
var mp3SampleRate = map[int][3]int{ // [version]{rate index 0,1,2}
|
||||
3: {44100, 48000, 32000}, // MPEG1
|
||||
2: {22050, 24000, 16000}, // MPEG2
|
||||
0: {11025, 12000, 8000}, // MPEG2.5
|
||||
}
|
||||
|
||||
// mp3FrameLen returns the byte length of the MPEG audio frame whose 4-byte
|
||||
// header is h, or 0 if h is not a valid frame header.
|
||||
func mp3FrameLen(h []byte) int64 {
|
||||
if h[0] != 0xff || h[1]&0xe0 != 0xe0 {
|
||||
return 0
|
||||
}
|
||||
version := int(h[1] >> 3 & 0x03) // 3=MPEG1, 2=MPEG2, 0=MPEG2.5
|
||||
layer := int(h[1] >> 1 & 0x03) // 3=I, 2=II, 1=III
|
||||
if version == 1 || layer == 0 {
|
||||
return 0 // reserved
|
||||
}
|
||||
brIndex := int(h[2] >> 4 & 0x0f)
|
||||
srIndex := int(h[2] >> 2 & 0x03)
|
||||
padding := int(h[2] >> 1 & 0x01)
|
||||
if brIndex == 0 || brIndex == 15 || srIndex == 3 {
|
||||
return 0
|
||||
}
|
||||
var bitrate int
|
||||
if version == 3 {
|
||||
bitrate = mp3BitrateV1[brIndex]
|
||||
} else {
|
||||
bitrate = mp3BitrateV2[brIndex]
|
||||
}
|
||||
rates, ok := mp3SampleRate[version]
|
||||
if !ok || bitrate == 0 {
|
||||
return 0
|
||||
}
|
||||
sampleRate := rates[srIndex]
|
||||
if sampleRate == 0 {
|
||||
return 0
|
||||
}
|
||||
bps := bitrate * 1000
|
||||
if layer == 3 { // Layer I
|
||||
return int64(((12 * bps / sampleRate) + padding) * 4)
|
||||
}
|
||||
// Layer II & III. MPEG2/2.5 Layer III uses 72 samples-per-byte factor.
|
||||
coef := 144
|
||||
if version != 3 && layer == 1 {
|
||||
coef = 72
|
||||
}
|
||||
return int64(coef*bps/sampleRate + padding)
|
||||
}
|
||||
|
||||
func validateOgg(d *document.Document, start, max int64) (int64, float64) {
|
||||
off := start
|
||||
first := true
|
||||
for off+27 <= max {
|
||||
hdr, err := d.Range(off, 27)
|
||||
if err != nil || len(hdr) < 27 || string(hdr[:4]) != "OggS" || hdr[4] != 0 {
|
||||
return 0, 0
|
||||
}
|
||||
headerType := hdr[5]
|
||||
if first && headerType&0x02 == 0 {
|
||||
return 0, 0.4
|
||||
}
|
||||
segCount := int(hdr[26])
|
||||
lacing, err := d.Range(off+27, int64(segCount))
|
||||
if err != nil || len(lacing) < segCount {
|
||||
return 0, 0
|
||||
}
|
||||
payload := int64(0)
|
||||
for _, v := range lacing {
|
||||
payload += int64(v)
|
||||
}
|
||||
pageLen := int64(27 + segCount)
|
||||
if payload < 0 || off+pageLen+payload > max {
|
||||
return 0, 0
|
||||
}
|
||||
off += pageLen + payload
|
||||
if headerType&0x04 != 0 {
|
||||
return off - start, 0.9
|
||||
}
|
||||
first = false
|
||||
}
|
||||
return 0, 0.6
|
||||
}
|
||||
|
||||
// validateMatroska reads the EBML header element and the following Segment
|
||||
// element's size to bound a Matroska/WebM stream. A Segment with "unknown size"
|
||||
// (all size bits set, as live-streamed files use) cannot be bounded this way.
|
||||
func validateMatroska(d *document.Document, start, max int64) (int64, float64) {
|
||||
buf, err := d.Range(start, min64(max-start, 4096))
|
||||
if err != nil || len(buf) < 8 {
|
||||
return 0, 0
|
||||
}
|
||||
// EBML header: ID 0x1A45DFA3, then a vint size.
|
||||
if !(buf[0] == 0x1a && buf[1] == 0x45 && buf[2] == 0xdf && buf[3] == 0xa3) {
|
||||
return 0, 0
|
||||
}
|
||||
hsz, n, unknown := ebmlSize(buf, 4)
|
||||
if n == 0 || unknown {
|
||||
return 0, 0.6
|
||||
}
|
||||
pos := 4 + n + int(hsz) // start of the Segment element
|
||||
if pos+4 > len(buf) {
|
||||
return 0, 0.6
|
||||
}
|
||||
// Segment: ID 0x18538067, then a vint size.
|
||||
if !(buf[pos] == 0x18 && buf[pos+1] == 0x53 && buf[pos+2] == 0x80 && buf[pos+3] == 0x67) {
|
||||
return 0, 0.6
|
||||
}
|
||||
segSz, sn, segUnknown := ebmlSize(buf, pos+4)
|
||||
if sn == 0 || segUnknown {
|
||||
return 0, 0.6 // live/streamed segment without a known size
|
||||
}
|
||||
end := int64(pos+4+sn) + segSz
|
||||
if end <= 0 || start+end > max {
|
||||
return 0, 0.6
|
||||
}
|
||||
return end, 0.9
|
||||
}
|
||||
|
||||
// ebmlSize decodes an EBML variable-length integer (data size form). The length
|
||||
// is the number of leading zero bits in the first byte plus one; the value is
|
||||
// the remaining bits. unknown is true when every value bit is set.
|
||||
func ebmlSize(buf []byte, pos int) (val int64, n int, unknown bool) {
|
||||
if pos >= len(buf) || buf[pos] == 0 {
|
||||
return 0, 0, false
|
||||
}
|
||||
first := buf[pos]
|
||||
mask := byte(0x80)
|
||||
length := 1
|
||||
for first&mask == 0 {
|
||||
mask >>= 1
|
||||
length++
|
||||
}
|
||||
if length > 8 || pos+length > len(buf) {
|
||||
return 0, 0, false
|
||||
}
|
||||
low := mask - 1
|
||||
v := int64(first & low)
|
||||
allOnes := (first & low) == low
|
||||
for i := 1; i < length; i++ {
|
||||
b := buf[pos+i]
|
||||
v = (v << 8) | int64(b)
|
||||
if b != 0xff {
|
||||
allOnes = false
|
||||
}
|
||||
}
|
||||
return v, length, allOnes
|
||||
}
|
||||
|
||||
func validateFLAC(d *document.Document, start, max int64) (int64, float64) {
|
||||
return 0, 0.8
|
||||
}
|
||||
|
||||
// validateBMP reads the BITMAPFILEHEADER size field (bfSize), sanity-checking
|
||||
// the embedded pixel-data offset against the declared length.
|
||||
func validateBMP(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 14)
|
||||
if err != nil || len(hdr) < 14 {
|
||||
return 0, 0
|
||||
}
|
||||
size := int64(binary.LittleEndian.Uint32(hdr[2:6]))
|
||||
dataOff := int64(binary.LittleEndian.Uint32(hdr[10:14]))
|
||||
if size < 26 || dataOff < 26 || dataOff > size || start+size > max {
|
||||
return 0, 0
|
||||
}
|
||||
return size, 0.8
|
||||
}
|
||||
|
||||
// validateMP4 walks the top-level boxes of an ISO base-media file (MP4/MOV/M4A)
|
||||
// from the file start (the ftyp box). 64-bit box sizes are honoured.
|
||||
func validateMP4(d *document.Document, start, max int64) (int64, float64) {
|
||||
pos := start
|
||||
boxes := 0
|
||||
for pos+8 <= max && boxes < 100000 {
|
||||
h, err := d.Range(pos, 16)
|
||||
if err != nil || len(h) < 8 {
|
||||
break
|
||||
}
|
||||
size := int64(binary.BigEndian.Uint32(h[0:4]))
|
||||
switch {
|
||||
case size == 1: // 64-bit largesize follows the type
|
||||
if len(h) < 16 {
|
||||
return 0, 0.4
|
||||
}
|
||||
size = int64(binary.BigEndian.Uint64(h[8:16]))
|
||||
case size == 0: // box extends to end of file
|
||||
return max - start, 0.8
|
||||
}
|
||||
if size < 8 {
|
||||
break
|
||||
}
|
||||
pos += size
|
||||
boxes++
|
||||
if pos > max {
|
||||
return 0, 0.4
|
||||
}
|
||||
}
|
||||
if pos > start && pos <= max && boxes > 0 {
|
||||
return pos - start, 0.85
|
||||
}
|
||||
return 0, 0.4
|
||||
}
|
||||
|
||||
func validateSQLite(d *document.Document, start, max int64) (int64, float64) {
|
||||
hdr, err := d.Range(start, 32)
|
||||
if err != nil || len(hdr) < 32 {
|
||||
return 0, 0
|
||||
}
|
||||
pageSize := int64(binary.BigEndian.Uint16(hdr[16:18]))
|
||||
if pageSize == 1 {
|
||||
pageSize = 65536
|
||||
}
|
||||
pages := int64(binary.BigEndian.Uint32(hdr[28:32]))
|
||||
if pages == 0 || pageSize == 0 {
|
||||
return 0, 0.5
|
||||
}
|
||||
return pageSize * pages, 0.9
|
||||
}
|
||||
|
||||
func min64(a, b int64) int64 {
|
||||
if a < b {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
Reference in New Issue
Block a user