1082 lines
27 KiB
Go
1082 lines
27 KiB
Go
package binsnoop
|
|
|
|
import (
|
|
"encoding/binary"
|
|
"fmt"
|
|
)
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// LZMA
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const LZMADescription = "LZMA compressed data"
|
|
|
|
func LZMAMagic() [][]byte {
|
|
// Common LZMA properties
|
|
properties := []byte{0x5D, 0x6E, 0x6D, 0x6C}
|
|
|
|
dictionarySizes := []uint32{
|
|
0x10000000,
|
|
0x20000000,
|
|
0x01000000,
|
|
0x02000000,
|
|
0x04000000,
|
|
0x00800000,
|
|
0x00400000,
|
|
0x00200000,
|
|
0x00100000,
|
|
0x00080000,
|
|
0x00020000,
|
|
0x00010000,
|
|
}
|
|
|
|
// Build magic signatures from property+dictionary_size combinations (little-endian)
|
|
var sigs [][]byte
|
|
for _, prop := range properties {
|
|
for _, dict := range dictionarySizes {
|
|
magic := make([]byte, 5)
|
|
magic[0] = prop
|
|
binary.LittleEndian.PutUint32(magic[1:], dict)
|
|
sigs = append(sigs, magic)
|
|
}
|
|
}
|
|
return sigs
|
|
}
|
|
|
|
func LZMAParser(data []byte, offset int) (*SignatureResult, error) {
|
|
// Streamed data has a reported size of -1 (0xFFFFFFFFFFFFFFFF)
|
|
const lzmaStreamSize = -1
|
|
|
|
const minSupportedDecompressedSize = 256
|
|
const maxSupportedDecompressedSize = 0xFFFFFFFF
|
|
|
|
lzmaStructure := []StructField{
|
|
{"properties", "u8"},
|
|
{"dictionary_size", "u32"},
|
|
{"decompressed_size", "u64"},
|
|
{"null_byte", "u8"},
|
|
}
|
|
|
|
lzmaData := data[offset:]
|
|
if len(lzmaData) < StructSize(lzmaStructure) {
|
|
return nil, fmt.Errorf("LZMA: not enough data for header")
|
|
}
|
|
|
|
hdr, err := Parse(lzmaData, lzmaStructure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LZMA: header parse error: %w", err)
|
|
}
|
|
|
|
// Make sure the expected NULL byte is NULL
|
|
if hdr["null_byte"] != 0 {
|
|
return nil, fmt.Errorf("LZMA: null byte is not zero")
|
|
}
|
|
|
|
// Sanity check the reported decompressed size.
|
|
// In Go int (signed 64-bit), 0xFFFFFFFFFFFFFFFF becomes -1.
|
|
decompressedSize := hdr["decompressed_size"]
|
|
validSize := decompressedSize == lzmaStreamSize ||
|
|
(decompressedSize >= minSupportedDecompressedSize &&
|
|
decompressedSize <= maxSupportedDecompressedSize)
|
|
if !validSize {
|
|
return nil, fmt.Errorf("LZMA: invalid decompressed size: %d", decompressedSize)
|
|
}
|
|
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: fmt.Sprintf(
|
|
"%s, properties: 0x%02X, dictionary size: %d bytes, uncompressed size: %d bytes",
|
|
LZMADescription,
|
|
hdr["properties"],
|
|
hdr["dictionary_size"],
|
|
decompressedSize,
|
|
),
|
|
Confidence: ConfidenceHigh,
|
|
}, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// bzip2
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const Bzip2Description = "bzip2 compressed data"
|
|
|
|
func Bzip2Magic() [][]byte {
|
|
// Magic bytes include version, block size, and compressed magic signature
|
|
return [][]byte{
|
|
[]byte("BZh91AY&SY"),
|
|
[]byte("BZh81AY&SY"),
|
|
[]byte("BZh71AY&SY"),
|
|
[]byte("BZh61AY&SY"),
|
|
[]byte("BZh51AY&SY"),
|
|
[]byte("BZh41AY&SY"),
|
|
[]byte("BZh31AY&SY"),
|
|
[]byte("BZh21AY&SY"),
|
|
[]byte("BZh11AY&SY"),
|
|
}
|
|
}
|
|
|
|
func Bzip2Parser(data []byte, offset int) (*SignatureResult, error) {
|
|
// Validate we have enough data for at least the magic + some compressed data
|
|
if offset+10 >= len(data) {
|
|
return nil, fmt.Errorf("bzip2: not enough data")
|
|
}
|
|
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: Bzip2Description,
|
|
Confidence: ConfidenceHigh,
|
|
}, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// XZ
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const XZDescription = "XZ compressed data"
|
|
|
|
func XZMagic() [][]byte {
|
|
return [][]byte{
|
|
{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00},
|
|
}
|
|
}
|
|
|
|
func XZParser(data []byte, offset int) (*SignatureResult, error) {
|
|
const xzCRCStart = 6
|
|
const xzCRCEnd = 8
|
|
const xzHeaderSize = 12
|
|
|
|
xzStructure := []StructField{
|
|
{"magic_p1", "u32"},
|
|
{"magic_p2", "u16"},
|
|
{"flags", "u16"},
|
|
{"header_crc", "u32"},
|
|
}
|
|
|
|
xzData := data[offset:]
|
|
if len(xzData) < StructSize(xzStructure) {
|
|
return nil, fmt.Errorf("XZ: not enough data for header")
|
|
}
|
|
|
|
hdr, err := Parse(xzData, xzStructure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("XZ: header parse error: %w", err)
|
|
}
|
|
|
|
// Validate the header CRC over the flags bytes (bytes 6..8)
|
|
if xzCRCEnd > len(xzData) {
|
|
return nil, fmt.Errorf("XZ: not enough data for CRC")
|
|
}
|
|
|
|
crcData := xzData[xzCRCStart:xzCRCEnd]
|
|
calculatedCRC := CRC32(crcData)
|
|
if calculatedCRC != uint32(hdr["header_crc"]) {
|
|
return nil, fmt.Errorf("XZ: header CRC mismatch: expected 0x%08X, got 0x%08X",
|
|
hdr["header_crc"], calculatedCRC)
|
|
}
|
|
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: XZDescription,
|
|
Confidence: ConfidenceHigh,
|
|
Size: xzHeaderSize,
|
|
}, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// ZSTD
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const ZSTDDescription = "ZSTD compressed data"
|
|
|
|
func ZSTDMagic() [][]byte {
|
|
return [][]byte{
|
|
{0x28, 0xB5, 0x2F, 0xFD},
|
|
}
|
|
}
|
|
|
|
func ZSTDParser(data []byte, offset int) (*SignatureResult, error) {
|
|
const eofChecksumSize = 4
|
|
const minBlockCount = 2
|
|
|
|
availableData := len(data)
|
|
|
|
// Parse the ZSTD frame header
|
|
zstdHeader, err := parseZSTDHeader(data[offset:])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("ZSTD: %w", err)
|
|
}
|
|
|
|
// Calculate first block header offset accounting for optional fields
|
|
nextBlockStart := offset + zstdHeader.fixedHeaderSize
|
|
|
|
// If single segment flag is not set, window descriptor byte is present
|
|
if !zstdHeader.singleSegmentFlag {
|
|
nextBlockStart++
|
|
}
|
|
|
|
// Dictionary ID field size
|
|
if zstdHeader.dictionaryIDFlag == 1 {
|
|
nextBlockStart++
|
|
} else if zstdHeader.dictionaryIDFlag == 2 {
|
|
nextBlockStart += 2
|
|
} else if zstdHeader.dictionaryIDFlag == 3 {
|
|
nextBlockStart += 4
|
|
}
|
|
|
|
// Frame content header field size
|
|
if zstdHeader.frameContentFlag == 0 && zstdHeader.singleSegmentFlag {
|
|
nextBlockStart++
|
|
} else if zstdHeader.frameContentFlag == 1 {
|
|
nextBlockStart += 2
|
|
} else if zstdHeader.frameContentFlag == 2 {
|
|
nextBlockStart += 4
|
|
} else if zstdHeader.frameContentFlag == 3 {
|
|
nextBlockStart += 8
|
|
}
|
|
|
|
var previousBlockStart *int
|
|
blockCount := 0
|
|
totalSize := 0
|
|
|
|
// Walk through blocks to determine end of ZSTD data
|
|
for IsOffsetSafe(availableData, nextBlockStart, previousBlockStart) {
|
|
blockHeader, err := parseZSTDBlockHeader(data[nextBlockStart:])
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
blockCount++
|
|
prev := nextBlockStart
|
|
previousBlockStart = &prev
|
|
nextBlockStart += blockHeader.headerSize + blockHeader.blockSize
|
|
|
|
if blockHeader.lastBlock {
|
|
totalSize = nextBlockStart - offset
|
|
if zstdHeader.contentChecksumPresent {
|
|
totalSize += eofChecksumSize
|
|
}
|
|
if blockCount >= minBlockCount {
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: fmt.Sprintf("%s, total size: %d bytes", ZSTDDescription, totalSize),
|
|
Confidence: ConfidenceHigh,
|
|
Size: totalSize,
|
|
}, nil
|
|
}
|
|
break
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("ZSTD: failed to validate block structure")
|
|
}
|
|
|
|
type zstdHeaderInfo struct {
|
|
fixedHeaderSize int
|
|
dictionaryIDFlag int
|
|
contentChecksumPresent bool
|
|
singleSegmentFlag bool
|
|
frameContentFlag int
|
|
}
|
|
|
|
func parseZSTDHeader(zstdData []byte) (*zstdHeaderInfo, error) {
|
|
const frameUnusedBitsMask = 0b00011000
|
|
const dictionaryIDMask = 0b11
|
|
const contentChecksumMask = 0b100
|
|
const singleSegmentMask = 0b100000
|
|
const frameContentMask = 0b11000000
|
|
const frameContentShift = 6
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"frame_header_descriptor", "u8"},
|
|
}
|
|
|
|
hdr, err := Parse(zstdData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("header parse error: %w", err)
|
|
}
|
|
|
|
// Unused bits should be zero
|
|
if (hdr["frame_header_descriptor"] & frameUnusedBitsMask) != 0 {
|
|
return nil, fmt.Errorf("unused bits are set in frame header descriptor")
|
|
}
|
|
|
|
return &zstdHeaderInfo{
|
|
fixedHeaderSize: StructSize(structure),
|
|
dictionaryIDFlag: hdr["frame_header_descriptor"] & dictionaryIDMask,
|
|
contentChecksumPresent: (hdr["frame_header_descriptor"] & contentChecksumMask) != 0,
|
|
singleSegmentFlag: (hdr["frame_header_descriptor"] & singleSegmentMask) != 0,
|
|
frameContentFlag: (hdr["frame_header_descriptor"] & frameContentMask) >> frameContentShift,
|
|
}, nil
|
|
}
|
|
|
|
type zstdBlockHeaderInfo struct {
|
|
headerSize int
|
|
blockType int
|
|
blockSize int
|
|
lastBlock bool
|
|
}
|
|
|
|
func parseZSTDBlockHeader(blockData []byte) (*zstdBlockHeaderInfo, error) {
|
|
const blockTypeMask = 0b110
|
|
const blockTypeShift = 1
|
|
const rleBlockType = 1
|
|
const reservedBlockType = 3
|
|
const lastBlockMask = 0b1
|
|
const blockSizeMask = 0b1111_1111_1111_1111_1111_1000
|
|
const blockSizeShift = 3
|
|
|
|
structure := []StructField{{"info_bits", "u24"}}
|
|
|
|
hdr, err := Parse(blockData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("block header parse error: %w", err)
|
|
}
|
|
|
|
info := &zstdBlockHeaderInfo{
|
|
headerSize: StructSize(structure),
|
|
lastBlock: (hdr["info_bits"] & lastBlockMask) != 0,
|
|
blockType: (hdr["info_bits"] & blockTypeMask) >> blockTypeShift,
|
|
blockSize: (hdr["info_bits"] & blockSizeMask) >> blockSizeShift,
|
|
}
|
|
|
|
// RLE block: single byte of raw data
|
|
if info.blockType == rleBlockType {
|
|
info.blockSize = 1
|
|
}
|
|
|
|
// Reserved block type is invalid
|
|
if info.blockType == reservedBlockType {
|
|
return nil, fmt.Errorf("reserved block type encountered")
|
|
}
|
|
|
|
return info, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// LZ4
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const LZ4Description = "LZ4 compressed data"
|
|
|
|
func LZ4Magic() [][]byte {
|
|
return [][]byte{
|
|
{0x04, 0x22, 0x4D, 0x18},
|
|
}
|
|
}
|
|
|
|
func LZ4Parser(data []byte, offset int) (*SignatureResult, error) {
|
|
const contentChecksumLen = 4
|
|
|
|
lz4Header, err := parseLZ4FileHeader(data[offset:])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LZ4: %w", err)
|
|
}
|
|
|
|
if offset+lz4Header.headerSize >= len(data) {
|
|
return nil, fmt.Errorf("LZ4: not enough data after header")
|
|
}
|
|
|
|
lz4Data := data[offset+lz4Header.headerSize:]
|
|
dataSize, err := getLZ4DataSize(lz4Data, lz4Header.blockChecksumPresent)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LZ4: %w", err)
|
|
}
|
|
|
|
totalSize := lz4Header.headerSize + dataSize
|
|
if lz4Header.contentChecksumPresent {
|
|
totalSize += contentChecksumLen
|
|
}
|
|
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: fmt.Sprintf("%s, total size: %d bytes", LZ4Description, totalSize),
|
|
Confidence: ConfidenceMedium,
|
|
Size: totalSize,
|
|
}, nil
|
|
}
|
|
|
|
type lz4FileHeaderInfo struct {
|
|
headerSize int
|
|
blockChecksumPresent bool
|
|
contentChecksumPresent bool
|
|
}
|
|
|
|
func parseLZ4FileHeader(lz4Data []byte) (*lz4FileHeaderInfo, error) {
|
|
const magicSize = 4
|
|
const lz4StructSize = 6
|
|
|
|
const bdReservedMask = 0b10001111
|
|
const flagsReservedMask = 0b00000010
|
|
|
|
const flagDictionaryPresent = 0b00000001
|
|
const flagContentSizePresent = 0b00001000
|
|
const flagBlockChecksumPresent = 0b00010000
|
|
const flagContentChecksumPresent = 0b00000100
|
|
|
|
const dictionaryLen = 4
|
|
const contentSizeLen = 8
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"flags", "u8"},
|
|
{"bd", "u8"},
|
|
}
|
|
|
|
hdr, err := Parse(lz4Data, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("file header parse error: %w", err)
|
|
}
|
|
|
|
// Reserved bits must not be set
|
|
if (hdr["flags"] & flagsReservedMask) != 0 {
|
|
return nil, fmt.Errorf("flags reserved bits are set")
|
|
}
|
|
if (hdr["bd"] & bdReservedMask) != 0 {
|
|
return nil, fmt.Errorf("bd reserved bits are set")
|
|
}
|
|
|
|
// Calculate CRC data boundaries (CRC covers descriptor excluding magic)
|
|
crcDataStart := magicSize
|
|
crcDataEnd := crcDataStart + (lz4StructSize - magicSize)
|
|
|
|
if (hdr["flags"] & flagContentSizePresent) != 0 {
|
|
crcDataEnd += contentSizeLen
|
|
}
|
|
if (hdr["flags"] & flagDictionaryPresent) != 0 {
|
|
crcDataEnd += dictionaryLen
|
|
}
|
|
|
|
if crcDataEnd >= len(lz4Data) {
|
|
return nil, fmt.Errorf("not enough data for header CRC")
|
|
}
|
|
|
|
crcData := lz4Data[crcDataStart:crcDataEnd]
|
|
actualCRC := lz4Data[crcDataEnd]
|
|
|
|
// LZ4 header CRC is the second byte of xxh32 hash over the descriptor
|
|
calculatedCRC := byte((xxh32(crcData, 0) >> 8) & 0xFF)
|
|
if actualCRC != calculatedCRC {
|
|
return nil, fmt.Errorf("header CRC mismatch: expected 0x%02X, got 0x%02X",
|
|
actualCRC, calculatedCRC)
|
|
}
|
|
|
|
return &lz4FileHeaderInfo{
|
|
headerSize: crcDataEnd + 1,
|
|
blockChecksumPresent: (hdr["flags"] & flagBlockChecksumPresent) != 0,
|
|
contentChecksumPresent: (hdr["flags"] & flagContentChecksumPresent) != 0,
|
|
}, nil
|
|
}
|
|
|
|
func getLZ4DataSize(lz4Data []byte, checksumPresent bool) (int, error) {
|
|
dataSize := 0
|
|
var lastOffset *int
|
|
available := len(lz4Data)
|
|
|
|
for IsOffsetSafe(available, dataSize, lastOffset) {
|
|
if dataSize >= len(lz4Data) {
|
|
break
|
|
}
|
|
|
|
blockHeader, err := parseLZ4BlockHeader(lz4Data[dataSize:], checksumPresent)
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
prev := dataSize
|
|
lastOffset = &prev
|
|
dataSize += blockHeader.headerSize + blockHeader.dataSize + blockHeader.checksumSize
|
|
|
|
if blockHeader.lastBlock {
|
|
return dataSize, nil
|
|
}
|
|
}
|
|
|
|
return 0, fmt.Errorf("failed to find LZ4 end marker")
|
|
}
|
|
|
|
type lz4BlockHeaderInfo struct {
|
|
dataSize int
|
|
headerSize int
|
|
checksumSize int
|
|
lastBlock bool
|
|
}
|
|
|
|
func parseLZ4BlockHeader(blockData []byte, checksumPresent bool) (*lz4BlockHeaderInfo, error) {
|
|
const sizeMask uint32 = 0x7FFFFFFF
|
|
const endMarker = 0
|
|
const checksumSize = 4
|
|
const blockStructSize = 4
|
|
|
|
structure := []StructField{{"block_size", "u32"}}
|
|
|
|
hdr, err := Parse(blockData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("block header parse error: %w", err)
|
|
}
|
|
|
|
info := &lz4BlockHeaderInfo{
|
|
headerSize: blockStructSize,
|
|
lastBlock: hdr["block_size"] == endMarker,
|
|
}
|
|
|
|
if checksumPresent {
|
|
info.checksumSize = checksumSize
|
|
}
|
|
|
|
// High bit is not part of the actual block size
|
|
info.dataSize = int(uint32(hdr["block_size"]) & sizeMask)
|
|
|
|
return info, nil
|
|
}
|
|
|
|
// xxh32 implements the XXHash32 algorithm used by LZ4 for header checksums.
|
|
func xxh32(input []byte, seed uint32) uint32 {
|
|
const (
|
|
prime1 uint32 = 0x9E3779B1
|
|
prime2 uint32 = 0x85EBCA77
|
|
prime3 uint32 = 0xC2B2AE3D
|
|
prime4 uint32 = 0x27D4EB2F
|
|
prime5 uint32 = 0x165667B1
|
|
)
|
|
|
|
n := len(input)
|
|
p := 0
|
|
var h32 uint32
|
|
|
|
if n >= 16 {
|
|
v1 := seed + prime1 + prime2
|
|
v2 := seed + prime2
|
|
v3 := seed
|
|
v4 := seed - prime1
|
|
for p+16 <= n {
|
|
v1 = xxh32Round(v1, binary.LittleEndian.Uint32(input[p:]), prime1, prime2)
|
|
v2 = xxh32Round(v2, binary.LittleEndian.Uint32(input[p+4:]), prime1, prime2)
|
|
v3 = xxh32Round(v3, binary.LittleEndian.Uint32(input[p+8:]), prime1, prime2)
|
|
v4 = xxh32Round(v4, binary.LittleEndian.Uint32(input[p+12:]), prime1, prime2)
|
|
p += 16
|
|
}
|
|
h32 = ((v1 << 1) | (v1 >> 31)) +
|
|
((v2 << 7) | (v2 >> 25)) +
|
|
((v3 << 12) | (v3 >> 20)) +
|
|
((v4 << 18) | (v4 >> 14))
|
|
} else {
|
|
h32 = seed + prime5
|
|
}
|
|
|
|
h32 += uint32(n)
|
|
|
|
for p+4 <= n {
|
|
h32 += binary.LittleEndian.Uint32(input[p:]) * prime3
|
|
h32 = ((h32 << 17) | (h32 >> 15)) * prime4
|
|
p += 4
|
|
}
|
|
|
|
for p < n {
|
|
h32 += uint32(input[p]) * prime5
|
|
h32 = ((h32 << 11) | (h32 >> 21)) * prime1
|
|
p++
|
|
}
|
|
|
|
h32 ^= h32 >> 15
|
|
h32 *= prime2
|
|
h32 ^= h32 >> 13
|
|
h32 *= prime3
|
|
h32 ^= h32 >> 16
|
|
|
|
return h32
|
|
}
|
|
|
|
func xxh32Round(acc, input, prime1, prime2 uint32) uint32 {
|
|
acc += input * prime2
|
|
acc = (acc << 13) | (acc >> 19)
|
|
acc *= prime1
|
|
return acc
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// LZOP
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const LZOPDescription = "LZO compressed data"
|
|
|
|
func LZOPMagic() [][]byte {
|
|
return [][]byte{
|
|
{0x89, 0x4C, 0x5A, 0x4F, 0x00, 0x0D, 0x0A, 0x1A, 0x0A},
|
|
}
|
|
}
|
|
|
|
func LZOPParser(data []byte, offset int) (*SignatureResult, error) {
|
|
lzopHeader, err := parseLZOPFileHeader(data[offset:])
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LZOP: %w", err)
|
|
}
|
|
|
|
if offset+lzopHeader.headerSize >= len(data) {
|
|
return nil, fmt.Errorf("LZOP: not enough data after header")
|
|
}
|
|
|
|
lzopData := data[offset+lzopHeader.headerSize:]
|
|
dataSize, err := getLZODataSize(lzopData, lzopHeader.blockChecksumPresent)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("LZOP: %w", err)
|
|
}
|
|
|
|
totalSize := lzopHeader.headerSize + dataSize
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: fmt.Sprintf("%s, total size: %d bytes", LZOPDescription, totalSize),
|
|
Confidence: ConfidenceHigh,
|
|
Size: totalSize,
|
|
}, nil
|
|
}
|
|
|
|
type lzopFileHeaderInfo struct {
|
|
headerSize int
|
|
blockChecksumPresent bool
|
|
}
|
|
|
|
func parseLZOPFileHeader(lzopData []byte) (*lzopFileHeaderInfo, error) {
|
|
const lzoMaxVersion = 0x1040
|
|
const lzoHeaderSizeP1 = 21
|
|
const lzoHeaderSizeP2 = 13
|
|
const filterSize = 4
|
|
const lzoChecksumSize = 4
|
|
|
|
const flagFilter = 0x0000_0800
|
|
const flagCRC32C = 0x0000_0200
|
|
const flagAdler32C = 0x0000_0002
|
|
|
|
structureP1 := []StructField{
|
|
{"magic_p1", "u8"},
|
|
{"magic_p2", "u64"},
|
|
{"version", "u16"},
|
|
{"lib_version", "u16"},
|
|
{"version_needed", "u16"},
|
|
{"method", "u8"},
|
|
{"level", "u8"},
|
|
{"flags", "u32"},
|
|
}
|
|
|
|
structureP2 := []StructField{
|
|
{"mode", "u32"},
|
|
{"mtime", "u32"},
|
|
{"gmt_diff", "u32"},
|
|
{"file_name_length", "u8"},
|
|
}
|
|
|
|
allowedMethods := []int{1, 2, 3}
|
|
|
|
hdrP1, err := Parse(lzopData, structureP1, "big")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("header part 1 parse error: %w", err)
|
|
}
|
|
|
|
// Sanity check methods
|
|
methodValid := false
|
|
for _, m := range allowedMethods {
|
|
if hdrP1["method"] == m {
|
|
methodValid = true
|
|
break
|
|
}
|
|
}
|
|
if !methodValid {
|
|
return nil, fmt.Errorf("invalid method: %d", hdrP1["method"])
|
|
}
|
|
|
|
// Sanity check version numbers
|
|
if hdrP1["version"] > lzoMaxVersion {
|
|
return nil, fmt.Errorf("version too high: 0x%04X", hdrP1["version"])
|
|
}
|
|
if hdrP1["version"] < hdrP1["version_needed"] {
|
|
return nil, fmt.Errorf("version %d < version_needed %d", hdrP1["version"], hdrP1["version_needed"])
|
|
}
|
|
|
|
headerP2Start := lzoHeaderSizeP1
|
|
if (hdrP1["flags"] & flagFilter) != 0 {
|
|
headerP2Start += filterSize
|
|
}
|
|
|
|
headerP2End := headerP2Start + lzoHeaderSizeP2
|
|
if headerP2End > len(lzopData) {
|
|
return nil, fmt.Errorf("not enough data for header part 2")
|
|
}
|
|
|
|
hdrP2, err := Parse(lzopData[headerP2Start:headerP2End], structureP2, "big")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("header part 2 parse error: %w", err)
|
|
}
|
|
|
|
headerSize := headerP2End + hdrP2["file_name_length"] + lzoChecksumSize
|
|
|
|
// Faithfully ported: in the Rust source this is & (AND) which always yields false
|
|
blockChecksumPresent := (hdrP1["flags"] & flagAdler32C & flagCRC32C) != 0
|
|
|
|
if headerSize > len(lzopData) {
|
|
return nil, fmt.Errorf("calculated header size exceeds available data")
|
|
}
|
|
|
|
return &lzopFileHeaderInfo{
|
|
headerSize: headerSize,
|
|
blockChecksumPresent: blockChecksumPresent,
|
|
}, nil
|
|
}
|
|
|
|
type lzopBlockHeaderInfo struct {
|
|
headerSize int
|
|
compressedSize int
|
|
uncompressedSize int
|
|
checksumSize int
|
|
}
|
|
|
|
func parseLZOPBlockHeader(lzoData []byte, compressedChecksumPresent bool) (*lzopBlockHeaderInfo, error) {
|
|
const blockHeaderSize = 12
|
|
const maxUncompressedBlockSize = 64 * 1024 * 1024
|
|
const lzoChecksumSize = 4
|
|
|
|
structure := []StructField{
|
|
{"uncompressed_size", "u32"},
|
|
{"compressed_size", "u32"},
|
|
{"uncompressed_checksum", "u32"},
|
|
}
|
|
|
|
hdr, err := Parse(lzoData, structure, "big")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("block header parse error: %w", err)
|
|
}
|
|
|
|
if hdr["compressed_size"] == 0 ||
|
|
hdr["uncompressed_size"] == 0 ||
|
|
hdr["uncompressed_checksum"] == 0 ||
|
|
hdr["uncompressed_size"] > maxUncompressedBlockSize {
|
|
return nil, fmt.Errorf("invalid block header values")
|
|
}
|
|
|
|
info := &lzopBlockHeaderInfo{
|
|
headerSize: blockHeaderSize,
|
|
compressedSize: hdr["compressed_size"],
|
|
uncompressedSize: hdr["uncompressed_size"],
|
|
}
|
|
|
|
if compressedChecksumPresent {
|
|
info.checksumSize = lzoChecksumSize
|
|
}
|
|
|
|
return info, nil
|
|
}
|
|
|
|
func parseLZOPEOFMarker(eofData []byte) (int, error) {
|
|
const eofMarkerValue = 0
|
|
const eofMarkerSize = 4
|
|
|
|
structure := []StructField{{"marker", "u32"}}
|
|
|
|
hdr, err := Parse(eofData, structure, "big")
|
|
if err != nil {
|
|
return 0, fmt.Errorf("EOF marker parse error: %w", err)
|
|
}
|
|
|
|
if hdr["marker"] != eofMarkerValue {
|
|
return 0, fmt.Errorf("invalid EOF marker: %d", hdr["marker"])
|
|
}
|
|
|
|
return eofMarkerSize, nil
|
|
}
|
|
|
|
func getLZODataSize(lzoData []byte, compressedChecksumPresent bool) (int, error) {
|
|
const minBlockCount = 2
|
|
|
|
available := len(lzoData)
|
|
var lastOffset *int
|
|
dataSize := 0
|
|
blockCount := 0
|
|
|
|
for IsOffsetSafe(available, dataSize, lastOffset) {
|
|
blockHeader, err := parseLZOPBlockHeader(lzoData[dataSize:], compressedChecksumPresent)
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
blockCount++
|
|
prev := dataSize
|
|
lastOffset = &prev
|
|
dataSize += blockHeader.headerSize + blockHeader.compressedSize + blockHeader.checksumSize
|
|
}
|
|
|
|
if blockCount >= minBlockCount {
|
|
if dataSize < len(lzoData) {
|
|
eofSize, err := parseLZOPEOFMarker(lzoData[dataSize:])
|
|
if err != nil {
|
|
return 0, fmt.Errorf("EOF marker error: %w", err)
|
|
}
|
|
dataSize += eofSize
|
|
return dataSize, nil
|
|
}
|
|
}
|
|
|
|
return 0, fmt.Errorf("failed to validate LZO block structure")
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// LZFSE
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const LZFSEDescription = "LZFSE compressed data"
|
|
|
|
func LZFSEMagic() [][]byte {
|
|
return [][]byte{
|
|
[]byte("bvx-"),
|
|
[]byte("bvx1"),
|
|
[]byte("bvx2"),
|
|
[]byte("bvxn"),
|
|
}
|
|
}
|
|
|
|
func LZFSEParser(data []byte, offset int) (*SignatureResult, error) {
|
|
available := len(data)
|
|
nextBlockOffset := offset
|
|
var previousBlockOffset *int
|
|
|
|
for IsOffsetSafe(available, nextBlockOffset, previousBlockOffset) {
|
|
prev := nextBlockOffset
|
|
previousBlockOffset = &prev
|
|
|
|
block, err := parseLZFSEBlockHeader(data[nextBlockOffset:])
|
|
if err != nil {
|
|
break
|
|
}
|
|
|
|
nextBlockOffset += block.headerSize + block.dataSize
|
|
|
|
if block.eof {
|
|
totalSize := nextBlockOffset - offset
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: fmt.Sprintf("%s, total size: %d bytes", LZFSEDescription, totalSize),
|
|
Confidence: ConfidenceHigh,
|
|
Size: totalSize,
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
return nil, fmt.Errorf("LZFSE: failed to find end-of-stream block")
|
|
}
|
|
|
|
type lzfseBlockInfo struct {
|
|
eof bool
|
|
dataSize int
|
|
headerSize int
|
|
}
|
|
|
|
func parseLZFSEBlockHeader(lzfseData []byte) (*lzfseBlockInfo, error) {
|
|
// LZFSE block types (little-endian u32 representations)
|
|
const endOfStream = 0x24787662 // "bvx$"
|
|
const uncompressed = 0x2D787662 // "bvx-"
|
|
const compressedV1 = 0x31787662 // "bvx1"
|
|
const compressedV2 = 0x32787662 // "bvx2"
|
|
const compressedLZVN = 0x6E787662 // "bvxn"
|
|
|
|
blockTypeStructure := []StructField{{"block_type", "u32"}}
|
|
|
|
hdr, err := Parse(lzfseData, blockTypeStructure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("block header parse error: %w", err)
|
|
}
|
|
|
|
blockType := hdr["block_type"]
|
|
|
|
switch blockType {
|
|
case endOfStream:
|
|
return &lzfseBlockInfo{eof: true, dataSize: 0, headerSize: 4}, nil
|
|
|
|
case uncompressed:
|
|
return parseLZFSEUncompressedBlock(lzfseData)
|
|
|
|
case compressedV1:
|
|
return parseLZFSECompressedV1Block(lzfseData)
|
|
|
|
case compressedV2:
|
|
return parseLZFSECompressedV2Block(lzfseData)
|
|
|
|
case compressedLZVN:
|
|
return parseLZFSECompressedLZVNBlock(lzfseData)
|
|
}
|
|
|
|
return nil, fmt.Errorf("unknown LZFSE block type: 0x%08X", blockType)
|
|
}
|
|
|
|
func parseLZFSEUncompressedBlock(lzfseData []byte) (*lzfseBlockInfo, error) {
|
|
const headerSize = 8
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"n_raw_bytes", "u32"},
|
|
}
|
|
|
|
hdr, err := Parse(lzfseData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("uncompressed block parse error: %w", err)
|
|
}
|
|
|
|
return &lzfseBlockInfo{
|
|
eof: false,
|
|
dataSize: hdr["n_raw_bytes"],
|
|
headerSize: headerSize,
|
|
}, nil
|
|
}
|
|
|
|
func parseLZFSECompressedV1Block(lzfseData []byte) (*lzfseBlockInfo, error) {
|
|
const headerSize = 770
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"n_raw_bytes", "u32"},
|
|
{"n_payload_bytes", "u32"},
|
|
{"n_literals", "u32"},
|
|
{"n_matches", "u32"},
|
|
{"n_literal_payload_bytes", "u32"},
|
|
{"n_lmd_payload_bytes", "u32"},
|
|
{"literal_bits", "u32"},
|
|
{"literal_state", "u64"},
|
|
{"lmd_bits", "u32"},
|
|
{"l_state", "u16"},
|
|
{"m_state", "u16"},
|
|
{"d_state", "u16"},
|
|
}
|
|
|
|
hdr, err := Parse(lzfseData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("compressed v1 block parse error: %w", err)
|
|
}
|
|
|
|
return &lzfseBlockInfo{
|
|
eof: false,
|
|
dataSize: hdr["n_literal_payload_bytes"] + hdr["n_lmd_payload_bytes"],
|
|
headerSize: headerSize,
|
|
}, nil
|
|
}
|
|
|
|
func parseLZFSECompressedV2Block(lzfseData []byte) (*lzfseBlockInfo, error) {
|
|
const nPayloadShift = 20
|
|
const lmdPayloadShift = 40
|
|
const payloadMask = 0b11111_11111_11111_11111
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"uncompressed_size", "u32"},
|
|
{"packed_field_1", "u64"},
|
|
{"packed_field_2", "u64"},
|
|
{"header_size", "u32"},
|
|
{"state_fields", "u32"},
|
|
}
|
|
|
|
hdr, err := Parse(lzfseData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("compressed v2 block parse error: %w", err)
|
|
}
|
|
|
|
nLMDPayloadBytes := (hdr["packed_field_2"] >> lmdPayloadShift) & payloadMask
|
|
nLiteralPayloadBytes := (hdr["packed_field_1"] >> nPayloadShift) & payloadMask
|
|
|
|
return &lzfseBlockInfo{
|
|
eof: false,
|
|
dataSize: nLMDPayloadBytes + nLiteralPayloadBytes,
|
|
headerSize: hdr["header_size"],
|
|
}, nil
|
|
}
|
|
|
|
func parseLZFSECompressedLZVNBlock(lzfseData []byte) (*lzfseBlockInfo, error) {
|
|
const headerSize = 12
|
|
|
|
structure := []StructField{
|
|
{"magic", "u32"},
|
|
{"n_raw_bytes", "u32"},
|
|
{"n_payload_bytes", "u32"},
|
|
}
|
|
|
|
hdr, err := Parse(lzfseData, structure, "little")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("compressed LZVN block parse error: %w", err)
|
|
}
|
|
|
|
return &lzfseBlockInfo{
|
|
eof: false,
|
|
dataSize: hdr["n_payload_bytes"],
|
|
headerSize: headerSize,
|
|
}, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Zlib
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const ZlibDescription = "Zlib compressed file"
|
|
|
|
func ZlibMagic() [][]byte {
|
|
return [][]byte{
|
|
{0x78, 0x9C},
|
|
{0x78, 0xDA},
|
|
{0x78, 0x5E},
|
|
}
|
|
}
|
|
|
|
func ZlibParser(data []byte, offset int) (*SignatureResult, error) {
|
|
if offset+2 > len(data) {
|
|
return nil, fmt.Errorf("zlib: not enough data")
|
|
}
|
|
|
|
// Zlib header: CMF (byte 0) and FLG (byte 1)
|
|
// The two-byte value (CMF*256 + FLG) must be a multiple of 31 (FCHECK)
|
|
cmf := int(data[offset])
|
|
flg := int(data[offset+1])
|
|
if (cmf*256+flg)%31 != 0 {
|
|
return nil, fmt.Errorf("zlib: FCHECK validation failed")
|
|
}
|
|
|
|
// CM (compression method) should be 8 (deflate)
|
|
cm := cmf & 0x0F
|
|
if cm != 8 {
|
|
return nil, fmt.Errorf("zlib: invalid compression method: %d", cm)
|
|
}
|
|
|
|
// CINFO should be <= 7
|
|
cinfo := (cmf >> 4) & 0x0F
|
|
if cinfo > 7 {
|
|
return nil, fmt.Errorf("zlib: invalid CINFO: %d", cinfo)
|
|
}
|
|
|
|
return &SignatureResult{
|
|
Offset: offset,
|
|
Description: ZlibDescription,
|
|
Confidence: ConfidenceHigh,
|
|
}, nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// compress'd
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const CompressdDescription = "compress'd data"
|
|
|
|
func CompressdMagic() [][]byte {
|
|
return [][]byte{
|
|
{0x1F, 0x9D, 0x90},
|
|
}
|
|
}
|
|
|
|
func CompressdParser(data []byte, offset int) (*SignatureResult, error) {
|
|
result := &SignatureResult{
|
|
Offset: offset,
|
|
Description: CompressdDescription,
|
|
Confidence: ConfidenceLow,
|
|
}
|
|
|
|
// Higher confidence when found at the beginning of a file
|
|
if offset == 0 {
|
|
result.Confidence = ConfidenceMedium
|
|
}
|
|
|
|
return result, nil
|
|
}
|