andy/nexus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

v3 candidate

Browse Source
This commit is contained in:
Andy Dixon
2026-06-16 11:51:07 +01:00
parent fcb19c96c8
commit 493bfed599
91 changed files with 28824 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+20 -2
View File
@@ -48,6 +48,7 @@ const (
maxInlineSFTPUploadBytes = 25 * 1024 * 1024
maxSFTPUploadChunkBytes = 1024 * 1024
maxStreamingSFTPUploadSize = int64(2 * 1024 * 1024 * 1024)
maxSSHPublicKeyBytes = 64 * 1024
)
// Service is the main API surface exposed to the frontend via Wails bindings.
@@ -66,6 +67,8 @@ type Service struct {
uploadMu sync.Mutex
uploads map[string]*sftpUpload
leviathan *leviathanState
}
type sftpUpload struct {
@@ -93,6 +96,7 @@ func NewService(
auditLog: auditLog,
prefs: prefs,
uploads: make(map[string]*sftpUpload),
leviathan: &leviathanState{},
}
}
@@ -1763,7 +1767,7 @@ func (s *Service) ListSSHKeyPairs() ([]SSHKeyInfo, error) {
seen[baseName] = true
// Read public key content
pubBytes, err := os.ReadFile(filepath.Join(sshDir, name))
pubBytes, err := readSSHPublicKeyFile(filepath.Join(sshDir, name))
if err != nil {
continue
}
@@ -1849,7 +1853,7 @@ func (s *Service) GetSSHPublicKey(name string) (string, error) {
if err != nil {
return "", err
}
data, err := os.ReadFile(pubPath)
data, err := readSSHPublicKeyFile(pubPath)
if err != nil {
return "", fmt.Errorf("unable to read public key: %w", err)
}
@@ -1891,6 +1895,20 @@ func sshKeyPairPaths(name string) (string, string, error) {
return privPath, pubPath, nil
}
func readSSHPublicKeyFile(path string) ([]byte, error) {
info, err := os.Stat(path)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("public key path is a directory")
}
if info.Size() > maxSSHPublicKeyBytes {
return nil, fmt.Errorf("public key file is too large: %d bytes exceeds %d bytes", info.Size(), maxSSHPublicKeyBytes)
}
return os.ReadFile(path)
}
func validateSSHKeyName(name string) (string, error) {
name = strings.TrimSpace(name)
if name == "" {
internal/api/api_test.go
+59
View File
@@ -2,11 +2,13 @@ package api
import (
"encoding/base64"
"os"
"path/filepath"
"strings"
"testing"
"github.com/andydixon/nexus/internal/audit"
"github.com/andydixon/nexus/internal/leviathan/document"
"github.com/andydixon/nexus/internal/models"
"github.com/andydixon/nexus/internal/storage"
)
@@ -169,6 +171,63 @@ func TestImportProfilesNormalizesSparseRAWProfile(t *testing.T) {
}
}
func TestLeviathanDocumentBytesRejectsOversizedDocument(t *testing.T) {
doc := document.New("large.bin", []byte("abcd"))
_, err := leviathanDocumentBytes(doc, 3, "test")
if err == nil {
t.Fatal("expected oversized document to be rejected")
}
if !strings.Contains(err.Error(), "too large") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestWriteLeviathanDocumentFileWritesPatchedBytes(t *testing.T) {
doc := document.New("patched.bin", []byte("abcd"))
if _, err := doc.Set(1, 'Z'); err != nil {
t.Fatalf("Set() failed: %v", err)
}
path := filepath.Join(t.TempDir(), "out.bin")
if err := writeLeviathanDocumentFile(path, doc); err != nil {
t.Fatalf("writeLeviathanDocumentFile() failed: %v", err)
}
got, err := os.ReadFile(path)
if err != nil {
t.Fatalf("ReadFile() failed: %v", err)
}
if string(got) != "aZcd" {
t.Fatalf("written bytes = %q, want %q", string(got), "aZcd")
}
}
func TestReadLeviathanSmallFileRejectsOversizedFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "big.img")
if err := os.WriteFile(path, []byte("abcd"), 0600); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
_, err := readLeviathanSmallFile(path, 3, "image")
if err == nil {
t.Fatal("expected oversized file to be rejected")
}
if !strings.Contains(err.Error(), "too large") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestReadSSHPublicKeyFileRejectsOversizedFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "big.pub")
if err := os.WriteFile(path, []byte(strings.Repeat("x", maxSSHPublicKeyBytes+1)), 0600); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
_, err := readSSHPublicKeyFile(path)
if err == nil {
t.Fatal("expected oversized public key to be rejected")
}
if !strings.Contains(err.Error(), "too large") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestCreateProfileNormalizesLegacyVNCSecretMode(t *testing.T) {
svc := newTestService(t)
profile := models.NewProfile(models.ProtocolVNC)
internal/api/leviathan.go
+907
View File
@@ -0,0 +1,907 @@
package api
import (
"archive/zip"
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
wailsRuntime "github.com/wailsapp/wails/v2/pkg/runtime"
"github.com/andydixon/nexus/internal/leviathan/analysis"
"github.com/andydixon/nexus/internal/leviathan/binsnoop"
"github.com/andydixon/nexus/internal/leviathan/disasm"
"github.com/andydixon/nexus/internal/leviathan/document"
"github.com/andydixon/nexus/internal/leviathan/parsers"
"github.com/andydixon/nexus/internal/leviathan/patch"
"github.com/andydixon/nexus/internal/leviathan/pdf"
"github.com/andydixon/nexus/internal/leviathan/spectrogram"
"github.com/andydixon/nexus/internal/leviathan/squashfs"
"github.com/andydixon/nexus/internal/leviathan/stego"
"github.com/andydixon/nexus/internal/leviathan/transforms"
)
type leviathanState struct {
mu sync.Mutex
doc *document.Document
cmp *document.Document
}
type leviathanDocState struct {
Name string `json:"name"`
Size int64 `json:"size"`
Dirty bool `json:"dirty"`
UndoLen int `json:"undoLen"`
RedoLen int `json:"redoLen"`
Compare *leviathanDocState `json:"compare,omitempty"`
PreviewHex []map[string]any `json:"previewHex"`
}
func (s *Service) leviathanEngine() *leviathanState {
if s.leviathan == nil {
s.leviathan = &leviathanState{}
}
return s.leviathan
}
// LeviathanOpenFile prompts for a binary and loads it into the Leviathan workbench.
func (s *Service) LeviathanOpenFile() (map[string]any, error) {
if s.ctx == nil {
return nil, fmt.Errorf("application context not set")
}
path, err := wailsRuntime.OpenFileDialog(s.ctx, wailsRuntime.OpenDialogOptions{
Title: "Open Binary",
})
if err != nil || path == "" {
return nil, err
}
return s.LeviathanLoadFile(path, false)
}
// LeviathanOpenCompare prompts for a binary to compare against the working file.
func (s *Service) LeviathanOpenCompare() (map[string]any, error) {
if s.ctx == nil {
return nil, fmt.Errorf("application context not set")
}
path, err := wailsRuntime.OpenFileDialog(s.ctx, wailsRuntime.OpenDialogOptions{
Title: "Open Compare Binary",
})
if err != nil || path == "" {
return nil, err
}
return s.LeviathanLoadFile(path, true)
}
// LeviathanLoadFile loads a path supplied by drag/drop or the native file picker.
func (s *Service) LeviathanLoadFile(path string, compare bool) (map[string]any, error) {
d, err := document.Open(path)
if err != nil {
return nil, err
}
engine := s.leviathanEngine()
engine.mu.Lock()
if compare {
engine.cmp = d
} else {
engine.doc = d
}
state := engine.stateLocked()
engine.mu.Unlock()
return map[string]any{"state": state}, nil
}
// LeviathanState returns the current binary workbench state.
func (s *Service) LeviathanState() (leviathanDocState, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
defer engine.mu.Unlock()
return engine.stateLocked(), nil
}
// LeviathanRunTool runs one Leviathan analysis/editing tool.
func (s *Service) LeviathanRunTool(tool string, params map[string]any) (map[string]any, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
if engine.doc == nil {
engine.mu.Unlock()
return nil, errors.New("open a file first")
}
if tool == "spectrogram-encode" || tool == "spectrogram-insert" || tool == "spectrogram-view" || tool == "case" {
doc := engine.doc
engine.mu.Unlock()
switch tool {
case "spectrogram-encode":
return s.leviathanSpectrogramEncode(doc, params)
case "spectrogram-insert":
return s.leviathanSpectrogramInsert(doc, params)
case "case":
return s.leviathanExportCase(doc)
default:
return leviathanSpectrogramView(doc, params)
}
}
defer engine.mu.Unlock()
return engine.runToolLocked(tool, params)
}
// LeviathanSetByte patches one byte in the working document.
func (s *Service) LeviathanSetByte(offset int64, value string) (map[string]any, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
defer engine.mu.Unlock()
if engine.doc == nil {
return nil, errors.New("open a file first")
}
value = strings.TrimSpace(value)
value = strings.TrimPrefix(strings.TrimPrefix(value, "0x"), "0X")
if len(value) == 1 {
value = "0" + value
}
if len(value) != 2 {
return nil, errors.New("byte value must be two hex digits")
}
b, err := hex.DecodeString(value)
if err != nil {
return nil, err
}
if _, err := engine.doc.Set(offset, b[0]); err != nil {
return nil, err
}
return map[string]any{"state": engine.stateLocked()}, nil
}
// LeviathanUndo reverts the latest byte edit.
func (s *Service) LeviathanUndo() (leviathanDocState, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
defer engine.mu.Unlock()
if engine.doc == nil {
return leviathanDocState{}, errors.New("open a file first")
}
engine.doc.Undo()
return engine.stateLocked(), nil
}
// LeviathanRedo reapplies the latest undone byte edit.
func (s *Service) LeviathanRedo() (leviathanDocState, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
defer engine.mu.Unlock()
if engine.doc == nil {
return leviathanDocState{}, errors.New("open a file first")
}
engine.doc.Redo()
return engine.stateLocked(), nil
}
// LeviathanRevert clears all working edits.
func (s *Service) LeviathanRevert() (leviathanDocState, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
defer engine.mu.Unlock()
if engine.doc == nil {
return leviathanDocState{}, errors.New("open a file first")
}
engine.doc.Revert()
return engine.stateLocked(), nil
}
// LeviathanSaveWorking prompts for a destination and writes the patched bytes.
func (s *Service) LeviathanSaveWorking() (string, error) {
engine := s.leviathanEngine()
engine.mu.Lock()
if engine.doc == nil {
engine.mu.Unlock()
return "", errors.New("open a file first")
}
doc := engine.doc
engine.mu.Unlock()
if s.ctx == nil {
return "", fmt.Errorf("application context not set")
}
path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{
Title: "Save Working Copy",
DefaultFilename: doc.Name() + ".patched",
})
if err != nil || path == "" {
return "", err
}
return path, writeLeviathanDocumentFile(path, doc)
}
// LeviathanExportJSON writes a tool result to disk.
func (s *Service) LeviathanExportJSON(name string, value map[string]any) (string, error) {
if s.ctx == nil {
return "", fmt.Errorf("application context not set")
}
path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{
Title: "Export JSON",
DefaultFilename: safeExportName(name, "leviathan-result.json"),
})
if err != nil || path == "" {
return "", err
}
b, err := json.MarshalIndent(value, "", " ")
if err != nil {
return "", err
}
return path, os.WriteFile(path, b, 0600)
}
func (l *leviathanState) stateLocked() leviathanDocState {
state := leviathanDocState{}
if l.doc != nil {
state = leviathanDocStateFor(l.doc, true)
}
if l.cmp != nil {
cmp := leviathanDocStateFor(l.cmp, false)
state.Compare = &cmp
}
return state
}
func leviathanDocStateFor(d *document.Document, preview bool) leviathanDocState {
s := leviathanDocState{
Name: d.Name(),
Size: d.Size(),
Dirty: d.IsDirty(),
UndoLen: d.HistoryLen(),
RedoLen: d.RedoLen(),
}
if preview {
s.PreviewHex = leviathanHexRows(d, 0, 512, 8)
}
return s
}
func (l *leviathanState) runToolLocked(tool string, params map[string]any) (map[string]any, error) {
start := int64Param(params, "start", 0)
length := int64Param(params, "length", -1)
switch tool {
case "overview":
return analysis.Overview(l.doc)
case "hashes":
return analysis.Hashes(l.doc, start, length)
case "entropy":
return analysis.Entropy(l.doc, start, length, int64Param(params, "window", 4096), int64Param(params, "step", 4096))
case "strings":
items, err := analysis.ExtractStrings(l.doc, start, length, intParam(params, "min", 4), []string{"ascii", "utf16le", "utf16be"}, intParam(params, "max", 5000))
return map[string]any{"items": items}, err
case "iocs":
return analysis.ExtractIOCs(l.doc, start, length, intParam(params, "min", 5))
case "carve":
items, err := analysis.Carve(l.doc, start, length, intParam(params, "max", 256))
return map[string]any{"items": items}, err
case "keys":
items, err := analysis.ScanKeys(l.doc)
return map[string]any{"items": items}, err
case "timestamps":
return analysis.ScanTimestamps(l.doc, start, length, intParam(params, "max", 1000))
case "search":
return l.runSearch(params, start, length)
case "multi-search":
return analysis.MultiSearch(l.doc, start, length, stringParam(params, "spec", ""), intParam(params, "max", 1000))
case "parse":
name, tree, err := parseLeviathanDocument(l.doc, stringParam(params, "kind", "auto"))
if err != nil {
return nil, err
}
return map[string]any{"parser": name, "tree": tree.ToMap()}, nil
case "disasm":
return l.runDisasm(params, start, length)
case "diff":
if l.cmp == nil {
return nil, errors.New("open a compare file first")
}
return analysis.Diff(l.doc, l.cmp, start, length)
case "stego":
out, err := stego.AnalyseAppended(l.doc)
if err != nil {
return nil, err
}
meta, _ := stego.ScanMetadata(l.doc)
out["metadata"] = meta
return out, nil
case "metadata":
items, err := stego.ScanMetadata(l.doc)
return map[string]any{"items": items}, err
case "pdf":
return pdf.Parse(l.doc)
case "squashfs":
fs, err := squashfs.Open(l.doc)
if err != nil {
return nil, err
}
entries, err := fs.Walk()
return map[string]any{"superblock": fs.Superblock(), "compression": fs.CompressionName(), "entries": entries}, err
case "binsnoop":
buf, err := l.rangeBytes(0, min64(l.doc.Size(), int64Param(params, "maxBytes", 64<<20)))
if err != nil {
return nil, err
}
scanner := binsnoop.New()
return map[string]any{"results": scanner.AnalyzeBytes(buf, l.doc.Name(), false), "signatures": binsnoop.ListSignatures()}, nil
case "rules":
rules, err := analysis.ParseRules(stringParam(params, "source", ""))
if err != nil {
return nil, err
}
items, err := analysis.RunRules(l.doc, start, length, rules, intParam(params, "max", 1000))
return map[string]any{"items": items}, err
case "patch-plan":
steps, err := patch.Parse(stringParam(params, "source", ""))
if err != nil {
return nil, err
}
plan, err := patch.Plan(l.doc, steps)
return map[string]any{"plan": plan}, err
case "patch-apply":
steps, err := patch.Parse(stringParam(params, "source", ""))
if err != nil {
return nil, err
}
if _, err := patch.Plan(l.doc, steps); err != nil {
return nil, err
}
return map[string]any{"state": l.stateLocked()}, patch.Apply(l.doc, steps)
case "transform":
return l.runTransform(params, start, length)
case "hex":
return map[string]any{"rows": leviathanHexRows(l.doc, start, int64Param(params, "bytes", 1024), intParam(params, "width", 16))}, nil
case "case":
return nil, errors.New("case export must run through the service")
default:
return nil, errors.New("unknown tool: " + tool)
}
}
func (l *leviathanState) runSearch(params map[string]any, start, length int64) (map[string]any, error) {
mode := stringParam(params, "mode", "text")
query := stringParam(params, "query", "")
max := intParam(params, "max", 1000)
switch mode {
case "hex":
b, err := hex.DecodeString(strings.ReplaceAll(query, " ", ""))
if err != nil {
return nil, err
}
hits, err := analysis.SearchBytes(l.doc, start, length, b, max)
return map[string]any{"hits": analysis.HitsToMaps(hits)}, err
case "regex":
hits, err := analysis.SearchRegex(l.doc, start, length, query, max)
return map[string]any{"hits": analysis.HitsToMaps(hits)}, err
default:
hits, err := analysis.SearchBytes(l.doc, start, length, []byte(query), max)
return map[string]any{"hits": analysis.HitsToMaps(hits)}, err
}
}
func (l *leviathanState) runDisasm(params map[string]any, start, length int64) (map[string]any, error) {
if length < 0 || length > 65536 {
length = 4096
}
buf, err := l.rangeBytes(start, length)
if err != nil {
return nil, err
}
return map[string]any{"items": disasm.Disassemble(buf, start, stringParam(params, "arch", "x86_64"), intParam(params, "max", 1000))}, nil
}
func (l *leviathanState) runTransform(params map[string]any, start, length int64) (map[string]any, error) {
if length < 0 || length > 8<<20 {
length = min64(l.doc.Size()-start, 1<<20)
}
buf, err := l.rangeBytes(start, length)
if err != nil {
return nil, err
}
steps := parseLeviathanTransformPipeline(stringParam(params, "pipeline", "hex.encode"))
results, err := transforms.ApplyPipeline(buf, steps)
if err != nil {
return nil, err
}
out := make([]map[string]any, 0, len(results))
for i, r := range results {
out = append(out, map[string]any{"step": i, "length": len(r), "preview": previewLeviathanBytes(r)})
}
return map[string]any{"steps": steps, "results": out}, nil
}
func leviathanSpectrogramView(doc *document.Document, params map[string]any) (map[string]any, error) {
buf, err := leviathanDocumentBytes(doc, spectrogram.MaxAudioBytes, "audio")
if err != nil {
return nil, err
}
pngBytes, summary, err := spectrogram.AudioSpectrogramPNG(buf, doc.Name(), leviathanSpectrogramViewOptions(params))
if err != nil {
return nil, err
}
return map[string]any{
"mime": "image/png",
"pngBase64": base64.StdEncoding.EncodeToString(pngBytes),
"summary": summary,
}, nil
}
func (s *Service) leviathanExportCase(doc *document.Document) (map[string]any, error) {
if s.ctx == nil {
return nil, fmt.Errorf("application context not set")
}
filename := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".case.zip"
path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{
Title: "Export Leviathan Case",
DefaultFilename: filename,
})
if err != nil || path == "" {
return nil, err
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return nil, err
}
defer f.Close()
zw := zip.NewWriter(f)
add := func(name string, data []byte) error {
w, err := zw.Create(name)
if err != nil {
return err
}
_, err = w.Write(data)
return err
}
hashes, _ := analysis.Hashes(doc, 0, -1)
overview, _ := analysis.Overview(doc)
manifest, _ := json.MarshalIndent(map[string]any{"name": doc.Name(), "size": doc.Size(), "exportedAt": time.Now().Format(time.RFC3339), "hashes": hashes}, "", " ")
overviewJSON, _ := json.MarshalIndent(overview, "", " ")
patchesJSON, _ := json.MarshalIndent(doc.Patches(), "", " ")
if err := add("manifest.json", manifest); err != nil {
return nil, err
}
if err := addDocument("working.bin", doc, zw); err != nil {
return nil, err
}
if err := add("overview.json", overviewJSON); err != nil {
return nil, err
}
if err := add("patches.json", patchesJSON); err != nil {
return nil, err
}
if err := zw.Close(); err != nil {
return nil, err
}
if err := f.Sync(); err != nil {
return nil, err
}
info, err := f.Stat()
if err != nil {
return nil, err
}
return map[string]any{"path": path, "bytes": info.Size()}, nil
}
func (s *Service) leviathanSpectrogramEncode(doc *document.Document, params map[string]any) (map[string]any, error) {
if s.ctx == nil {
return nil, fmt.Errorf("application context not set")
}
imageBytes, err := leviathanDocumentBytes(doc, spectrogram.MaxImageBytes, "image")
if err != nil {
return nil, err
}
wavBytes, summary, err := spectrogram.ImageToWAV(imageBytes, leviathanSpectrogramEncodeOptions(params))
if err != nil {
return nil, err
}
defaultName := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".spectrogram.wav"
path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{
Title: "Save Spectrogram WAV",
DefaultFilename: safeExportName(defaultName, "spectrogram.wav"),
Filters: []wailsRuntime.FileFilter{
{DisplayName: "WAV Audio (*.wav)", Pattern: "*.wav"},
},
})
if err != nil || path == "" {
return nil, err
}
if filepath.Ext(strings.ToLower(path)) != ".wav" {
path += ".wav"
}
if err := os.WriteFile(path, wavBytes, 0600); err != nil {
return nil, err
}
summary["path"] = path
return map[string]any{"path": path, "summary": summary}, nil
}
func (s *Service) leviathanSpectrogramInsert(doc *document.Document, params map[string]any) (map[string]any, error) {
if s.ctx == nil {
return nil, fmt.Errorf("application context not set")
}
imagePath := strings.TrimSpace(stringParam(params, "imagePath", ""))
if imagePath == "" {
return nil, errors.New("select an image to insert")
}
imageBytes, err := readLeviathanSmallFile(imagePath, spectrogram.MaxImageBytes, "image")
if err != nil {
return nil, fmt.Errorf("read image: %w", err)
}
audioBytes, err := leviathanDocumentBytes(doc, spectrogram.MaxAudioBytes, "audio")
if err != nil {
return nil, err
}
wavBytes, summary, err := spectrogram.InsertImageIntoAudio(imageBytes, audioBytes, doc.Name(), leviathanSpectrogramInsertOptions(params))
if err != nil {
return nil, err
}
defaultName := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".with-spectrogram.wav"
path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{
Title: "Save Audio With Spectrogram Image",
DefaultFilename: safeExportName(defaultName, "with-spectrogram.wav"),
Filters: []wailsRuntime.FileFilter{
{DisplayName: "WAV Audio (*.wav)", Pattern: "*.wav"},
},
})
if err != nil || path == "" {
return nil, err
}
if filepath.Ext(strings.ToLower(path)) != ".wav" {
path += ".wav"
}
if err := os.WriteFile(path, wavBytes, 0600); err != nil {
return nil, err
}
summary["path"] = path
summary["imagePath"] = imagePath
return map[string]any{"path": path, "summary": summary}, nil
}
func writeLeviathanDocumentFile(path string, doc *document.Document) error {
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer f.Close()
if err := doc.ForEachChunk(0, doc.Size(), func(_ int64, chunk []byte) error {
_, err := f.Write(chunk)
return err
}); err != nil {
return err
}
return f.Sync()
}
func addDocument(name string, doc *document.Document, zw *zip.Writer) error {
w, err := zw.Create(name)
if err != nil {
return err
}
return doc.ForEachChunk(0, doc.Size(), func(_ int64, chunk []byte) error {
_, err := w.Write(chunk)
return err
})
}
func leviathanDocumentBytes(doc *document.Document, maxBytes int64, label string) ([]byte, error) {
size := doc.Size()
if maxBytes > 0 && size > maxBytes {
return nil, fmt.Errorf("%s file is too large for this operation: %s exceeds %s", label, humanBytes(size), humanBytes(maxBytes))
}
return doc.Range(0, size)
}
func readLeviathanSmallFile(path string, maxBytes int64, label string) ([]byte, error) {
info, err := os.Stat(path)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("%s path is a directory", label)
}
if maxBytes > 0 && info.Size() > maxBytes {
return nil, fmt.Errorf("%s file is too large for this operation: %s exceeds %s", label, humanBytes(info.Size()), humanBytes(maxBytes))
}
return os.ReadFile(path)
}
func humanBytes(n int64) string {
const unit = 1024
if n < unit {
return fmt.Sprintf("%d B", n)
}
value := float64(n)
for _, suffix := range []string{"KiB", "MiB", "GiB", "TiB"} {
value /= unit
if value < unit {
return fmt.Sprintf("%.1f %s", value, suffix)
}
}
return fmt.Sprintf("%.1f PiB", value/unit)
}
func (l *leviathanState) rangeBytes(start, length int64) ([]byte, error) {
if start < 0 {
start = 0
}
if length < 0 || start+length > l.doc.Size() {
length = l.doc.Size() - start
}
if length < 0 {
length = 0
}
return l.doc.Range(start, length)
}
func parseLeviathanDocument(d *document.Document, kind string) (string, *parsers.Field, error) {
kind = strings.ToLower(strings.TrimSpace(kind))
if kind == "" || kind == "auto" {
return parsers.DetectAndParse(d)
}
var tree *parsers.Field
var err error
switch kind {
case "png":
tree, err = parsers.ParsePNG(d)
case "jpeg", "jpg":
tree, err = parsers.ParseJPEG(d)
kind = "jpeg"
case "gif":
tree, err = parsers.ParseGIF(d)
case "bmp":
tree, err = parsers.ParseBMP(d)
case "zip":
tree, err = parsers.ParseZIP(d)
case "elf":
tree, err = parsers.ParseELF(d)
case "pe":
tree, err = parsers.ParsePE(d)
case "macho":
tree, err = parsers.ParseMachO(d)
case "riff":
tree, err = parsers.ParseRIFF(d)
case "asn1":
tree, err = parsers.ParseASN1(d, 0, d.Size())
case "cert":
tree, err = parsers.ParseCerts(d)
default:
return "", nil, errors.New("unknown parser: " + kind)
}
return kind, tree, err
}
func leviathanHexRows(d *document.Document, start, length int64, width int) []map[string]any {
if width <= 0 {
width = 16
}
if length < 0 || length > 8192 {
length = 8192
}
rows := []map[string]any{}
for off := start; off < start+length && off < d.Size(); off += int64(width) {
n := int64(width)
if off+n > d.Size() {
n = d.Size() - off
}
buf, err := d.Range(off, n)
if err != nil {
break
}
rows = append(rows, map[string]any{"offset": off, "hex": spacedLeviathanHex(buf), "bytes": byteHexCells(buf), "ascii": asciiLeviathanPreview(buf)})
}
return rows
}
func parseLeviathanTransformPipeline(src string) []transforms.Step {
parts := strings.Split(src, "|")
steps := []transforms.Step{}
for _, part := range parts {
fields := strings.Fields(strings.TrimSpace(part))
if len(fields) == 0 {
continue
}
op := fields[0]
arg := ""
if len(fields) > 1 {
arg = strings.Join(fields[1:], " ")
}
switch op {
case "hex":
op = "hex.encode"
case "unhex":
op = "hex.decode"
case "base64", "b64":
op = "base64.encode"
case "unbase64", "unb64":
op = "base64.decode"
case "url":
op = "url.encode"
case "unurl":
op = "url.decode"
}
steps = append(steps, transforms.Step{Op: op, Arg: arg})
}
return steps
}
func spacedLeviathanHex(buf []byte) string {
return strings.Join(byteHexCells(buf), " ")
}
func byteHexCells(buf []byte) []string {
parts := make([]string, len(buf))
for i, b := range buf {
parts[i] = fmt.Sprintf("%02x", b)
}
return parts
}
func asciiLeviathanPreview(buf []byte) string {
var b strings.Builder
for _, c := range buf {
if c >= 32 && c < 127 {
b.WriteByte(c)
} else {
b.WriteByte('.')
}
}
return b.String()
}
func previewLeviathanBytes(buf []byte) string {
if len(buf) > 2048 {
buf = buf[:2048]
}
if isMostlyLeviathanText(buf) {
return string(buf)
}
return spacedLeviathanHex(buf)
}
func isMostlyLeviathanText(buf []byte) bool {
if len(buf) == 0 {
return true
}
printable := 0
for _, b := range buf {
if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b < 127) {
printable++
}
}
return printable*100/len(buf) > 85
}
func intParam(m map[string]any, key string, def int) int {
return int(int64Param(m, key, int64(def)))
}
func int64Param(m map[string]any, key string, def int64) int64 {
v, ok := m[key]
if !ok || v == nil {
return def
}
switch n := v.(type) {
case int64:
return n
case int:
return int64(n)
case float64:
return int64(n)
case string:
if n == "" {
return def
}
if parsed, err := strconv.ParseInt(n, 0, 64); err == nil {
return parsed
}
}
return def
}
func stringParam(m map[string]any, key, def string) string {
if v, ok := m[key]; ok && v != nil {
return fmt.Sprint(v)
}
return def
}
func floatParam(m map[string]any, key string, def float64) float64 {
v, ok := m[key]
if !ok || v == nil {
return def
}
switch n := v.(type) {
case float64:
return n
case float32:
return float64(n)
case int:
return float64(n)
case int64:
return float64(n)
case string:
if n == "" {
return def
}
if parsed, err := strconv.ParseFloat(n, 64); err == nil {
return parsed
}
}
return def
}
func boolParam(m map[string]any, key string, def bool) bool {
v, ok := m[key]
if !ok || v == nil {
return def
}
switch b := v.(type) {
case bool:
return b
case string:
if b == "" {
return def
}
if parsed, err := strconv.ParseBool(b); err == nil {
return parsed
}
}
return def
}
func leviathanSpectrogramEncodeOptions(params map[string]any) spectrogram.EncodeOptions {
return spectrogram.EncodeOptions{
DurationSec: floatParam(params, "durationSec", 5),
SampleRate: intParam(params, "sampleRate", 44100),
MinHz: floatParam(params, "minHz", 200),
MaxHz: floatParam(params, "maxHz", 20000),
Bins: intParam(params, "bins", 160),
Columns: intParam(params, "columns", 768),
Invert: boolParam(params, "invert", true),
PreserveAR: boolParam(params, "preserveAspectRatio", true),
Gain: floatParam(params, "gain", 0.85),
}
}
func leviathanSpectrogramInsertOptions(params map[string]any) spectrogram.InsertOptions {
return spectrogram.InsertOptions{
EncodeOptions: leviathanSpectrogramEncodeOptions(params),
StartSec: floatParam(params, "startSec", 0),
MixLevel: floatParam(params, "mixLevel", 0.35),
}
}
func leviathanSpectrogramViewOptions(params map[string]any) spectrogram.ViewOptions {
return spectrogram.ViewOptions{
FFTSize: intParam(params, "fftSize", 2048),
Height: intParam(params, "height", 1024),
MaxWidth: intParam(params, "maxWidth", 1400),
Aspect: floatParam(params, "aspectRatio", 1),
MinHz: floatParam(params, "minHz", 0),
MaxHz: floatParam(params, "maxHz", 0),
Brightness: floatParam(params, "brightness", 1),
Contrast: floatParam(params, "contrast", 1),
}
}
func min64(a, b int64) int64 {
if a < b {
return a
}
return b
}
func safeExportName(name, fallback string) string {
name = filepath.Base(strings.TrimSpace(name))
if name == "" || name == "." || strings.ContainsRune(name, 0) {
return fallback
}
return name
}
internal/githubclient/githubclient.go
+62 -32
View File
@@ -19,6 +19,11 @@ import (
"github.com/andydixon/nexus/internal/models"
)
const (
maxGitHubJSONBodyBytes = 4 << 20
maxGitHubErrorBodyBytes = 64 << 10
)
// Client wraps the GitHub REST API for repository listing.
type Client struct {
baseURL string
@@ -41,19 +46,19 @@ func New(baseURL, token string) *Client {
// apiRepo is the subset of the GitHub API response we care about.
type apiRepo struct {
FullName string `json:"full_name"`
Name string `json:"name"`
Description string `json:"description"`
HTMLURL string `json:"html_url"`
DefaultBranch string `json:"default_branch"`
Private bool `json:"private"`
Fork bool `json:"fork"`
Archived bool `json:"archived"`
Language string `json:"language"`
Stars int `json:"stargazers_count"`
PushedAt time.Time `json:"pushed_at"`
UpdatedAt time.Time `json:"updated_at"`
Owner struct {
FullName string `json:"full_name"`
Name string `json:"name"`
Description string `json:"description"`
HTMLURL string `json:"html_url"`
DefaultBranch string `json:"default_branch"`
Private bool `json:"private"`
Fork bool `json:"fork"`
Archived bool `json:"archived"`
Language string `json:"language"`
Stars int `json:"stargazers_count"`
PushedAt time.Time `json:"pushed_at"`
UpdatedAt time.Time `json:"updated_at"`
Owner struct {
Login string `json:"login"`
} `json:"owner"`
}
@@ -90,7 +95,7 @@ func (c *Client) fetchRepos(ctx context.Context, progress func(done int, total i
return nil, fmt.Errorf("fetching repos page %d: %w", page, err)
}
body, readErr := io.ReadAll(resp.Body)
body, readErr := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes)
resp.Body.Close()
if readErr != nil {
return nil, fmt.Errorf("reading response body: %w", readErr)
@@ -128,19 +133,19 @@ func (c *Client) fetchRepos(ctx context.Context, progress func(done int, total i
result := make([]models.GitHubRepo, len(all))
for i, r := range all {
result[i] = models.GitHubRepo{
FullName: r.FullName,
Name: r.Name,
Owner: r.Owner.Login,
Description: r.Description,
HTMLURL: r.HTMLURL,
FullName: r.FullName,
Name: r.Name,
Owner: r.Owner.Login,
Description: r.Description,
HTMLURL: r.HTMLURL,
DefaultBranch: r.DefaultBranch,
Private: r.Private,
Fork: r.Fork,
Archived: r.Archived,
Language: r.Language,
Stars: r.Stars,
PushedAt: r.PushedAt.Format(time.RFC3339),
UpdatedAt: r.UpdatedAt.Format(time.RFC3339),
Private: r.Private,
Fork: r.Fork,
Archived: r.Archived,
Language: r.Language,
Stars: r.Stars,
PushedAt: r.PushedAt.Format(time.RFC3339),
UpdatedAt: r.UpdatedAt.Format(time.RFC3339),
}
}
@@ -208,8 +213,7 @@ func (c *Client) FetchLatestCommit(ctx context.Context, fullName string, ref str
return "", "", fmt.Errorf("authentication failed (HTTP %d)", resp.StatusCode)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return "", "", fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, string(body))
return "", "", fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, readGitHubErrorBody(resp.Body))
}
var payload struct {
@@ -225,7 +229,11 @@ func (c *Client) FetchLatestCommit(ctx context.Context, fullName string, ref str
} `json:"commit"`
}
if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil {
body, err := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes)
if err != nil {
return "", "", fmt.Errorf("reading commit response: %w", err)
}
if err := json.Unmarshal(body, &payload); err != nil {
return "", "", fmt.Errorf("decoding commit: %w", err)
}
@@ -270,8 +278,7 @@ func (c *Client) DownloadRepoArchive(ctx context.Context, fullName string, ref s
return fmt.Errorf("authentication failed (HTTP %d) — check your access token", resp.StatusCode)
}
if resp.StatusCode != http.StatusOK {
body, _ := io.ReadAll(resp.Body)
return fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, string(body))
return fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, readGitHubErrorBody(resp.Body))
}
file, err := os.Create(destPath)
@@ -310,8 +317,31 @@ func (c *Client) ValidateToken(ctx context.Context) (string, error) {
var user struct {
Login string `json:"login"`
}
if err := json.NewDecoder(resp.Body).Decode(&user); err != nil {
body, err := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes)
if err != nil {
return "", fmt.Errorf("reading user response: %w", err)
}
if err := json.Unmarshal(body, &user); err != nil {
return "", fmt.Errorf("decoding user: %w", err)
}
return user.Login, nil
}
func readLimitedGitHubBody(r io.Reader, maxBytes int64) ([]byte, error) {
body, err := io.ReadAll(io.LimitReader(r, maxBytes+1))
if err != nil {
return nil, err
}
if int64(len(body)) > maxBytes {
return nil, fmt.Errorf("response body exceeds %d bytes", maxBytes)
}
return body, nil
}
func readGitHubErrorBody(r io.Reader) string {
body, err := readLimitedGitHubBody(r, maxGitHubErrorBodyBytes)
if err != nil {
return "<response body unavailable: " + err.Error() + ">"
}
return strings.TrimSpace(string(body))
}
internal/githubclient/githubclient_test.go
+26
View File
@@ -0,0 +1,26 @@
package githubclient
import (
"strings"
"testing"
)
func TestReadLimitedGitHubBodyRejectsOversizedResponse(t *testing.T) {
_, err := readLimitedGitHubBody(strings.NewReader("abcd"), 3)
if err == nil {
t.Fatal("expected oversized response to be rejected")
}
if !strings.Contains(err.Error(), "response body exceeds") {
t.Fatalf("unexpected error: %v", err)
}
}
func TestReadLimitedGitHubBodyAllowsResponseAtLimit(t *testing.T) {
got, err := readLimitedGitHubBody(strings.NewReader("abc"), 3)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if string(got) != "abc" {
t.Fatalf("body = %q, want abc", string(got))
}
}
internal/leviathan/analysis/carve.go
+876
View File
@@ -0,0 +1,876 @@
package analysis
import (
"bytes"
"compress/gzip"
"context"
"encoding/binary"
"io"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Signature describes a carveable object. `validate` is run against
// the candidate start and returns the object length if it is a real
// object, or 0 to reject. `trailer`, if present, must appear before
// the object ends (for simple trailer-based terminations).
type Signature struct {
Kind string
Magic []byte
Extension string
// Anchor is how many bytes into the object the magic appears. It is 0 for
// almost every format (magic at offset 0), but a few — notably ISO-BMFF
// (MP4/MOV), whose `ftyp` magic follows a 4-byte box size — carry it.
Anchor int64
Validate func(d *document.Document, start, max int64) (length int64, confidence float64)
}
// carveSignatures is the phase-1 signature set. We stay conservative
// and only declare a carve when we can determine length reliably.
var carveSignatures = []Signature{
{Kind: "png", Magic: []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, Extension: "png", Validate: validatePNG},
{Kind: "jpeg", Magic: []byte{0xff, 0xd8, 0xff}, Extension: "jpg", Validate: validateJPEG},
{Kind: "gif87a", Magic: []byte("GIF87a"), Extension: "gif", Validate: validateGIF},
{Kind: "gif89a", Magic: []byte("GIF89a"), Extension: "gif", Validate: validateGIF},
{Kind: "zip", Magic: []byte{'P', 'K', 0x03, 0x04}, Extension: "zip", Validate: validateZIP},
{Kind: "pdf", Magic: []byte("%PDF-"), Extension: "pdf", Validate: validatePDF},
{Kind: "elf", Magic: []byte{0x7f, 'E', 'L', 'F'}, Extension: "elf", Validate: validateELF},
{Kind: "mz", Magic: []byte{'M', 'Z'}, Extension: "exe", Validate: validatePE},
{Kind: "gzip", Magic: []byte{0x1f, 0x8b, 0x08}, Extension: "gz", Validate: validateGZIP},
{Kind: "bzip2", Magic: []byte{'B', 'Z', 'h'}, Extension: "bz2", Validate: validateBzip2},
{Kind: "sevenz", Magic: []byte{'7', 'z', 0xbc, 0xaf, 0x27, 0x1c}, Extension: "7z", Validate: validate7z},
{Kind: "rar4", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x00}, Extension: "rar", Validate: validateRAR},
{Kind: "rar5", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00}, Extension: "rar", Validate: validateRAR},
{Kind: "wav", Magic: []byte("RIFF"), Extension: "wav", Validate: validateRIFF},
{Kind: "mach-o-le32", Magic: []byte{0xce, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
{Kind: "mach-o-le64", Magic: []byte{0xcf, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO},
{Kind: "mach-o-be32", Magic: []byte{0xfe, 0xed, 0xfa, 0xce}, Extension: "macho", Validate: validateMachO},
{Kind: "mach-o-be64", Magic: []byte{0xfe, 0xed, 0xfa, 0xcf}, Extension: "macho", Validate: validateMachO},
{Kind: "sqlite", Magic: []byte("SQLite format 3\x00"), Extension: "sqlite", Validate: validateSQLite},
{Kind: "mp3", Magic: []byte("ID3"), Extension: "mp3", Validate: validateID3},
{Kind: "ogg", Magic: []byte("OggS"), Extension: "ogg", Validate: validateOgg},
{Kind: "matroska", Magic: []byte{0x1a, 0x45, 0xdf, 0xa3}, Extension: "mkv", Validate: validateMatroska},
{Kind: "flac", Magic: []byte("fLaC"), Extension: "flac", Validate: validateFLAC},
{Kind: "bmp", Magic: []byte{'B', 'M'}, Extension: "bmp", Validate: validateBMP},
{Kind: "mp4", Magic: []byte("ftyp"), Anchor: 4, Extension: "mp4", Validate: validateMP4},
}
// Carve scans the document for known signatures and returns a list of
// discovered objects with offset, length, kind, and confidence.
func Carve(d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
return CarveContext(context.Background(), d, start, length, maxHits)
}
func CarveContext(ctx context.Context, d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) {
if length < 0 {
length = d.Size() - start
}
if maxHits <= 0 {
maxHits = 10_000
}
results := make([]map[string]any, 0, 64)
// Build a flat list of (magicBytes, sigIndex) for a simple first-pass scan.
// Phase 1 uses per-signature naive scan chunks.
for idx, sig := range carveSignatures {
if err := ctx.Err(); err != nil {
return nil, err
}
hits, err := SearchBytesContext(ctx, d, start, length, sig.Magic, maxHits)
if err != nil {
return nil, err
}
for _, h := range hits {
if err := ctx.Err(); err != nil {
return nil, err
}
// The object begins Anchor bytes before the magic (0 for most formats).
objStart := h.Offset - sig.Anchor
if objStart < start {
continue
}
ln, conf := int64(0), 0.4
if sig.Validate != nil {
ln, conf = sig.Validate(d, objStart, start+length)
}
if ln <= 0 {
// record low-confidence magic hit only for kinds where length is not determinable
if sig.Kind == "mz" || sig.Kind == "elf" || sig.Kind == "mach-o-le32" ||
sig.Kind == "mach-o-le64" || sig.Kind == "mach-o-be32" || sig.Kind == "mach-o-be64" ||
sig.Kind == "sqlite" || sig.Kind == "mp3" || sig.Kind == "matroska" ||
sig.Kind == "flac" || sig.Kind == "bzip2" || sig.Kind == "rar4" || sig.Kind == "rar5" {
results = append(results, map[string]any{
"kind": sig.Kind,
"extension": sig.Extension,
"offset": objStart,
"length": int64(0),
"confidence": 0.3,
"status": "magic-only",
"sigIndex": idx,
})
}
continue
}
results = append(results, map[string]any{
"kind": sig.Kind,
"extension": sig.Extension,
"offset": objStart,
"length": ln,
"confidence": conf,
"status": "valid",
"sigIndex": idx,
})
if len(results) >= maxHits {
return results, nil
}
}
}
return results, nil
}
// --- validators ---
func validatePNG(d *document.Document, start, max int64) (int64, float64) {
// Walk chunks until IEND
off := start + 8
for off < max {
hdr, err := d.Range(off, 8)
if err != nil || len(hdr) < 8 {
return 0, 0
}
size := int64(binary.BigEndian.Uint32(hdr[0:4]))
typ := string(hdr[4:8])
// Skip payload + CRC (4 bytes)
off += 8 + size + 4
if typ == "IEND" {
return off - start, 0.95
}
if off > max {
return 0, 0
}
}
return 0, 0
}
func validateJPEG(d *document.Document, start, max int64) (int64, float64) {
// Walk segments until EOI (0xFFD9)
off := start + 2
for off < max-1 {
buf, err := d.Range(off, 2)
if err != nil || len(buf) < 2 {
return 0, 0
}
if buf[0] != 0xff {
return 0, 0
}
marker := buf[1]
off += 2
// No-length markers
if marker == 0xd9 { // EOI
return off - start, 0.95
}
if marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker == 0x00 {
continue
}
// Read length
lb, err := d.Range(off, 2)
if err != nil || len(lb) < 2 {
return 0, 0
}
ln := int64(binary.BigEndian.Uint16(lb))
off += ln
if marker == 0xda { // SOS, scan until next 0xff!=0
for off < max {
b, err := d.Range(off, 2)
if err != nil || len(b) < 2 {
return 0, 0
}
if b[0] == 0xff && b[1] != 0x00 && !(b[1] >= 0xd0 && b[1] <= 0xd7) {
break
}
off++
}
}
}
return 0, 0
}
func validateGIF(d *document.Document, start, max int64) (int64, float64) {
// Minimal: locate trailer 0x3B
buf, err := d.Range(start, min64(max-start, 1<<20))
if err != nil {
return 0, 0
}
idx := bytes.IndexByte(buf[13:], 0x3b)
if idx < 0 {
return 0, 0
}
return int64(idx + 13 + 1), 0.7
}
func validateZIP(d *document.Document, start, max int64) (int64, float64) {
// Search forward for End of Central Directory record 0x06054b50.
const sig = "\x50\x4b\x05\x06"
limit := min64(max-start, 64<<20)
buf, err := d.Range(start, limit)
if err != nil {
return 0, 0
}
idx := bytes.LastIndex(buf, []byte(sig))
if idx < 0 {
return 0, 0
}
// EOCD record is at least 22 bytes; read comment length.
if idx+22 > len(buf) {
return 0, 0
}
commentLen := int(binary.LittleEndian.Uint16(buf[idx+20 : idx+22]))
end := int64(idx + 22 + commentLen)
return end, 0.9
}
func validatePDF(d *document.Document, start, max int64) (int64, float64) {
limit := min64(max-start, 64<<20)
buf, err := d.Range(start, limit)
if err != nil {
return 0, 0
}
idx := bytes.LastIndex(buf, []byte("%%EOF"))
if idx < 0 {
return 0, 0
}
return int64(idx + 5), 0.85
}
func validateELF(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 64)
if err != nil || len(hdr) < 52 {
return 0, 0
}
if hdr[4] != 1 && hdr[4] != 2 {
return 0, 0
}
is64 := hdr[4] == 2
little := hdr[5] == 1
var end int64
if is64 {
if len(hdr) < 64 {
return 0, 0
}
// e_shoff at 40, e_shentsize at 58, e_shnum at 60
off := binary.LittleEndian.Uint64(hdr[40:48])
entSz := uint64(binary.LittleEndian.Uint16(hdr[58:60]))
num := uint64(binary.LittleEndian.Uint16(hdr[60:62]))
if !little {
off = binary.BigEndian.Uint64(hdr[40:48])
entSz = uint64(binary.BigEndian.Uint16(hdr[58:60]))
num = uint64(binary.BigEndian.Uint16(hdr[60:62]))
}
end = int64(off + entSz*num)
} else {
off := binary.LittleEndian.Uint32(hdr[32:36])
entSz := uint32(binary.LittleEndian.Uint16(hdr[46:48]))
num := uint32(binary.LittleEndian.Uint16(hdr[48:50]))
if !little {
off = binary.BigEndian.Uint32(hdr[32:36])
entSz = uint32(binary.BigEndian.Uint16(hdr[46:48]))
num = uint32(binary.BigEndian.Uint16(hdr[48:50]))
}
end = int64(off) + int64(entSz)*int64(num)
}
if end <= 0 || start+end > max {
return 0, 0.5
}
return end, 0.9
}
func validatePE(d *document.Document, start, max int64) (int64, float64) {
// e_lfanew at offset 0x3C points to PE header
hdr, err := d.Range(start, 64)
if err != nil || len(hdr) < 64 {
return 0, 0
}
peOff := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40]))
if peOff < 0x40 || start+peOff+4 > max {
return 0, 0
}
sig, err := d.Range(start+peOff, 4)
if err != nil || string(sig) != "PE\x00\x00" {
return 0, 0.3
}
coff, err := d.Range(start+peOff+4, 20)
if err != nil || len(coff) < 20 {
return 0, 0.4
}
numSections := int(binary.LittleEndian.Uint16(coff[2:4]))
sizeOpt := int64(binary.LittleEndian.Uint16(coff[16:18]))
if numSections <= 0 || numSections > 96 || sizeOpt < 0 || sizeOpt > 4096 {
return 0, 0.5
}
sectOff := start + peOff + 24 + sizeOpt
end := peOff + 24 + sizeOpt + int64(numSections*40)
for i := 0; i < numSections; i++ {
sh, err := d.Range(sectOff+int64(i*40), 40)
if err != nil || len(sh) < 40 {
return 0, 0.5
}
rawSize := int64(binary.LittleEndian.Uint32(sh[16:20]))
rawPtr := int64(binary.LittleEndian.Uint32(sh[20:24]))
if rawSize < 0 || rawPtr < 0 {
return 0, 0.5
}
if rawSize > 0 && rawPtr+rawSize > end {
end = rawPtr + rawSize
}
}
if end <= peOff || start+end > max {
return 0, 0.5
}
return end, 0.85
}
func validateGZIP(d *document.Document, start, max int64) (int64, float64) {
limit := min64(max-start, 64<<20)
buf, err := d.Range(start, limit)
if err != nil || len(buf) < 18 {
return 0, 0
}
br := bytes.NewReader(buf)
zr, err := gzip.NewReader(br)
if err != nil {
return 0, 0
}
zr.Multistream(false)
if _, err := io.Copy(io.Discard, io.LimitReader(zr, 512<<20)); err != nil {
_ = zr.Close()
return 0, 0
}
if err := zr.Close(); err != nil {
return 0, 0
}
consumed := int64(len(buf)) - int64(br.Len())
if consumed <= 10 || start+consumed > max {
return 0, 0
}
return consumed, 0.9
}
// validateBzip2 locates the bit-aligned end-of-stream marker (0x177245385090,
// the "sqrt(pi)" magic) and returns the stream length including the trailing
// 32-bit combined CRC. bzip2 carries no length in its header, so this is the
// only reliable way to bound the object.
func validateBzip2(d *document.Document, start, max int64) (int64, float64) {
if start+4 > max {
return 0, 0.4
}
hdr, err := d.Range(start, 4)
if err != nil || len(hdr) < 4 || hdr[0] != 'B' || hdr[1] != 'Z' || hdr[2] != 'h' || hdr[3] < '1' || hdr[3] > '9' {
return 0, 0
}
limit := min64(max-start, 64<<20)
buf, err := d.Range(start, limit)
if err != nil || len(buf) < 14 {
return 0, 0.4
}
const eos uint64 = 0x177245385090
const mask48 uint64 = (1 << 48) - 1
var w uint64
for i := 0; i < len(buf); i++ {
w = (w << 8) | uint64(buf[i])
// Check all eight bit alignments ending at or before this byte.
for s := 0; s < 8; s++ {
if (w>>uint(s))&mask48 == eos {
endBit := (int64(i)+1)*8 - int64(s) + 32 // include stream CRC
endByte := (endBit + 7) / 8
if endByte > 14 && start+endByte <= max {
return endByte, 0.85
}
}
}
}
return 0, 0.4
}
func validate7z(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 32)
if err != nil || len(hdr) < 32 {
return 0, 0
}
nextHdrOff := binary.LittleEndian.Uint64(hdr[12:20])
nextHdrSz := binary.LittleEndian.Uint64(hdr[20:28])
end := int64(32 + nextHdrOff + nextHdrSz)
if end <= 0 || start+end > max {
return 0, 0.5
}
return end, 0.85
}
// validateRAR walks the block headers of a RAR4 or RAR5 archive to the
// end-of-archive marker. The two formats differ structurally, so the version
// is detected from the signature byte at offset 6.
func validateRAR(d *document.Document, start, max int64) (int64, float64) {
sig, err := d.Range(start, 8)
if err != nil || len(sig) < 8 {
return 0, 0
}
if sig[6] == 0x01 {
return validateRAR5(d, start, max)
}
return validateRAR4(d, start, max)
}
func validateRAR4(d *document.Document, start, max int64) (int64, float64) {
pos := start + 7 // 7-byte signature
for n := 0; n < 100000; n++ {
h, err := d.Range(pos, 11)
if err != nil || len(h) < 7 {
return 0, 0.4
}
flags := binary.LittleEndian.Uint16(h[3:5])
headSize := int64(binary.LittleEndian.Uint16(h[5:7]))
if headSize < 7 {
return 0, 0.4
}
blockSize := headSize
if flags&0x8000 != 0 { // ADD_SIZE present
if len(h) < 11 {
return 0, 0.4
}
blockSize += int64(binary.LittleEndian.Uint32(h[7:11]))
}
pos += blockSize
if h[2] == 0x7b { // terminator block
if pos > start && pos <= max {
return pos - start, 0.85
}
return 0, 0.4
}
if pos > max {
return 0, 0.4
}
}
return 0, 0.4
}
func validateRAR5(d *document.Document, start, max int64) (int64, float64) {
pos := start + 8 // 8-byte signature
for n := 0; n < 100000; n++ {
// Each block: CRC32(4) + vint(header_size) + header_size bytes [+ data].
h, err := d.Range(pos, 4+11)
if err != nil || len(h) < 5 {
return 0, 0.4
}
headerSize, vlen := rar5Vint(h, 4)
if vlen == 0 || headerSize == 0 {
return 0, 0.4
}
hdrStart := pos + 4 + int64(vlen)
body, err := d.Range(hdrStart, int64(headerSize))
if err != nil || int64(len(body)) < int64(headerSize) {
return 0, 0.4
}
htype, t1 := rar5Vint(body, 0)
hflags, t2 := rar5Vint(body, t1)
p := t1 + t2
var dataSize uint64
if hflags&0x0001 != 0 { // extra area present
_, te := rar5Vint(body, p)
p += te
}
if hflags&0x0002 != 0 { // data area present
dataSize, _ = rar5Vint(body, p)
}
pos = hdrStart + int64(headerSize) + int64(dataSize)
if htype == 5 { // end-of-archive header
if pos > start && pos <= max {
return pos - start, 0.85
}
return 0, 0.4
}
if pos > max {
return 0, 0.4
}
}
return 0, 0.4
}
// rar5Vint decodes a RAR5 little-endian base-128 variable integer.
func rar5Vint(buf []byte, pos int) (uint64, int) {
var v uint64
shift := uint(0)
for n := 0; pos+n < len(buf) && n < 10; n++ {
b := buf[pos+n]
v |= uint64(b&0x7f) << shift
if b&0x80 == 0 {
return v, n + 1
}
shift += 7
}
return 0, 0
}
func validateRIFF(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 12)
if err != nil || len(hdr) < 12 {
return 0, 0
}
size := int64(binary.LittleEndian.Uint32(hdr[4:8]))
return size + 8, 0.9
}
func validateMachO(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 32)
if err != nil || len(hdr) < 28 {
return 0, 0
}
var (
bo binary.ByteOrder
is64 bool
hdrSize int64 = 28
)
switch binary.BigEndian.Uint32(hdr[0:4]) {
case 0xfeedface:
bo = binary.BigEndian
case 0xfeedfacf:
bo = binary.BigEndian
is64 = true
hdrSize = 32
case 0xcefaedfe:
bo = binary.LittleEndian
case 0xcffaedfe:
bo = binary.LittleEndian
is64 = true
hdrSize = 32
default:
return 0, 0
}
if is64 && len(hdr) < 32 {
return 0, 0
}
ncmds := bo.Uint32(hdr[16:20])
sizeofcmds := bo.Uint32(hdr[20:24])
if ncmds == 0 || ncmds > 10000 || sizeofcmds > 128<<20 {
return 0, 0
}
cmdBytes, err := d.Range(start+hdrSize, int64(sizeofcmds))
if err != nil || len(cmdBytes) < int(sizeofcmds) {
return 0, 0
}
end := hdrSize + int64(sizeofcmds)
off := 0
for i := uint32(0); i < ncmds; i++ {
if off+8 > len(cmdBytes) {
return 0, 0
}
cmd := bo.Uint32(cmdBytes[off : off+4])
cmdSize := int(bo.Uint32(cmdBytes[off+4 : off+8]))
if cmdSize < 8 || off+cmdSize > len(cmdBytes) {
return 0, 0
}
switch cmd {
case 0x1: // LC_SEGMENT
if cmdSize >= 56 {
fileOff := int64(bo.Uint32(cmdBytes[off+32 : off+36]))
fileSize := int64(bo.Uint32(cmdBytes[off+36 : off+40]))
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
end = fileOff + fileSize
}
}
case 0x19: // LC_SEGMENT_64
if cmdSize >= 72 {
fileOff := int64(bo.Uint64(cmdBytes[off+40 : off+48]))
fileSize := int64(bo.Uint64(cmdBytes[off+48 : off+56]))
if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end {
end = fileOff + fileSize
}
}
}
off += cmdSize
}
if end <= hdrSize || start+end > max {
return 0, 0
}
return end, 0.85
}
func validateID3(d *document.Document, start, max int64) (int64, float64) {
// ID3v2 header: "ID3" + ver(2) + flags(1) + size(4, synchsafe).
hdr, err := d.Range(start, 10)
if err != nil || len(hdr) < 10 {
return 0, 0.3
}
if hdr[3] == 0xff || hdr[4] == 0xff {
return 0, 0.3
}
// Synchsafe 28-bit size (7 bits per byte).
sz := (int64(hdr[6]&0x7f) << 21) | (int64(hdr[7]&0x7f) << 14) |
(int64(hdr[8]&0x7f) << 7) | int64(hdr[9]&0x7f)
if sz < 0 || sz > 32*1024*1024 {
return 0, 0.4
}
// Skip the ID3v2 tag (header + synchsafe body, plus a 10-byte footer if the
// footer-present flag is set), then walk MPEG audio frames to the end.
audioStart := start + 10 + sz
if hdr[5]&0x10 != 0 {
audioStart += 10
}
if end := walkMP3Frames(d, audioStart, max); end > audioStart {
return end - start, 0.85
}
return 0, 0.7
}
// walkMP3Frames follows MPEG-1/2/2.5 Layer I/II/III frame headers from start
// until a byte that is not a valid frame sync, returning the offset just past
// the last frame. APE/Lyrics3 trailers are not stripped — the audio extent is
// what matters for carving.
func walkMP3Frames(d *document.Document, start, max int64) int64 {
pos := start
frames := 0
for pos+4 <= max && frames < 5_000_000 {
h, err := d.Range(pos, 4)
if err != nil || len(h) < 4 {
break
}
ln := mp3FrameLen(h)
if ln <= 0 {
break
}
pos += ln
frames++
}
if frames == 0 {
return 0
}
if pos > max {
return max
}
return pos
}
var mp3BitrateV1 = [16]int{0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0}
var mp3BitrateV2 = [16]int{0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0}
var mp3SampleRate = map[int][3]int{ // [version]{rate index 0,1,2}
3: {44100, 48000, 32000}, // MPEG1
2: {22050, 24000, 16000}, // MPEG2
0: {11025, 12000, 8000}, // MPEG2.5
}
// mp3FrameLen returns the byte length of the MPEG audio frame whose 4-byte
// header is h, or 0 if h is not a valid frame header.
func mp3FrameLen(h []byte) int64 {
if h[0] != 0xff || h[1]&0xe0 != 0xe0 {
return 0
}
version := int(h[1] >> 3 & 0x03) // 3=MPEG1, 2=MPEG2, 0=MPEG2.5
layer := int(h[1] >> 1 & 0x03) // 3=I, 2=II, 1=III
if version == 1 || layer == 0 {
return 0 // reserved
}
brIndex := int(h[2] >> 4 & 0x0f)
srIndex := int(h[2] >> 2 & 0x03)
padding := int(h[2] >> 1 & 0x01)
if brIndex == 0 || brIndex == 15 || srIndex == 3 {
return 0
}
var bitrate int
if version == 3 {
bitrate = mp3BitrateV1[brIndex]
} else {
bitrate = mp3BitrateV2[brIndex]
}
rates, ok := mp3SampleRate[version]
if !ok || bitrate == 0 {
return 0
}
sampleRate := rates[srIndex]
if sampleRate == 0 {
return 0
}
bps := bitrate * 1000
if layer == 3 { // Layer I
return int64(((12 * bps / sampleRate) + padding) * 4)
}
// Layer II & III. MPEG2/2.5 Layer III uses 72 samples-per-byte factor.
coef := 144
if version != 3 && layer == 1 {
coef = 72
}
return int64(coef*bps/sampleRate + padding)
}
func validateOgg(d *document.Document, start, max int64) (int64, float64) {
off := start
first := true
for off+27 <= max {
hdr, err := d.Range(off, 27)
if err != nil || len(hdr) < 27 || string(hdr[:4]) != "OggS" || hdr[4] != 0 {
return 0, 0
}
headerType := hdr[5]
if first && headerType&0x02 == 0 {
return 0, 0.4
}
segCount := int(hdr[26])
lacing, err := d.Range(off+27, int64(segCount))
if err != nil || len(lacing) < segCount {
return 0, 0
}
payload := int64(0)
for _, v := range lacing {
payload += int64(v)
}
pageLen := int64(27 + segCount)
if payload < 0 || off+pageLen+payload > max {
return 0, 0
}
off += pageLen + payload
if headerType&0x04 != 0 {
return off - start, 0.9
}
first = false
}
return 0, 0.6
}
// validateMatroska reads the EBML header element and the following Segment
// element's size to bound a Matroska/WebM stream. A Segment with "unknown size"
// (all size bits set, as live-streamed files use) cannot be bounded this way.
func validateMatroska(d *document.Document, start, max int64) (int64, float64) {
buf, err := d.Range(start, min64(max-start, 4096))
if err != nil || len(buf) < 8 {
return 0, 0
}
// EBML header: ID 0x1A45DFA3, then a vint size.
if !(buf[0] == 0x1a && buf[1] == 0x45 && buf[2] == 0xdf && buf[3] == 0xa3) {
return 0, 0
}
hsz, n, unknown := ebmlSize(buf, 4)
if n == 0 || unknown {
return 0, 0.6
}
pos := 4 + n + int(hsz) // start of the Segment element
if pos+4 > len(buf) {
return 0, 0.6
}
// Segment: ID 0x18538067, then a vint size.
if !(buf[pos] == 0x18 && buf[pos+1] == 0x53 && buf[pos+2] == 0x80 && buf[pos+3] == 0x67) {
return 0, 0.6
}
segSz, sn, segUnknown := ebmlSize(buf, pos+4)
if sn == 0 || segUnknown {
return 0, 0.6 // live/streamed segment without a known size
}
end := int64(pos+4+sn) + segSz
if end <= 0 || start+end > max {
return 0, 0.6
}
return end, 0.9
}
// ebmlSize decodes an EBML variable-length integer (data size form). The length
// is the number of leading zero bits in the first byte plus one; the value is
// the remaining bits. unknown is true when every value bit is set.
func ebmlSize(buf []byte, pos int) (val int64, n int, unknown bool) {
if pos >= len(buf) || buf[pos] == 0 {
return 0, 0, false
}
first := buf[pos]
mask := byte(0x80)
length := 1
for first&mask == 0 {
mask >>= 1
length++
}
if length > 8 || pos+length > len(buf) {
return 0, 0, false
}
low := mask - 1
v := int64(first & low)
allOnes := (first & low) == low
for i := 1; i < length; i++ {
b := buf[pos+i]
v = (v << 8) | int64(b)
if b != 0xff {
allOnes = false
}
}
return v, length, allOnes
}
func validateFLAC(d *document.Document, start, max int64) (int64, float64) {
return 0, 0.8
}
// validateBMP reads the BITMAPFILEHEADER size field (bfSize), sanity-checking
// the embedded pixel-data offset against the declared length.
func validateBMP(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 14)
if err != nil || len(hdr) < 14 {
return 0, 0
}
size := int64(binary.LittleEndian.Uint32(hdr[2:6]))
dataOff := int64(binary.LittleEndian.Uint32(hdr[10:14]))
if size < 26 || dataOff < 26 || dataOff > size || start+size > max {
return 0, 0
}
return size, 0.8
}
// validateMP4 walks the top-level boxes of an ISO base-media file (MP4/MOV/M4A)
// from the file start (the ftyp box). 64-bit box sizes are honoured.
func validateMP4(d *document.Document, start, max int64) (int64, float64) {
pos := start
boxes := 0
for pos+8 <= max && boxes < 100000 {
h, err := d.Range(pos, 16)
if err != nil || len(h) < 8 {
break
}
size := int64(binary.BigEndian.Uint32(h[0:4]))
switch {
case size == 1: // 64-bit largesize follows the type
if len(h) < 16 {
return 0, 0.4
}
size = int64(binary.BigEndian.Uint64(h[8:16]))
case size == 0: // box extends to end of file
return max - start, 0.8
}
if size < 8 {
break
}
pos += size
boxes++
if pos > max {
return 0, 0.4
}
}
if pos > start && pos <= max && boxes > 0 {
return pos - start, 0.85
}
return 0, 0.4
}
func validateSQLite(d *document.Document, start, max int64) (int64, float64) {
hdr, err := d.Range(start, 32)
if err != nil || len(hdr) < 32 {
return 0, 0
}
pageSize := int64(binary.BigEndian.Uint16(hdr[16:18]))
if pageSize == 1 {
pageSize = 65536
}
pages := int64(binary.BigEndian.Uint32(hdr[28:32]))
if pages == 0 || pageSize == 0 {
return 0, 0.5
}
return pageSize * pages, 0.9
}
func min64(a, b int64) int64 {
if a < b {
return a
}
return b
}
internal/leviathan/analysis/diff.go
+244
View File
@@ -0,0 +1,244 @@
package analysis
import (
"context"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// DiffRegion describes a contiguous run of differing bytes.
type DiffRegion struct {
Offset int64 `json:"offset"`
Length int64 `json:"length"`
// kinds: "edit" (both sides have bytes), "add" (only B), "remove" (only A)
Kind string `json:"kind"`
}
type DiffBlock struct {
Offset int64 `json:"offset"`
Length int64 `json:"length"`
ChangedBytes int64 `json:"changedBytes"`
EditBytes int64 `json:"editBytes"`
AddBytes int64 `json:"addBytes"`
RemoveBytes int64 `json:"removeBytes"`
Similarity float64 `json:"similarity"`
}
// Diff compares two documents by byte. It handles documents of
// different sizes by treating the tail of the longer as add/remove.
// Chunks at both sides. Returns a compact list of regions.
func Diff(a, b *document.Document, start, length int64) (map[string]any, error) {
return DiffContext(context.Background(), a, b, start, length)
}
func DiffContext(ctx context.Context, a, b *document.Document, start, length int64) (map[string]any, error) {
aSize := a.Size()
bSize := b.Size()
maxSize := aSize
if bSize > maxSize {
maxSize = bSize
}
if length < 0 {
length = maxSize - start
}
end := start + length
if end > maxSize {
end = maxSize
}
const chunk = 1 << 20
regions := make([]DiffRegion, 0, 64)
var inRun bool
var runStart int64
var runKind string
flush := func(off int64) {
if inRun {
regions = append(regions, DiffRegion{Offset: runStart, Length: off - runStart, Kind: runKind})
inRun = false
}
}
for off := start; off < end; {
if err := ctx.Err(); err != nil {
return nil, err
}
winEnd := off + chunk
if winEnd > end {
winEnd = end
}
// Read both sides, tolerating short reads for the shorter file.
var ab, bb []byte
if off < aSize {
n := winEnd - off
if off+n > aSize {
n = aSize - off
}
var err error
ab, err = a.Range(off, n)
if err != nil {
return nil, err
}
}
if off < bSize {
n := winEnd - off
if off+n > bSize {
n = bSize - off
}
var err error
bb, err = b.Range(off, n)
if err != nil {
return nil, err
}
}
for i := int64(0); i < winEnd-off; i++ {
if i%4096 == 0 {
if err := ctx.Err(); err != nil {
return nil, err
}
}
curOff := off + i
var av, bv byte
var av_ok, bv_ok bool
if curOff < aSize && int64(len(ab)) > i {
av = ab[i]
av_ok = true
}
if curOff < bSize && int64(len(bb)) > i {
bv = bb[i]
bv_ok = true
}
kind := ""
if av_ok && bv_ok {
if av != bv {
kind = "edit"
}
} else if av_ok && !bv_ok {
kind = "remove"
} else if !av_ok && bv_ok {
kind = "add"
}
if kind == "" {
flush(curOff)
continue
}
if !inRun {
inRun = true
runStart = curOff
runKind = kind
} else if kind != runKind {
flush(curOff)
inRun = true
runStart = curOff
runKind = kind
}
}
off = winEnd
}
flush(end)
// Convert to JS-friendly output + summary
out := make([]map[string]any, len(regions))
editBytes := int64(0)
addBytes := int64(0)
removeBytes := int64(0)
for i, r := range regions {
out[i] = map[string]any{
"offset": r.Offset,
"length": r.Length,
"kind": r.Kind,
}
switch r.Kind {
case "edit":
editBytes += r.Length
case "add":
addBytes += r.Length
case "remove":
removeBytes += r.Length
}
}
const blockSize int64 = 4096
blocks := diffBlocksFromRegions(regions, start, end, blockSize)
blockOut := make([]map[string]any, len(blocks))
for i, b := range blocks {
blockOut[i] = map[string]any{
"offset": b.Offset,
"length": b.Length,
"changedBytes": b.ChangedBytes,
"editBytes": b.EditBytes,
"addBytes": b.AddBytes,
"removeBytes": b.RemoveBytes,
"similarity": b.Similarity,
}
}
return map[string]any{
"regions": out,
"regionCount": len(regions),
"blocks": blockOut,
"blockCount": len(blocks),
"blockSize": blockSize,
"editBytes": editBytes,
"addBytes": addBytes,
"removeBytes": removeBytes,
"aSize": aSize,
"bSize": bSize,
}, nil
}
func diffBlocksFromRegions(regions []DiffRegion, start, end, blockSize int64) []DiffBlock {
if blockSize <= 0 || end <= start || len(regions) == 0 {
return nil
}
out := []DiffBlock{}
for alignedStart := start - start%blockSize; alignedStart < end; alignedStart += blockSize {
blockStart := alignedStart
blockEnd := alignedStart + blockSize
if blockEnd > end {
blockEnd = end
}
if blockEnd <= start {
continue
}
if blockStart < start {
blockStart = start
}
b := DiffBlock{Offset: blockStart, Length: blockEnd - blockStart}
for _, r := range regions {
rEnd := r.Offset + r.Length
if rEnd <= blockStart {
continue
}
if r.Offset >= blockEnd {
break
}
ovStart := r.Offset
if ovStart < blockStart {
ovStart = blockStart
}
ovEnd := rEnd
if ovEnd > blockEnd {
ovEnd = blockEnd
}
changed := ovEnd - ovStart
if changed <= 0 {
continue
}
b.ChangedBytes += changed
switch r.Kind {
case "edit":
b.EditBytes += changed
case "add":
b.AddBytes += changed
case "remove":
b.RemoveBytes += changed
}
}
if b.ChangedBytes == 0 {
continue
}
b.Similarity = 1 - float64(b.ChangedBytes)/float64(b.Length)
out = append(out, b)
}
return out
}
internal/leviathan/analysis/entropy.go
+125
View File
@@ -0,0 +1,125 @@
package analysis
import (
"context"
"math"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Entropy computes Shannon entropy per sliding window over the range.
//
// The returned map has:
//
// windowSize: int
// step: int
// samples: []float64 — entropy per window (0..8)
// offsets: []int64 — start offset of each window
// mean/min/max: summary stats
// highCount: number of windows with entropy >= 7.5 (candidate compressed/encrypted)
func Entropy(d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
return EntropyContext(context.Background(), d, start, length, windowSize, step)
}
func EntropyContext(ctx context.Context, d *document.Document, start, length, windowSize, step int64) (map[string]any, error) {
if windowSize <= 0 {
windowSize = 4096
}
if step <= 0 {
step = windowSize
}
if length < 0 {
length = d.Size() - start
}
if length <= 0 {
return map[string]any{
"windowSize": windowSize, "step": step,
"samples": []float64{}, "offsets": []int64{},
"mean": 0.0, "min": 0.0, "max": 0.0, "highCount": 0,
}, nil
}
// We run in a streaming fashion: advance through chunks, refill
// a rolling window, and emit a sample whenever the cursor hits
// a step boundary.
samples := make([]float64, 0, length/step+1)
offsets := make([]int64, 0, length/step+1)
// For bounded memory we read the document range chunk by chunk
// but we still need a window of size `windowSize`. Windows can be
// as small as 256 or as large as 1MB; we cap to length.
if windowSize > length {
windowSize = length
}
// Simple approach: iterate in step-sized hops, reading window
// bytes at each hop. This scales O(n * (w/step)) but window and
// step are usually the same for UI.
mean := 0.0
minE := math.Inf(1)
maxE := math.Inf(-1)
high := 0
for off := start; off < start+length; off += step {
if err := ctx.Err(); err != nil {
return nil, err
}
winLen := windowSize
if off+winLen > start+length {
winLen = start + length - off
}
buf, err := d.Range(off, winLen)
if err != nil {
return nil, err
}
e := shannon(buf)
samples = append(samples, e)
offsets = append(offsets, off)
mean += e
if e < minE {
minE = e
}
if e > maxE {
maxE = e
}
if e >= 7.5 {
high++
}
}
if len(samples) > 0 {
mean /= float64(len(samples))
} else {
minE, maxE = 0, 0
}
return map[string]any{
"windowSize": windowSize,
"step": step,
"samples": samples,
"offsets": offsets,
"mean": mean,
"min": minE,
"max": maxE,
"highCount": high,
}, nil
}
func shannon(buf []byte) float64 {
if len(buf) == 0 {
return 0
}
var counts [256]int
for _, b := range buf {
counts[b]++
}
n := float64(len(buf))
e := 0.0
for _, c := range counts {
if c == 0 {
continue
}
p := float64(c) / n
e -= p * math.Log2(p)
}
return e
}
internal/leviathan/analysis/fuzzy.go
+224
View File
@@ -0,0 +1,224 @@
package analysis
import (
"fmt"
"strconv"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// fuzzy.go implements context-triggered piecewise hashing (CTPH) compatible
// with ssdeep. CTPH produces a short signature of the form
// "blocksize:hash1:hash2" that stays similar when the input changes only a
// little, so investigators can cluster related samples or match a file
// against a known-bad ssdeep hash even after minor edits.
//
// The signature format and the rolling/sum hashes follow Andrew Tridgell's
// spamsum / Jesse Kornblum's ssdeep so the output interoperates with the
// standard tooling (e.g. an empty input hashes to "3::").
const (
spamsumLength = 64
rollingWindow = 7
minBlocksize = 3
hashInit = uint32(0x28021967)
hashPrime = uint32(0x01000193)
)
const b64alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"
type rollState struct {
h1, h2, h3 uint32
window [rollingWindow]byte
n int
}
func (r *rollState) roll(c byte) uint32 {
r.h2 -= r.h1
r.h2 += rollingWindow * uint32(c)
r.h1 += uint32(c)
r.h1 -= uint32(r.window[r.n%rollingWindow])
r.window[r.n%rollingWindow] = c
r.n++
r.h3 = r.h3 << 5
r.h3 ^= uint32(c)
return r.h1 + r.h2 + r.h3
}
func sumHash(c byte, h uint32) uint32 { return (h * hashPrime) ^ uint32(c) }
// FuzzyDigest computes the ssdeep-compatible signature for data.
func FuzzyDigest(data []byte) string {
bs := uint32(minBlocksize)
for uint64(bs)*spamsumLength < uint64(len(data)) {
bs *= 2
}
for {
sig1, sig2 := ctphSigs(data, bs)
if bs > minBlocksize && len(sig1) < spamsumLength/2 {
bs /= 2
continue
}
return fmt.Sprintf("%d:%s:%s", bs, sig1, sig2)
}
}
func ctphSigs(data []byte, bs uint32) (string, string) {
var rs rollState
h1, h2 := hashInit, hashInit
var sig1, sig2 strings.Builder
var lastRoll uint32
for _, c := range data {
h1 = sumHash(c, h1)
h2 = sumHash(c, h2)
rh := rs.roll(c)
lastRoll = rh
if rh%bs == bs-1 && sig1.Len() < spamsumLength-1 {
sig1.WriteByte(b64alphabet[h1%64])
h1 = hashInit
}
if rh%(bs*2) == bs*2-1 && sig2.Len() < spamsumLength/2-1 {
sig2.WriteByte(b64alphabet[h2%64])
h2 = hashInit
}
}
if lastRoll != 0 {
sig1.WriteByte(b64alphabet[h1%64])
sig2.WriteByte(b64alphabet[h2%64])
}
return sig1.String(), sig2.String()
}
// FuzzyHash computes the CTPH signature over a document range.
func FuzzyHash(d *document.Document, start, length int64) (map[string]any, error) {
if length < 0 {
length = d.Size() - start
}
buf, err := d.Range(start, length)
if err != nil {
return nil, err
}
sig := FuzzyDigest(buf)
bs := strings.SplitN(sig, ":", 2)[0]
return map[string]any{
"algo": "ssdeep",
"signature": sig,
"blocksize": bs,
"length": length,
}, nil
}
// FuzzyCompare returns a 0-100 similarity score between two ssdeep
// signatures. 0 means unrelated (or incomparable block sizes); 100 means the
// signatures are identical.
func FuzzyCompare(a, b string) int {
bs1, a1, a2, ok1 := parseSig(a)
bs2, b1, b2, ok2 := parseSig(b)
if !ok1 || !ok2 {
return 0
}
if bs1 != bs2 && bs1 != bs2*2 && bs2 != bs1*2 {
return 0
}
switch {
case bs1 == bs2:
s1 := scoreStrings(a1, b1, bs1)
s2 := scoreStrings(a2, b2, bs1*2)
if s2 > s1 {
return s2
}
return s1
case bs1 == bs2*2:
return scoreStrings(a1, b2, bs1)
default: // bs2 == bs1*2
return scoreStrings(a2, b1, bs2)
}
}
func parseSig(s string) (uint32, string, string, bool) {
parts := strings.SplitN(s, ":", 3)
if len(parts) != 3 {
return 0, "", "", false
}
bs, err := strconv.ParseUint(parts[0], 10, 32)
if err != nil || bs == 0 {
return 0, "", "", false
}
return uint32(bs), parts[1], parts[2], true
}
func scoreStrings(s1, s2 string, blocksize uint32) int {
if !hasCommonSubstring(s1, s2, rollingWindow) {
return 0
}
if s1 == s2 {
return 100
}
d := editDistance(s1, s2)
if len(s1)+len(s2) == 0 {
return 0
}
score := (d * spamsumLength) / (len(s1) + len(s2))
score = (100 * score) / spamsumLength
score = 100 - score
// Small block sizes describe little data, so cap the score they can claim.
cap := int(blocksize) / minBlocksize * minInt(len(s1), len(s2))
if score > cap {
score = cap
}
if score > 100 {
score = 100
}
if score < 0 {
score = 0
}
return score
}
func hasCommonSubstring(s1, s2 string, n int) bool {
if len(s1) < n || len(s2) < n {
return false
}
seen := make(map[string]struct{}, len(s1))
for i := 0; i+n <= len(s1); i++ {
seen[s1[i:i+n]] = struct{}{}
}
for i := 0; i+n <= len(s2); i++ {
if _, ok := seen[s2[i:i+n]]; ok {
return true
}
}
return false
}
// editDistance is classic Levenshtein with unit insert/delete/substitute cost.
func editDistance(s1, s2 string) int {
m, n := len(s1), len(s2)
prev := make([]int, n+1)
cur := make([]int, n+1)
for j := 0; j <= n; j++ {
prev[j] = j
}
for i := 1; i <= m; i++ {
cur[0] = i
for j := 1; j <= n; j++ {
cost := 1
if s1[i-1] == s2[j-1] {
cost = 0
}
cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost)
}
prev, cur = cur, prev
}
return prev[n]
}
func minInt(a, b int) int {
if a < b {
return a
}
return b
}
internal/leviathan/analysis/hash.go
+63
View File
@@ -0,0 +1,63 @@
package analysis
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/hex"
"hash"
"hash/crc32"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// HashSet contains CRC32/MD5/SHA-1/SHA-256/SHA-512 for a range.
type HashSet struct {
Length int64 `json:"length"`
CRC32 string `json:"crc32"`
MD5 string `json:"md5"`
SHA1 string `json:"sha1"`
SHA256 string `json:"sha256"`
SHA512 string `json:"sha512"`
}
// Hashes computes all supported hashes over the document range.
// The computation runs chunk-by-chunk so large files do not force a
// full-buffer duplication.
func Hashes(d *document.Document, start, length int64) (map[string]any, error) {
if length < 0 {
length = d.Size() - start
}
crc := crc32.NewIEEE()
mh := md5.New()
s1 := sha1.New()
s256 := sha256.New()
s512 := sha512.New()
writers := []hash.Hash{mh, s1, s256, s512}
total := int64(0)
err := d.ForEachChunk(start, length, func(_ int64, chunk []byte) error {
crc.Write(chunk)
for _, w := range writers {
w.Write(chunk)
}
total += int64(len(chunk))
return nil
})
if err != nil {
return nil, err
}
return map[string]any{
"length": total,
"crc32": toHex(crc.Sum(nil)),
"md5": toHex(mh.Sum(nil)),
"sha1": toHex(s1.Sum(nil)),
"sha256": toHex(s256.Sum(nil)),
"sha512": toHex(s512.Sum(nil)),
}, nil
}
func toHex(b []byte) string { return hex.EncodeToString(b) }
internal/leviathan/analysis/inspector.go
+162
View File
@@ -0,0 +1,162 @@
package analysis
import (
"encoding/binary"
"fmt"
"math"
"time"
"unicode/utf16"
"unicode/utf8"
)
// Inspect interprets bytes starting at `buf[0]` as many candidate
// types. It is resilient: types that don't have enough bytes simply
// return empty strings.
func Inspect(buf []byte, offset int64) map[string]any {
out := map[string]any{
"offset": offset,
"length": len(buf),
}
// 8-bit
if len(buf) >= 1 {
out["u8"] = int(buf[0])
out["i8"] = int(int8(buf[0]))
out["char"] = printable(buf[0])
out["bin8"] = fmt.Sprintf("%08b", buf[0])
}
// 16-bit
if len(buf) >= 2 {
le16 := binary.LittleEndian.Uint16(buf)
be16 := binary.BigEndian.Uint16(buf)
out["u16_le"] = int(le16)
out["u16_be"] = int(be16)
out["i16_le"] = int(int16(le16))
out["i16_be"] = int(int16(be16))
}
// 32-bit
if len(buf) >= 4 {
le32 := binary.LittleEndian.Uint32(buf)
be32 := binary.BigEndian.Uint32(buf)
out["u32_le"] = uint(le32)
out["u32_be"] = uint(be32)
out["i32_le"] = int(int32(le32))
out["i32_be"] = int(int32(be32))
out["f32_le"] = float64(math.Float32frombits(le32))
out["f32_be"] = float64(math.Float32frombits(be32))
}
// 64-bit
if len(buf) >= 8 {
le64 := binary.LittleEndian.Uint64(buf)
be64 := binary.BigEndian.Uint64(buf)
out["u64_le"] = float64(le64)
out["u64_be"] = float64(be64)
out["i64_le"] = float64(int64(le64))
out["i64_be"] = float64(int64(be64))
out["f64_le"] = math.Float64frombits(le64)
out["f64_be"] = math.Float64frombits(be64)
}
// Timestamp guesses
if len(buf) >= 4 {
le := binary.LittleEndian.Uint32(buf)
out["unix_le"] = formatUnix(int64(le))
}
if len(buf) >= 8 {
le := binary.LittleEndian.Uint64(buf)
out["filetime_le"] = formatFiletime(le)
out["webkit_le"] = formatWebkit(le)
out["oletime_le"] = formatOLEDate(math.Float64frombits(le))
}
// ASCII preview (up to 32 bytes)
n := len(buf)
if n > 32 {
n = 32
}
ascii := make([]byte, n)
for i := 0; i < n; i++ {
if buf[i] >= 0x20 && buf[i] <= 0x7e {
ascii[i] = buf[i]
} else {
ascii[i] = '.'
}
}
out["ascii"] = string(ascii)
// UTF-8 attempt
if utf8.Valid(buf[:n]) {
out["utf8"] = string(buf[:n])
}
// UTF-16 LE/BE preview
if len(buf) >= 2 {
lu := n / 2
if lu > 0 {
u := make([]uint16, lu)
for i := 0; i < lu; i++ {
u[i] = uint16(buf[i*2]) | uint16(buf[i*2+1])<<8
}
out["utf16le"] = string(utf16.Decode(u))
for i := 0; i < lu; i++ {
u[i] = uint16(buf[i*2])<<8 | uint16(buf[i*2+1])
}
out["utf16be"] = string(utf16.Decode(u))
}
}
return out
}
func printable(b byte) string {
if b >= 0x20 && b <= 0x7e {
return string([]byte{b})
}
return "."
}
func formatUnix(sec int64) string {
if sec < 0 || sec > 4102444800 { // past 2100
return ""
}
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
}
// Windows FILETIME epoch 1601-01-01 in 100ns ticks.
const filetimeEpochDelta = 11644473600
func formatFiletime(v uint64) string {
if v == 0 {
return ""
}
sec := int64(v/10_000_000) - filetimeEpochDelta
if sec < 0 || sec > 4102444800 {
return ""
}
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
}
// WebKit/Chrome epoch 1601-01-01 in microseconds.
func formatWebkit(v uint64) string {
if v == 0 {
return ""
}
sec := int64(v/1_000_000) - filetimeEpochDelta
if sec < 0 || sec > 4102444800 {
return ""
}
return time.Unix(sec, 0).UTC().Format(time.RFC3339)
}
// OLE Automation date: days since 1899-12-30, as float64.
func formatOLEDate(days float64) string {
if days < 2 || days > 75000 {
return ""
}
base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
sec := int64(days * 86400)
return base.Add(time.Duration(sec) * time.Second).Format(time.RFC3339)
}
internal/leviathan/analysis/ioc.go
+89
View File
@@ -0,0 +1,89 @@
package analysis
import (
"context"
"regexp"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ioc regexes. They run against printable ASCII strings extracted
// from the document, which is the standard approach for IOC hunting
// and avoids matching noise in raw binary data.
var iocRegexes = map[string]*regexp.Regexp{
"url": regexp.MustCompile(`(?i)\b(?:https?|ftp)://[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`),
"domain": regexp.MustCompile(`(?i)\b(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.)+(?:com|net|org|io|co|uk|de|fr|ru|cn|info|xyz|app|dev|sh|me|gov|mil|edu|biz)\b`),
"email": regexp.MustCompile(`(?i)\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b`),
"ipv4": regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`),
"ipv6": regexp.MustCompile(`\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b`),
"guid": regexp.MustCompile(`\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`),
"md5": regexp.MustCompile(`\b[a-fA-F0-9]{32}\b`),
"sha1": regexp.MustCompile(`\b[a-fA-F0-9]{40}\b`),
"sha256": regexp.MustCompile(`\b[a-fA-F0-9]{64}\b`),
"winpath": regexp.MustCompile(`[A-Za-z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*`),
"unc": regexp.MustCompile(`\\\\[A-Za-z0-9._\-]+(?:\\[^\\/:*?"<>|\r\n]+)+`),
"unixpath": regexp.MustCompile(`(?:/(?:etc|var|usr|opt|home|tmp|root|srv|bin|sbin|lib|lib64)(?:/[A-Za-z0-9._\-]+)+)`),
"regkey": regexp.MustCompile(`(?i)\b(?:HKLM|HKCU|HKCR|HKU|HKCC|HKEY_[A-Z_]+)\\[^\s"<>]+`),
"onion": regexp.MustCompile(`(?i)\b[a-z2-7]{16,56}\.onion\b`),
"btc": regexp.MustCompile(`\b(?:[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-z0-9]{20,90})\b`),
}
// ExtractIOCs runs the IOC regexes over ASCII strings first and then
// UTF-16 LE/BE strings. Returns a map of category -> slice of hits
// with offsets into the original file.
func ExtractIOCs(d *document.Document, start, length int64, minLen int) (map[string]any, error) {
return ExtractIOCsContext(context.Background(), d, start, length, minLen)
}
func ExtractIOCsContext(ctx context.Context, d *document.Document, start, length int64, minLen int) (map[string]any, error) {
if length < 0 {
length = d.Size() - start
}
if minLen <= 0 {
minLen = 5
}
strings, err := ExtractStringsContext(ctx, d, start, length, minLen, []string{"ascii", "utf16le", "utf16be"}, 200_000)
if err != nil {
return nil, err
}
result := map[string]any{}
totals := map[string]int{}
for category, re := range iocRegexes {
if err := ctx.Err(); err != nil {
return nil, err
}
hits := make([]map[string]any, 0)
for _, s := range strings {
if err := ctx.Err(); err != nil {
return nil, err
}
str, _ := s["value"].(string)
base, _ := s["offset"].(int64)
for _, loc := range re.FindAllStringIndex(str, -1) {
hits = append(hits, map[string]any{
"category": category,
"offset": base + int64(loc[0]),
"length": loc[1] - loc[0],
"value": str[loc[0]:loc[1]],
})
}
}
result[category] = hits
totals[category] = len(hits)
}
// flat list for UI convenience
all := []map[string]any{}
for _, v := range result {
if s, ok := v.([]map[string]any); ok {
all = append(all, s...)
}
}
return map[string]any{
"byCategory": result,
"all": all,
"totals": totals,
"totalStrings": len(strings),
}, nil
}
internal/leviathan/analysis/keys.go
+367
View File
@@ -0,0 +1,367 @@
package analysis
import (
"bytes"
"crypto/ecdsa"
"crypto/ed25519"
"crypto/rsa"
"crypto/x509"
"encoding/pem"
"fmt"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// KeyArtefact is a single key-ish object we've found in the raw file.
// Length is the span we recognised; Note carries any decoding detail.
type KeyArtefact struct {
Kind string `json:"kind"`
Format string `json:"format"` // "pem" | "der" | "ssh" | "p12"
Offset int64 `json:"offset"`
Length int64 `json:"length"`
Summary string `json:"summary"`
Note string `json:"note,omitempty"`
}
// ScanKeys walks the whole document looking for cryptographic artefacts:
// - PEM blocks with common key-related labels
// - OpenSSH keys (authorized_keys lines and the "openssh-key-v1" private-key envelope)
// - PKCS#12 magic (1.2.840.113549.1.12.1 appears in headers)
// - Loose DER SEQUENCEs that parse as RSA/EC/PKCS#8 keys
//
// We cap the scan at 64 MiB. Users with larger blobs can select a range
// and call this against that subset via a future "keys.scanRange" bridge.
func ScanKeys(d *document.Document) ([]KeyArtefact, error) {
size := d.Size()
if size > 64*1024*1024 {
return nil, fmt.Errorf("keys.scan: file > 64 MiB; use selection")
}
buf, err := d.Range(0, size)
if err != nil {
return nil, err
}
out := []KeyArtefact{}
out = append(out, scanPEMKeys(buf)...)
out = append(out, scanOpenSSHPubKeys(buf)...)
out = append(out, scanOpenSSHPrivKeys(buf)...)
out = append(out, scanPKCS12(buf)...)
// Loose DER pass — for raw PKCS#8 / PKCS#1 blobs that aren't wrapped in PEM.
out = append(out, scanLooseDERKeys(buf)...)
return out, nil
}
// Known PEM labels that indicate key-ish content.
var pemKeyLabels = map[string]string{
"RSA PRIVATE KEY": "rsa-private",
"RSA PUBLIC KEY": "rsa-public",
"EC PRIVATE KEY": "ec-private",
"EC PUBLIC KEY": "ec-public",
"DSA PRIVATE KEY": "dsa-private",
"PRIVATE KEY": "pkcs8-private",
"ENCRYPTED PRIVATE KEY": "pkcs8-encrypted",
"PUBLIC KEY": "pkix-public",
"OPENSSH PRIVATE KEY": "openssh-private",
"SSH2 PUBLIC KEY": "ssh2-public",
"PGP PRIVATE KEY BLOCK": "pgp-private",
"PGP PUBLIC KEY BLOCK": "pgp-public",
"CERTIFICATE REQUEST": "csr",
"NEW CERTIFICATE REQUEST": "csr",
"DH PARAMETERS": "dh-params",
"PARAMETERS": "params",
}
func scanPEMKeys(buf []byte) []KeyArtefact {
out := []KeyArtefact{}
rest := buf
origLen := int64(len(buf))
for {
block, next := pem.Decode(rest)
if block == nil {
break
}
beginIdx := bytes.Index(rest, []byte("-----BEGIN"))
if beginIdx < 0 {
break
}
start := origLen - int64(len(rest)) + int64(beginIdx)
end := origLen - int64(len(next))
kind, match := pemKeyLabels[block.Type]
if !match {
rest = next
continue
}
art := KeyArtefact{
Kind: kind, Format: "pem",
Offset: start, Length: end - start,
}
art.Summary = describePEMBlock(block)
out = append(out, art)
rest = next
}
return out
}
func describePEMBlock(b *pem.Block) string {
name := b.Type
switch b.Type {
case "RSA PRIVATE KEY":
if k, err := x509.ParsePKCS1PrivateKey(b.Bytes); err == nil {
return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen())
}
case "RSA PUBLIC KEY":
if k, err := x509.ParsePKCS1PublicKey(b.Bytes); err == nil {
return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen())
}
case "EC PRIVATE KEY":
if k, err := x509.ParseECPrivateKey(b.Bytes); err == nil {
return fmt.Sprintf("%s — curve %s", name, k.Curve.Params().Name)
}
case "PRIVATE KEY":
if k, err := x509.ParsePKCS8PrivateKey(b.Bytes); err == nil {
return fmt.Sprintf("%s — %s", name, describePrivateKey(k))
}
case "PUBLIC KEY":
if k, err := x509.ParsePKIXPublicKey(b.Bytes); err == nil {
return fmt.Sprintf("%s — %s", name, describePublicKey(k))
}
case "OPENSSH PRIVATE KEY":
return fmt.Sprintf("%s (%d B payload)", name, len(b.Bytes))
case "ENCRYPTED PRIVATE KEY":
return fmt.Sprintf("%s — PKCS#8 encrypted (%d B)", name, len(b.Bytes))
}
return fmt.Sprintf("%s (%d B)", name, len(b.Bytes))
}
func describePrivateKey(k any) string {
switch key := k.(type) {
case *rsa.PrivateKey:
return fmt.Sprintf("RSA %d-bit", key.N.BitLen())
case *ecdsa.PrivateKey:
return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name)
case ed25519.PrivateKey:
return "Ed25519"
}
return fmt.Sprintf("%T", k)
}
func describePublicKey(k any) string {
switch key := k.(type) {
case *rsa.PublicKey:
return fmt.Sprintf("RSA %d-bit", key.N.BitLen())
case *ecdsa.PublicKey:
return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name)
case ed25519.PublicKey:
return "Ed25519"
}
return fmt.Sprintf("%T", k)
}
// OpenSSH authorized_keys / id_*.pub format is one line per key:
// "<algo> <base64> [comment]\n". We search for well-known algos and
// delimit by the next newline. False positives on text logs are possible,
// so we validate by base64-decoding.
var sshAlgos = []string{
"ssh-rsa", "ssh-dss", "ssh-ed25519",
"ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521",
"sk-ssh-ed25519@openssh.com", "sk-ecdsa-sha2-nistp256@openssh.com",
}
func scanOpenSSHPubKeys(buf []byte) []KeyArtefact {
out := []KeyArtefact{}
for _, algo := range sshAlgos {
needle := []byte(algo + " ")
off := 0
for {
idx := bytes.Index(buf[off:], needle)
if idx < 0 {
break
}
absStart := off + idx
// Must be at start of line or after whitespace.
if absStart > 0 {
prev := buf[absStart-1]
if prev != '\n' && prev != '\r' && prev != ' ' && prev != '\t' {
off = absStart + 1
continue
}
}
// End of line.
endOfLine := absStart + len(needle)
for endOfLine < len(buf) && buf[endOfLine] != '\n' && buf[endOfLine] != '\r' {
endOfLine++
}
line := buf[absStart:endOfLine]
// Expect at least one space-separated payload that decodes.
parts := bytes.SplitN(line, []byte(" "), 3)
if len(parts) < 2 || len(parts[1]) < 16 || !isBase64Like(parts[1]) {
off = absStart + 1
continue
}
art := KeyArtefact{
Kind: "openssh-public", Format: "ssh",
Offset: int64(absStart), Length: int64(len(line)),
Summary: fmt.Sprintf("%s (%d-byte payload b64)", algo, len(parts[1])),
}
if len(parts) == 3 && len(parts[2]) > 0 {
art.Note = "comment: " + string(parts[2])
}
out = append(out, art)
off = endOfLine
}
}
return out
}
func scanOpenSSHPrivKeys(buf []byte) []KeyArtefact {
// Already caught by PEM scan (label "OPENSSH PRIVATE KEY"). We also
// look for the magic string inside the PEM body — "openssh-key-v1\0".
out := []KeyArtefact{}
needle := []byte("openssh-key-v1\x00")
off := 0
for {
idx := bytes.Index(buf[off:], needle)
if idx < 0 {
break
}
absStart := off + idx
out = append(out, KeyArtefact{
Kind: "openssh-private-magic", Format: "ssh",
Offset: int64(absStart), Length: int64(len(needle)),
Summary: "OpenSSH v1 private-key magic",
Note: "magic only; no envelope length inferred",
})
off = absStart + len(needle)
}
return out
}
// PKCS#12 (.p12 / .pfx) files begin with an ASN.1 SEQUENCE; the second
// element is an OID for the bag type. Its encoded form is reasonably
// distinctive: "\x06\x0b\x2a\x86\x48\x86\xf7\x0d\x01\x07\x01" (data).
// The simplest reliable magic for our purposes is the OID for
// pkcs-12 (1.2.840.113549.1.12.1): "\x06\x0a\x2a\x86\x48\x86\xf7\x0d\x01\x0c\x01".
func scanPKCS12(buf []byte) []KeyArtefact {
out := []KeyArtefact{}
needle := []byte{0x06, 0x0a, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x0c, 0x01}
off := 0
for {
idx := bytes.Index(buf[off:], needle)
if idx < 0 {
break
}
absStart := off + idx
out = append(out, KeyArtefact{
Kind: "pkcs12-oid", Format: "p12",
Offset: int64(absStart), Length: int64(len(needle)),
Summary: "pkcs-12 OID (1.2.840.113549.1.12.1)",
Note: "bag-type marker; may indicate a .p12 or .pfx container",
})
off = absStart + len(needle)
}
return out
}
// scanLooseDERKeys walks DER SEQUENCE headers and tries to parse each
// candidate as a known key format. False negatives are fine; we only
// emit an artefact on a positive parse.
func scanLooseDERKeys(buf []byte) []KeyArtefact {
out := []KeyArtefact{}
// Use the same SEQUENCE 0x30 0x82 xx yy (long-form, 2 length bytes)
// pattern as the cert scanner. Minimum useful key length ~200 bytes.
//
// Each candidate runs up to six x509 parse attempts over a blob up to
// 64 KiB, so an unfiltered scan of a buffer dense with "30 82" pairs is
// O(n) expensive parses and could freeze the UI. A cheap structural
// pre-filter (a DER key body begins with a SEQUENCE 0x30 or an INTEGER
// 0x02) rejects almost all false candidates without parsing, and a hard
// cap bounds adversarial input that defeats the pre-filter.
const maxDERAttempts = 1 << 16
derAttempts := 0
for i := 0; i+4 < len(buf); i++ {
if buf[i] != 0x30 || buf[i+1] != 0x82 {
continue
}
if c := buf[i+4]; c != 0x30 && c != 0x02 {
continue
}
total := int(buf[i+2])<<8 | int(buf[i+3])
if total < 80 || i+4+total > len(buf) {
continue
}
if derAttempts++; derAttempts > maxDERAttempts {
break
}
blob := buf[i : i+4+total]
// Skip anything that looks like a certificate (already caught by certs panel).
if _, err := x509.ParseCertificate(blob); err == nil {
i += 4 + total - 1
continue
}
if _, err := x509.ParsePKCS1PrivateKey(blob); err == nil {
out = append(out, KeyArtefact{
Kind: "rsa-private", Format: "der",
Offset: int64(i), Length: int64(4 + total),
Summary: "RSA private key (PKCS#1 DER)",
})
i += 4 + total - 1
continue
}
if _, err := x509.ParsePKCS1PublicKey(blob); err == nil {
out = append(out, KeyArtefact{
Kind: "rsa-public", Format: "der",
Offset: int64(i), Length: int64(4 + total),
Summary: "RSA public key (PKCS#1 DER)",
})
i += 4 + total - 1
continue
}
if k, err := x509.ParseECPrivateKey(blob); err == nil {
out = append(out, KeyArtefact{
Kind: "ec-private", Format: "der",
Offset: int64(i), Length: int64(4 + total),
Summary: "EC private key (SEC1 DER) curve=" + k.Curve.Params().Name,
})
i += 4 + total - 1
continue
}
if k, err := x509.ParsePKCS8PrivateKey(blob); err == nil {
out = append(out, KeyArtefact{
Kind: "pkcs8-private", Format: "der",
Offset: int64(i), Length: int64(4 + total),
Summary: "PKCS#8 private key — " + describePrivateKey(k),
})
i += 4 + total - 1
continue
}
if k, err := x509.ParsePKIXPublicKey(blob); err == nil {
out = append(out, KeyArtefact{
Kind: "pkix-public", Format: "der",
Offset: int64(i), Length: int64(4 + total),
Summary: "PKIX public key — " + describePublicKey(k),
})
i += 4 + total - 1
continue
}
}
return out
}
func isBase64Like(b []byte) bool {
if len(b) == 0 {
return false
}
for _, c := range b {
if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') ||
c == '+' || c == '/' || c == '=' {
continue
}
return false
}
return true
}
// Stringer helper used by the bridge; avoids leaking internal types to JS.
func (a KeyArtefact) String() string {
return strings.TrimSpace(fmt.Sprintf("%s %s @0x%x %s", a.Kind, a.Format, a.Offset, a.Summary))
}
internal/leviathan/analysis/multisearch.go
+113
View File
@@ -0,0 +1,113 @@
package analysis
import (
"encoding/hex"
"errors"
"fmt"
"regexp"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
type MultiSearchQuery struct {
Mode string
Query string
}
func ParseMultiSearchSpec(spec string) ([]MultiSearchQuery, error) {
spec = strings.ReplaceAll(spec, ";", "\n")
out := []MultiSearchQuery{}
for _, line := range strings.Split(spec, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") {
continue
}
parts := strings.Fields(line)
if len(parts) < 2 {
return nil, errors.New("multi-search lines require: text|hex|regex <query>")
}
mode := strings.ToLower(parts[0])
query := strings.TrimSpace(strings.TrimPrefix(line, parts[0]))
switch mode {
case "text", "hex", "regex":
default:
return nil, fmt.Errorf("unknown multi-search mode %q", mode)
}
if query == "" {
return nil, errors.New("empty multi-search query")
}
out = append(out, MultiSearchQuery{Mode: mode, Query: query})
}
if len(out) == 0 {
return nil, errors.New("no multi-search queries")
}
return out, nil
}
func MultiSearch(d *document.Document, start, length int64, spec string, maxPerQuery int) (map[string]any, error) {
queries, err := ParseMultiSearchSpec(spec)
if err != nil {
return nil, err
}
if maxPerQuery <= 0 {
maxPerQuery = 1000
}
results := make([]map[string]any, 0, len(queries))
all := []map[string]any{}
for i, q := range queries {
var hits []SearchHit
switch q.Mode {
case "text":
hits, err = SearchBytes(d, start, length, []byte(q.Query), maxPerQuery)
case "hex":
var needle []byte
needle, err = parseMultiSearchHex(q.Query)
if err == nil {
hits, err = SearchBytes(d, start, length, needle, maxPerQuery)
}
case "regex":
if _, err = regexp.Compile(q.Query); err == nil {
hits, err = SearchRegex(d, start, length, q.Query, maxPerQuery)
}
}
if err != nil {
return nil, fmt.Errorf("query %d: %w", i+1, err)
}
maps := HitsToMaps(hits)
name := fmt.Sprintf("%s:%s", q.Mode, q.Query)
for _, h := range maps {
h["mode"] = q.Mode
h["query"] = q.Query
h["name"] = name
all = append(all, h)
}
results = append(results, map[string]any{
"mode": q.Mode,
"query": q.Query,
"count": len(hits),
"hits": maps,
})
}
return map[string]any{
"queries": results,
"all": all,
"total": len(all),
}, nil
}
func parseMultiSearchHex(s string) ([]byte, error) {
s = strings.Map(func(r rune) rune {
switch r {
case ' ', '\t', '\n', '\r', ':', '-':
return -1
default:
return r
}
}, strings.TrimSpace(s))
s = strings.TrimPrefix(strings.ToLower(s), "0x")
if len(s)%2 != 0 {
s = "0" + s
}
return hex.DecodeString(s)
}
internal/leviathan/analysis/overview.go
+88
View File
@@ -0,0 +1,88 @@
package analysis
import (
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Overview returns a compact first-pass report intended for both UI
// surfaces. It favours fast, bounded summaries over exhaustive output.
func Overview(d *document.Document) (map[string]any, error) {
size := d.Size()
hashes, err := Hashes(d, 0, -1)
if err != nil {
return nil, err
}
entropy, err := Entropy(d, 0, min64(size, 8<<20), 4096, 0)
if err != nil {
return nil, err
}
stringsOut, err := ExtractStrings(d, 0, min64(size, 16<<20), 4, []string{"ascii", "utf16le", "utf16be"}, 5000)
if err != nil {
return nil, err
}
iocs, err := ExtractIOCs(d, 0, min64(size, 16<<20), 5)
if err != nil {
return nil, err
}
carves, err := Carve(d, 0, min64(size, 64<<20), 128)
if err != nil {
return nil, err
}
return map[string]any{
"name": d.Name(),
"size": size,
"hashes": hashes,
"entropy": overviewEntropy(entropy),
"strings": overviewStrings(stringsOut),
"iocs": iocs["totals"],
"carve": overviewCarve(carves),
"limits": map[string]any{
"entropyBytes": min64(size, 8<<20),
"stringsBytes": min64(size, 16<<20),
"iocBytes": min64(size, 16<<20),
"carveBytes": min64(size, 64<<20),
},
}, nil
}
func overviewEntropy(in map[string]any) map[string]any {
return map[string]any{
"windowSize": in["windowSize"],
"step": in["step"],
"mean": in["mean"],
"min": in["min"],
"max": in["max"],
"highCount": in["highCount"],
}
}
func overviewStrings(items []map[string]any) map[string]any {
counts := map[string]int{}
for _, item := range items {
if enc, ok := item["encoding"].(string); ok {
counts[enc]++
}
}
return map[string]any{
"total": len(items),
"byEncoding": counts,
}
}
func overviewCarve(items []map[string]any) map[string]any {
counts := map[string]int{}
valid := 0
for _, item := range items {
if kind, ok := item["kind"].(string); ok {
counts[kind]++
}
if status, ok := item["status"].(string); ok && status == "valid" {
valid++
}
}
return map[string]any{
"total": len(items),
"valid": valid,
"byKind": counts,
}
}
internal/leviathan/analysis/rules.go
+648
View File
@@ -0,0 +1,648 @@
package analysis
import (
"context"
"encoding/hex"
"errors"
"regexp"
"strconv"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Rule represents a lightweight YARA-like rule. A rule matches when
// Condition evaluates to true for the document. Phase 1 supports:
// - "all of them" (every string pattern must appear)
// - "any of them" (at least one pattern must appear)
// - "<count> of them" (at least N patterns)
// - named identifiers referencing specific strings with $x or !$x
type Rule struct {
Name string
Tags []string
Strings []RuleString
Condition string
}
// RuleString is one string directive inside a rule.
type RuleString struct {
ID string
Kind string // "ascii", "hex", "regex"
Value string // original source text
// compiled:
raw []byte
regex *regexp.Regexp
nocase bool
ascii bool
wide bool
fullword bool
}
// ParseRules parses a simplified YARA-like source:
//
// rule Name : tag1 tag2 {
// strings:
// $a = "hello" nocase
// $b = { 4d 5a ?? ?? }
// $c = /foo[0-9]+/
// condition:
// any of them
// }
//
// Hex patterns support `??` wildcards by being converted to a regex.
func ParseRules(src string) ([]Rule, error) {
rules := []Rule{}
s := strings.TrimSpace(src)
if s == "" {
return rules, nil
}
// Split on "rule " keywords from start of line.
entries := splitRules(s)
for _, e := range entries {
r, err := parseRule(e)
if err != nil {
return rules, err
}
rules = append(rules, r)
}
return rules, nil
}
func splitRules(src string) []string {
out := []string{}
i := 0
for i < len(src) {
idx := strings.Index(src[i:], "rule ")
if idx < 0 {
break
}
j := i + idx
// find matching close brace (naïve but workable for phase 1)
depth := 0
start := j
k := j
for k < len(src) {
c := src[k]
if c == '{' {
depth++
} else if c == '}' {
depth--
if depth == 0 {
out = append(out, src[start:k+1])
k++
break
}
}
k++
}
i = k
}
return out
}
var ruleHeaderRe = regexp.MustCompile(`(?s)^\s*rule\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s*:\s*([^{]*))?\s*\{(.*)\}\s*$`)
func parseRule(src string) (Rule, error) {
m := ruleHeaderRe.FindStringSubmatch(src)
if m == nil {
return Rule{}, errors.New("malformed rule")
}
r := Rule{Name: m[1]}
if tags := strings.TrimSpace(m[2]); tags != "" {
r.Tags = strings.Fields(tags)
}
body := m[3]
if i := strings.Index(body, "strings:"); i >= 0 {
sEnd := strings.Index(body, "condition:")
sect := body[i+len("strings:") : sEnd]
for _, line := range strings.Split(sect, "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "//") {
continue
}
rs, err := parseRuleString(line)
if err != nil {
return r, err
}
r.Strings = append(r.Strings, rs)
}
}
if i := strings.Index(body, "condition:"); i >= 0 {
r.Condition = strings.TrimSpace(body[i+len("condition:"):])
} else {
return r, errors.New("rule missing condition")
}
return r, nil
}
var ruleStringRe = regexp.MustCompile(`^\s*(\$[A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.+?)\s*$`)
func parseRuleString(line string) (RuleString, error) {
m := ruleStringRe.FindStringSubmatch(line)
if m == nil {
return RuleString{}, errors.New("bad string line: " + line)
}
id := m[1]
rest := strings.TrimSpace(m[2])
value, modifiers, err := splitRuleValueModifiers(rest)
if err != nil {
return RuleString{}, err
}
rs := RuleString{ID: id, Value: value}
for _, mod := range modifiers {
switch strings.ToLower(mod) {
case "nocase":
rs.nocase = true
case "ascii":
rs.ascii = true
case "wide":
rs.wide = true
case "fullword":
rs.fullword = true
default:
return rs, errors.New("unknown string modifier: " + mod)
}
}
switch {
case strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) && len(value) >= 2:
rs.Kind = "ascii"
rs.raw = []byte(value[1 : len(value)-1])
if !rs.ascii && !rs.wide {
rs.ascii = true
}
case strings.HasPrefix(value, "{") && strings.HasSuffix(value, "}"):
rs.Kind = "hex"
pattern := regexFromHexPattern(value[1 : len(value)-1])
re, err := regexp.Compile("(?s)" + pattern)
if err != nil {
return rs, err
}
rs.regex = re
case strings.HasPrefix(value, "/") && strings.HasSuffix(value, "/"):
rs.Kind = "regex"
pat := value[1 : len(value)-1]
if rs.nocase {
pat = "(?i)" + pat
}
re, err := regexp.Compile(pat)
if err != nil {
return rs, err
}
rs.regex = re
default:
return rs, errors.New("unknown string form: " + value)
}
return rs, nil
}
func splitRuleValueModifiers(rest string) (string, []string, error) {
rest = strings.TrimSpace(rest)
if rest == "" {
return "", nil, errors.New("empty string value")
}
var end int
switch rest[0] {
case '"':
escaped := false
end = -1
for i := 1; i < len(rest); i++ {
c := rest[i]
if escaped {
escaped = false
continue
}
if c == '\\' {
escaped = true
continue
}
if c == '"' {
end = i + 1
break
}
}
if end < 0 {
return "", nil, errors.New("unterminated string literal")
}
case '{':
idx := strings.IndexByte(rest, '}')
if idx < 0 {
return "", nil, errors.New("unterminated hex string")
}
end = idx + 1
case '/':
escaped := false
end = -1
for i := 1; i < len(rest); i++ {
c := rest[i]
if escaped {
escaped = false
continue
}
if c == '\\' {
escaped = true
continue
}
if c == '/' {
end = i + 1
break
}
}
if end < 0 {
return "", nil, errors.New("unterminated regex")
}
default:
return "", nil, errors.New("unknown string form: " + rest)
}
return strings.TrimSpace(rest[:end]), strings.Fields(strings.TrimSpace(rest[end:])), nil
}
func regexFromHexPattern(hexPat string) string {
var sb strings.Builder
hexPat = strings.ReplaceAll(hexPat, " ", "")
for i := 0; i < len(hexPat); {
if i+1 >= len(hexPat) {
break
}
a, b := hexPat[i], hexPat[i+1]
if a == '?' && b == '?' {
sb.WriteString(`[\x00-\xff]`)
} else {
decoded, err := hex.DecodeString(string([]byte{a, b}))
if err != nil {
// bail to literal to avoid panic
sb.WriteString(regexp.QuoteMeta(string([]byte{a, b})))
} else {
sb.WriteString(regexp.QuoteMeta(string(decoded)))
}
}
i += 2
}
return sb.String()
}
// RuleMatch is a per-rule result.
type RuleMatch struct {
Name string
Tags []string
Matched bool
Hits map[string][]SearchHit
}
// RunRules scans rules against the document range and returns matches.
func RunRules(d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) {
return RunRulesContext(context.Background(), d, start, length, rules, maxPerString)
}
func RunRulesContext(ctx context.Context, d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) {
if maxPerString <= 0 {
maxPerString = 1000
}
out := make([]map[string]any, 0, len(rules))
for _, r := range rules {
if err := ctx.Err(); err != nil {
return nil, err
}
hits := map[string][]SearchHit{}
for _, s := range r.Strings {
if err := ctx.Err(); err != nil {
return nil, err
}
switch s.Kind {
case "ascii":
found, err := searchRuleTextContext(ctx, d, start, length, s, maxPerString)
if err != nil {
return nil, err
}
hits[s.ID] = found
case "hex", "regex":
found, err := SearchRegexContext(ctx, d, start, length, s.regex.String(), maxPerString)
if err != nil {
return nil, err
}
if s.fullword {
found = filterFullWordHits(d, found, false)
}
hits[s.ID] = found
}
}
matched := evalCondition(r, hits)
var hitFlat []map[string]any
for id, hs := range hits {
for _, h := range hs {
hitFlat = append(hitFlat, map[string]any{
"id": id,
"offset": h.Offset,
"length": h.Length,
"sample": h.Sample,
})
}
}
out = append(out, map[string]any{
"name": r.Name,
"tags": toAnySlice(r.Tags),
"matched": matched,
"hits": hitFlat,
})
}
return out, nil
}
func searchRuleText(d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) {
return searchRuleTextContext(context.Background(), d, start, length, s, max)
}
func searchRuleTextContext(ctx context.Context, d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) {
out := []SearchHit{}
if s.ascii {
var (
found []SearchHit
err error
)
if s.nocase {
found, err = SearchRegexContext(ctx, d, start, length, "(?i)"+regexp.QuoteMeta(string(s.raw)), max)
} else {
found, err = SearchBytesContext(ctx, d, start, length, s.raw, max)
}
if err != nil {
return nil, err
}
if s.fullword {
found = filterFullWordHits(d, found, false)
}
out = append(out, found...)
}
if s.wide {
wide := utf16LEBytes(s.raw)
var (
found []SearchHit
err error
)
if s.nocase {
pat := wideNoCasePattern(s.raw)
found, err = SearchRegexContext(ctx, d, start, length, pat, max)
} else {
found, err = SearchBytesContext(ctx, d, start, length, wide, max)
}
if err != nil {
return nil, err
}
if s.fullword {
found = filterFullWordHits(d, found, true)
}
out = append(out, found...)
}
if len(out) > max {
out = out[:max]
}
return out, nil
}
func utf16LEBytes(in []byte) []byte {
out := make([]byte, 0, len(in)*2)
for _, b := range in {
out = append(out, b, 0)
}
return out
}
func wideNoCasePattern(in []byte) string {
var sb strings.Builder
sb.WriteString("(?i)")
for _, b := range in {
sb.WriteString(regexp.QuoteMeta(string([]byte{b, 0})))
}
return sb.String()
}
func filterFullWordHits(d *document.Document, hits []SearchHit, wide bool) []SearchHit {
out := hits[:0]
for _, h := range hits {
if isFullWordHit(d, h, wide) {
out = append(out, h)
}
}
return out
}
func isFullWordHit(d *document.Document, h SearchHit, wide bool) bool {
if wide {
beforeOff := h.Offset - 2
afterOff := h.Offset + int64(h.Length)
beforeWord := false
afterWord := false
if beforeOff >= 0 {
if b, err := d.Range(beforeOff, 2); err == nil && len(b) == 2 && b[1] == 0 {
beforeWord = isWordByte(b[0])
}
}
if afterOff+1 < d.Size() {
if b, err := d.Range(afterOff, 2); err == nil && len(b) == 2 && b[1] == 0 {
afterWord = isWordByte(b[0])
}
}
return !beforeWord && !afterWord
}
beforeWord := false
afterWord := false
if h.Offset > 0 {
if b, err := d.At(h.Offset - 1); err == nil {
beforeWord = isWordByte(b)
}
}
if h.Offset+int64(h.Length) < d.Size() {
if b, err := d.At(h.Offset + int64(h.Length)); err == nil {
afterWord = isWordByte(b)
}
}
return !beforeWord && !afterWord
}
func isWordByte(b byte) bool {
return (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9') || b == '_'
}
func toAnySlice(in []string) []any {
out := make([]any, len(in))
for i, s := range in {
out[i] = s
}
return out
}
func evalCondition(r Rule, hits map[string][]SearchHit) bool {
cond := strings.TrimSpace(r.Condition)
condLower := strings.ToLower(cond)
present := 0
for _, s := range r.Strings {
if len(hits[s.ID]) > 0 {
present++
}
}
switch {
case condLower == "any of them":
return present > 0
case condLower == "all of them":
return present == len(r.Strings)
case strings.HasSuffix(condLower, "of them"):
// "<n> of them"
parts := strings.Fields(condLower)
if len(parts) >= 3 {
n, ok := parseRuleCount(parts[0])
return ok && present >= n
}
}
if ok, matched := evalRuleSetCondition(cond, hits); ok {
return matched
}
p := conditionParser{tokens: tokenizeCondition(cond), hits: hits}
return p.parse()
}
func evalRuleSetCondition(cond string, hits map[string][]SearchHit) (bool, bool) {
tokens := tokenizeCondition(cond)
if len(tokens) < 4 || strings.ToLower(tokens[1]) != "of" || tokens[2] != "(" || tokens[len(tokens)-1] != ")" {
return false, false
}
ids := []string{}
for _, tok := range tokens[3 : len(tokens)-1] {
if tok == "," {
continue
}
if strings.HasPrefix(tok, "$") {
ids = append(ids, tok)
}
}
if len(ids) == 0 {
return true, false
}
count := 0
for _, id := range ids {
if len(hits[id]) > 0 {
count++
}
}
switch strings.ToLower(tokens[0]) {
case "any":
return true, count > 0
case "all":
return true, count == len(ids)
default:
n, ok := parseRuleCount(strings.ToLower(tokens[0]))
return true, ok && count >= n
}
}
func parseRuleCount(s string) (int, bool) {
switch s {
case "1", "one":
return 1, true
case "2", "two":
return 2, true
case "3", "three":
return 3, true
default:
n, err := strconv.Atoi(s)
return n, err == nil
}
}
type conditionParser struct {
tokens []string
pos int
hits map[string][]SearchHit
}
func (p *conditionParser) parse() bool {
v := p.parseOr()
return v
}
func (p *conditionParser) parseOr() bool {
v := p.parseAnd()
for strings.EqualFold(p.peek(), "or") {
p.next()
v = p.parseAnd() || v
}
return v
}
func (p *conditionParser) parseAnd() bool {
v := p.parseNot()
for strings.EqualFold(p.peek(), "and") {
p.next()
v = p.parseNot() && v
}
return v
}
func (p *conditionParser) parseNot() bool {
if strings.EqualFold(p.peek(), "not") || p.peek() == "!" {
p.next()
return !p.parseNot()
}
return p.parsePrimary()
}
func (p *conditionParser) parsePrimary() bool {
tok := p.next()
switch {
case tok == "":
return false
case tok == "(":
v := p.parseOr()
if p.peek() == ")" {
p.next()
}
return v
case strings.EqualFold(tok, "true"):
return true
case strings.EqualFold(tok, "false"):
return false
case strings.HasPrefix(tok, "$"):
return len(p.hits[tok]) > 0
default:
return false
}
}
func (p *conditionParser) peek() string {
if p.pos >= len(p.tokens) {
return ""
}
return p.tokens[p.pos]
}
func (p *conditionParser) next() string {
if p.pos >= len(p.tokens) {
return ""
}
tok := p.tokens[p.pos]
p.pos++
return tok
}
func tokenizeCondition(cond string) []string {
out := []string{}
for i := 0; i < len(cond); {
c := cond[i]
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
i++
continue
}
if strings.ContainsRune("(),!", rune(c)) {
out = append(out, string(c))
i++
continue
}
start := i
for i < len(cond) {
c = cond[i]
if c == ' ' || c == '\t' || c == '\n' || c == '\r' || strings.ContainsRune("(),!", rune(c)) {
break
}
i++
}
out = append(out, cond[start:i])
}
return out
}
internal/leviathan/analysis/search.go
+189
View File
@@ -0,0 +1,189 @@
package analysis
import (
"bytes"
"context"
"errors"
"regexp"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// SearchHit describes one search result.
type SearchHit struct {
Offset int64 `json:"offset"`
Length int `json:"length"`
Sample string `json:"sample"` // short hex preview
}
// SearchBytes finds `needle` in the document range and returns hit
// offsets. Scans chunk by chunk with an overlap equal to len(needle)-1
// so cross-chunk matches are not missed.
func SearchBytes(d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) {
return SearchBytesContext(context.Background(), d, start, length, needle, max)
}
func SearchBytesContext(ctx context.Context, d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) {
if len(needle) == 0 {
return nil, errors.New("empty needle")
}
if max <= 0 {
max = 10_000
}
if length < 0 {
length = d.Size() - start
}
hits := make([]SearchHit, 0, 64)
overlap := int64(len(needle) - 1)
// We chunk in windows of chunkSize + overlap.
const chunkSize = 1 << 20 // 1MB
off := start
endExclusive := start + length
for off < endExclusive && len(hits) < max {
if err := ctx.Err(); err != nil {
return nil, err
}
winEnd := off + chunkSize
if winEnd > endExclusive {
winEnd = endExclusive
}
readEnd := winEnd + overlap
if readEnd > endExclusive {
readEnd = endExclusive
}
buf, err := d.Range(off, readEnd-off)
if err != nil {
return nil, err
}
// Limit search to bytes whose start lies in [0, winEnd-off)
searchLimit := winEnd - off
scan := 0
for {
if err := ctx.Err(); err != nil {
return nil, err
}
idx := bytes.Index(buf[scan:], needle)
if idx < 0 {
break
}
abs := int64(scan+idx) + off
relStart := abs - off
if relStart >= searchLimit {
break
}
hits = append(hits, SearchHit{
Offset: abs,
Length: len(needle),
Sample: hexPreview(buf[scan+idx:], len(needle)),
})
if len(hits) >= max {
break
}
scan = scan + idx + 1
if scan >= len(buf) {
break
}
}
off = winEnd
}
return hits, nil
}
// SearchRegex runs a Go regexp across the document range, matching
// byte-wise. It decodes matches chunk by chunk with overlap, bounded
// by a generous max hit count.
func SearchRegex(d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) {
return SearchRegexContext(context.Background(), d, start, length, pattern, max)
}
func SearchRegexContext(ctx context.Context, d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) {
re, err := regexp.Compile(pattern)
if err != nil {
return nil, err
}
if max <= 0 {
max = 10_000
}
if length < 0 {
length = d.Size() - start
}
// For regex, a conservative overlap is 4 KiB so matches spanning
// chunk boundaries are still caught. Very long matches may still
// be missed in extreme cases.
const chunkSize = 1 << 20
const overlap = 4096
hits := make([]SearchHit, 0, 64)
off := start
endExclusive := start + length
for off < endExclusive && len(hits) < max {
if err := ctx.Err(); err != nil {
return nil, err
}
winEnd := off + chunkSize
if winEnd > endExclusive {
winEnd = endExclusive
}
readEnd := winEnd + overlap
if readEnd > endExclusive {
readEnd = endExclusive
}
buf, err := d.Range(off, readEnd-off)
if err != nil {
return nil, err
}
searchLimit := winEnd - off
locs := re.FindAllIndex(buf, max-len(hits))
for _, loc := range locs {
if err := ctx.Err(); err != nil {
return nil, err
}
relStart := int64(loc[0])
if relStart >= searchLimit {
continue
}
abs := relStart + off
hits = append(hits, SearchHit{
Offset: abs,
Length: loc[1] - loc[0],
Sample: hexPreview(buf[loc[0]:loc[1]], loc[1]-loc[0]),
})
}
off = winEnd
}
return hits, nil
}
func hexPreview(b []byte, n int) string {
if n > 32 {
n = 32
}
if n > len(b) {
n = len(b)
}
const hexchars = "0123456789abcdef"
out := make([]byte, n*2)
for i := 0; i < n; i++ {
out[i*2] = hexchars[b[i]>>4]
out[i*2+1] = hexchars[b[i]&0x0f]
}
return string(out)
}
// HitsToMaps converts hits for bridge return.
func HitsToMaps(hits []SearchHit) []map[string]any {
out := make([]map[string]any, len(hits))
for i, h := range hits {
out[i] = map[string]any{
"offset": h.Offset,
"length": h.Length,
"sample": h.Sample,
}
}
return out
}
internal/leviathan/analysis/strings.go
+198
View File
@@ -0,0 +1,198 @@
package analysis
import (
"context"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// StringHit represents one extracted string.
type StringHit struct {
Offset int64 `json:"offset"`
Length int `json:"length"`
Encoding string `json:"encoding"` // ascii | utf16le | utf16be
Value string `json:"value"`
}
// ExtractStrings finds printable runs in the specified range.
// Supports ASCII and UTF-16 LE/BE. minLen is in characters.
func ExtractStrings(d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) {
return ExtractStringsContext(context.Background(), d, start, length, minLen, encodings, max)
}
func ExtractStringsContext(ctx context.Context, d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) {
if minLen <= 0 {
minLen = 4
}
if max <= 0 {
max = 50_000
}
if length < 0 {
length = d.Size() - start
}
doASCII, doLE, doBE := false, false, false
if len(encodings) == 0 {
doASCII, doLE, doBE = true, true, true
} else {
for _, e := range encodings {
switch e {
case "ascii":
doASCII = true
case "utf16le":
doLE = true
case "utf16be":
doBE = true
}
}
}
out := make([]map[string]any, 0, 256)
// ASCII
if doASCII {
if err := appendASCII(ctx, d, start, length, minLen, &out, max); err != nil {
return nil, err
}
}
// UTF-16
if len(out) < max && doLE {
if err := appendUTF16(ctx, d, start, length, minLen, true, &out, max); err != nil {
return nil, err
}
}
if len(out) < max && doBE {
if err := appendUTF16(ctx, d, start, length, minLen, false, &out, max); err != nil {
return nil, err
}
}
return out, nil
}
func appendASCII(ctx context.Context, d *document.Document, start, length int64, minLen int, out *[]map[string]any, max int) error {
// Chunked scan with a carry for runs crossing chunk boundaries.
var runStart int64 = -1
var runBuf []byte
emit := func() {
if runStart >= 0 && len(runBuf) >= minLen {
*out = append(*out, map[string]any{
"offset": runStart,
"length": len(runBuf),
"encoding": "ascii",
"value": string(runBuf),
})
}
runStart = -1
runBuf = runBuf[:0]
}
if err := d.ForEachChunk(start, length, func(off int64, chunk []byte) error {
if err := ctx.Err(); err != nil {
return err
}
for i, b := range chunk {
if i%4096 == 0 {
if err := ctx.Err(); err != nil {
return err
}
}
if isPrintableASCII(b) {
if runStart < 0 {
runStart = off + int64(i)
runBuf = make([]byte, 0, 32)
}
runBuf = append(runBuf, b)
} else {
if len(*out) >= max {
return nil
}
emit()
}
}
return nil
}); err != nil {
return err
}
emit()
return nil
}
func appendUTF16(ctx context.Context, d *document.Document, start, length int64, minLen int, le bool, out *[]map[string]any, max int) error {
// Step by 2. Produce one string for each contiguous run of printable code units.
if length < 2 {
return nil
}
// Read chunks aligned to 2 bytes so pairs don't split.
const chunkSize = 1 << 20
end := start + length
var runStart int64 = -1
var runBuf []rune
enc := "utf16le"
if !le {
enc = "utf16be"
}
emit := func() {
if runStart >= 0 && len(runBuf) >= minLen {
*out = append(*out, map[string]any{
"offset": runStart,
"length": len(runBuf) * 2,
"encoding": enc,
"value": string(runBuf),
})
}
runStart = -1
runBuf = runBuf[:0]
}
for off := start; off+1 < end; {
if err := ctx.Err(); err != nil {
return err
}
winEnd := off + chunkSize
if winEnd > end {
winEnd = end
}
if (winEnd-off)%2 != 0 {
winEnd-- // keep pair alignment
}
buf, err := d.Range(off, winEnd-off)
if err != nil {
return err
}
for i := 0; i+1 < len(buf); i += 2 {
if i%4096 == 0 {
if err := ctx.Err(); err != nil {
return err
}
}
var cu uint16
if le {
cu = uint16(buf[i]) | uint16(buf[i+1])<<8
} else {
cu = uint16(buf[i])<<8 | uint16(buf[i+1])
}
if cu >= 0x20 && cu <= 0x7e {
if runStart < 0 {
runStart = off + int64(i)
runBuf = make([]rune, 0, 32)
}
runBuf = append(runBuf, rune(cu))
} else {
if len(*out) >= max {
emit()
return nil
}
emit()
}
}
off = winEnd
}
emit()
return nil
}
func isPrintableASCII(b byte) bool {
return b == '\t' || (b >= 0x20 && b <= 0x7e)
}
internal/leviathan/analysis/timestamps.go
+106
View File
@@ -0,0 +1,106 @@
package analysis
import (
"encoding/binary"
"math"
"time"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ScanTimestamps walks the range looking for plausible timestamps in
// common encodings. This is a heuristic: it reports any 4/8-byte
// window whose decoded time falls in [1990-01-01, 2050-12-31].
func ScanTimestamps(d *document.Document, start, length int64, maxHits int) (map[string]any, error) {
if length < 0 {
length = d.Size() - start
}
if maxHits <= 0 {
maxHits = 20_000
}
minT := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC).Unix()
maxT := time.Date(2050, 12, 31, 0, 0, 0, 0, time.UTC).Unix()
ok := func(sec int64) bool { return sec >= minT && sec <= maxT }
hits := make([]map[string]any, 0, 256)
counts := map[string]int{}
const chunk = 1 << 20
end := start + length
overlap := int64(7) // enough for 8-byte types
for off := start; off < end && len(hits) < maxHits; {
winEnd := off + chunk
if winEnd > end {
winEnd = end
}
readEnd := winEnd + overlap
if readEnd > end {
readEnd = end
}
buf, err := d.Range(off, readEnd-off)
if err != nil {
return nil, err
}
limit := int(winEnd - off)
for i := 0; i < limit && len(hits) < maxHits; i += 4 {
if i+4 <= len(buf) {
vLE := int64(int32(binary.LittleEndian.Uint32(buf[i:])))
if ok(vLE) {
hits = append(hits, hit("unix_le", off+int64(i), 4, time.Unix(vLE, 0).UTC()))
counts["unix_le"]++
continue
}
vBE := int64(int32(binary.BigEndian.Uint32(buf[i:])))
if ok(vBE) {
hits = append(hits, hit("unix_be", off+int64(i), 4, time.Unix(vBE, 0).UTC()))
counts["unix_be"]++
continue
}
}
if i+8 <= len(buf) {
v := binary.LittleEndian.Uint64(buf[i:])
// Windows FILETIME
sec := int64(v/10_000_000) - filetimeEpochDelta
if ok(sec) {
hits = append(hits, hit("filetime_le", off+int64(i), 8, time.Unix(sec, 0).UTC()))
counts["filetime_le"]++
continue
}
// WebKit (microseconds since 1601)
sec = int64(v/1_000_000) - filetimeEpochDelta
if ok(sec) {
hits = append(hits, hit("webkit_le", off+int64(i), 8, time.Unix(sec, 0).UTC()))
counts["webkit_le"]++
continue
}
// OLE automation date
fdays := math.Float64frombits(v)
if fdays > 2 && fdays < 75000 {
base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
t := base.Add(time.Duration(fdays*86400) * time.Second)
if t.Unix() >= minT && t.Unix() <= maxT {
hits = append(hits, hit("ole_le", off+int64(i), 8, t))
counts["ole_le"]++
continue
}
}
}
}
off = winEnd
}
return map[string]any{
"hits": hits,
"counts": counts,
}, nil
}
func hit(kind string, off int64, size int, t time.Time) map[string]any {
return map[string]any{
"kind": kind,
"offset": off,
"length": size,
"iso": t.Format(time.RFC3339),
"unix": t.Unix(),
}
}
internal/leviathan/binsnoop/README.md
+277
View File
@@ -0,0 +1,277 @@
# binsnoop
A pure-Go binary analysis library for detecting and extracting embedded files, firmware images, compressed data, filesystems, and more from binary blobs. Ported from the [binwalk](https://github.com/ReFirmLabs/binwalk) Rust codebase with like-for-like scanning logic.
**Zero external dependencies** — everything is self-contained, including the Aho-Corasick multi-pattern matcher, CRC32 validation, and binary structure parsing.
## Features
- **111 built-in signatures** covering archives, compression, filesystems, images, executables, firmware headers, cryptographic constructs, and more
- **Aho-Corasick** multi-pattern scanning with overlapping match support
- **Signature validation** — every magic byte match is verified by a dedicated parser that checks headers, checksums, and structural integrity
- **Confidence scoring** — results are ranked Low / Medium / High so callers can filter noise
- **Extraction support** — carve identified objects out of a binary, or invoke external tools (7z, unsquashfs, etc.) to fully extract them
- **Filtering** — include/exclude signatures by name, restrict short signatures to offset 0, or add custom signatures
- **JSON-friendly** — all result types carry `json` struct tags for easy serialisation
## Installation
```bash
go get binsnoop
```
Or add to your `go.mod`:
```
require binsnoop v0.0.0
```
## Quick Start
```go
package main
import (
"fmt"
"binsnoop"
)
func main() {
scanner := binsnoop.New()
// Scan a file
results, err := scanner.ScanFile("firmware.bin")
if err != nil {
panic(err)
}
for _, r := range results {
fmt.Printf("[0x%08X] %s (confidence: %d, size: %d)\n",
r.Offset, r.Description, r.Confidence, r.Size)
}
}
```
## API Reference
### Creating a Scanner
```go
// Default scanner with all 111 signatures
scanner := binsnoop.New()
// Scanner with options
scanner, err := binsnoop.NewWithOptions(binsnoop.Options{
Include: []string{"gzip", "squashfs", "uimage"}, // only these signatures
Exclude: []string{"copyright"}, // remove these signatures
FullSearch: true, // match short sigs at any offset
AdditionalSignatures: []binsnoop.Signature{...}, // add custom signatures
})
```
### Scanning
```go
// Scan raw bytes
results := scanner.ScanBytes(data)
// Scan a file on disk
results, err := scanner.ScanFile("/path/to/firmware.bin")
```
Both return `[]SignatureResult` sorted by offset.
### Analysis (Scan + Extraction)
```go
// Analyse raw bytes (with optional extraction)
analysis := scanner.AnalyzeBytes(data, "firmware.bin", true)
// Analyse a file on disk
analysis, err := scanner.AnalyzeFile("/path/to/firmware.bin", true)
```
Returns `*AnalysisResults` containing the file map and extraction outcomes.
### Carving
Extract a single identified object from raw data:
```go
err := binsnoop.CarveResult(data, results[0], "/tmp/output")
```
This writes the bytes at the result's offset/size to a file in the output directory.
### Listing Signatures
```go
names := binsnoop.ListSignatures() // []string of all registered signature names
```
### Scanner Info
```go
scanner.SignatureCount() // number of registered signatures (111)
scanner.PatternCount() // total magic byte patterns across all signatures
```
## Types
### `SignatureResult`
```go
type SignatureResult struct {
Offset int `json:"offset"` // byte offset in the input data
ID string `json:"id"` // unique identifier for this match
Size int `json:"size"` // size in bytes (0 = unknown)
Name string `json:"name"` // signature name (e.g. "gzip")
Confidence uint8 `json:"confidence"` // 0=Low, 128=Medium, 250=High
Description string `json:"description"` // human-readable description
AlwaysDisplay bool `json:"always_display"` // show even if low confidence
ExtractionDeclined bool `json:"extraction_declined"` // parser advises against extraction
PreferredExtractor *Extractor `json:"-"` // extractor to use for this result
}
```
### `Options`
```go
type Options struct {
Include []string // restrict to these signature names
Exclude []string // remove these signature names
FullSearch bool // short sigs match at any offset
AdditionalSignatures []Signature // user-defined signatures
}
```
### `AnalysisResults`
```go
type AnalysisResults struct {
FilePath string `json:"file_path"`
FileMap []SignatureResult `json:"file_map"`
Extractions map[string]*ExtractionResult `json:"extractions"`
}
```
### Confidence Levels
| Constant | Value | Meaning |
|----------|-------|---------|
| `ConfidenceLow` | 0 | Magic matched but weak validation |
| `ConfidenceMedium` | 128 | Header parsed and mostly valid |
| `ConfidenceHigh` | 250 | Full structural validation passed |
## Custom Signatures
You can add your own signatures at scanner creation:
```go
mySignature := binsnoop.Signature{
Name: "my_format",
Magic: [][]byte{[]byte("MYMAGIC")},
Parser: func(data []byte, offset int) (*binsnoop.SignatureResult, error) {
// Validate the match and return a result
if offset+16 > len(data) {
return nil, fmt.Errorf("insufficient data")
}
return &binsnoop.SignatureResult{
Offset: offset,
Size: 1024,
Description: "My custom format, size: 1024 bytes",
Confidence: binsnoop.ConfidenceHigh,
}, nil
},
Description: "My custom file format",
}
scanner, _ := binsnoop.NewWithOptions(binsnoop.Options{
AdditionalSignatures: []binsnoop.Signature{mySignature},
})
```
## Supported Signatures
### Archives (12)
gzip, zip, tarball, 7zip, rar, cab, deb, cpio, arj, iso9660, dahua\_zip
### Compression (10)
lzma, bzip2, xz, zstd, lz4, lzop, lzfse, zlib, compress'd
### Filesystems (13)
squashfs, cramfs, jffs2, yaffs, romfs, ext, fat, ntfs, apfs, btrfs, logfs, ubi, ubifs
### Images & Media (7)
png, jpeg, bmp, gif, svg, riff, pdf
### Executables (2)
elf, pe
### Firmware (32)
uimage, uboot, trx, tplink, tplink\_rtos, dlob, arcadyan, chk, seama, cfe, jboot\_arm, jboot\_stag, jboot\_sch2, packimg, binhdr, autel, android\_sparse, android\_bootimg, dtb, dlink\_tlv, dlke, shrs, dkbs, dms, encfw, encrpted\_img, wince, mh01, csman, dxbc, matter\_ota, rtk
### Cryptographic (17)
aes\_sbox, aes\_forward\_table, aes\_reverse\_table, aes\_rcon, aes\_acceleration\_table, rsa, openssl, gpg\_signed, pem\_certificate, pem\_public\_key, pem\_private\_key, pkcs\_der\_hash, luks, crc32, sha256, md5
### Miscellaneous (18)
copyright, wind\_kernel, vxworks\_symtab, ecos, linux\_kernel, linux\_boot\_image, linux\_arm\_zimage, linux\_arm64\_boot\_image, efigpt, mbr, uefi\_pi\_volume, uefi\_capsule, pchrom, pcapng, pjl, srecord, dmg, qnx\_ifs, dpapi, qcow
## Extraction
Many signatures have associated extractors that invoke external command-line tools. When you call `AnalyzeFile` or `AnalyzeBytes` with `extract: true`, binsnoop will:
1. Carve the identified data to a temporary file
2. Run the configured external tool (e.g. `7z x`, `unsquashfs`, `jefferson`)
3. Return the extraction results including output directory and success/failure status
**Common external tools used:**
| Tool | Signatures |
|------|-----------|
| `7z` | zip, cpio, arj, iso9660, xz, lzma, squashfs, dmg, efigpt |
| `gzip` | gzip |
| `bzip2` | bzip2 |
| `zstd` | zstd |
| `lz4` | lz4 |
| `unsquashfs` | squashfs |
| `jefferson` | jffs2 |
| `unyaffs` | yaffs |
| `tsk_recover` | ext, fat, ntfs |
| `uefi-firmware-parser` | uefi\_pi\_volume, uefi\_capsule, pchrom |
| `simg2img` | android\_sparse |
| `dtc` | dtb |
## How It Works
### Scanning Algorithm
1. **Short-signature pass** — signatures marked `Short` (e.g. zlib, MBR) are checked only at offset 0 (unless `FullSearch` is enabled)
2. **Aho-Corasick scan** — all magic byte patterns are compiled into an Aho-Corasick automaton for efficient multi-pattern matching. When a high-confidence sized match is found, scanning restarts from the end of that match
3. **Validation** — every magic match is passed to its signature parser which validates headers, checksums, and structure
4. **Post-processing** — results are sorted by offset, overlapping/duplicate matches are resolved (higher confidence wins), and unknown sizes are estimated based on the next significant match or EOF
### Architecture
```
binsnoop.go Public API (New, ScanBytes, ScanFile, AnalyzeFile, CarveResult, etc.)
scanner.go Core scan engine (Aho-Corasick integration, post-processing)
match.go Self-contained Aho-Corasick trie with overlapping match support
parse.go Generic binary structure parser (u8/u16/u24/u32/u64, big/little endian)
types.go Core types (SignatureResult, Signature, Extractor, Options, etc.)
common.go Utility functions (CRC32, endian helpers, string helpers)
extract.go Carving and external tool execution
magic.go Signature registry — wires all 111 signatures together
sig_archives.go Archive signature parsers
sig_compression.go Compression signature parsers
sig_filesystems.go Filesystem signature parsers
sig_images.go Image/media signature parsers
sig_executables.go Executable signature parsers
sig_firmware.go Firmware header signature parsers
sig_crypto.go Cryptographic signature parsers
sig_misc.go Miscellaneous signature parsers
```
## License
This project is a Go port of [binwalk](https://github.com/ReFirmLabs/binwalk) by ReFirmLabs. See the original project for licensing details.
internal/leviathan/binsnoop/binsnoop.go
+177
View File
@@ -0,0 +1,177 @@
// Package binsnoop identifies and optionally extracts files embedded inside binary data.
//
// It is a Go port of the binwalk scanning engine, implementing all magic byte signatures,
// structure parsers, and the Aho-Corasick based multi-pattern matching algorithm.
//
// # Quick Start
//
// scanner := binsnoop.New()
// data, _ := os.ReadFile("firmware.bin")
// results := scanner.Scan(data)
// for _, r := range results {
// fmt.Printf("0x%08X %s\n", r.Offset, r.Description)
// }
//
// # Extraction
//
// // Extract a specific result
// err := binsnoop.CarveResult(data, results[0], "/tmp/output")
//
// # Filtering
//
// // Only scan for specific signatures
// scanner := binsnoop.NewWithOptions(Options{Include: []string{"gzip", "squashfs"}})
package binsnoop
import (
"fmt"
"os"
"path/filepath"
"strings"
)
// Options configures the Scanner.
type Options struct {
// Include restricts scanning to only these signature names.
Include []string
// Exclude removes these signature names from scanning.
Exclude []string
// FullSearch makes short signatures match at any offset, not just offset 0.
FullSearch bool
// AdditionalSignatures adds user-defined signatures to the scanner.
AdditionalSignatures []Signature
}
// New creates a Scanner with all default signatures enabled.
func New() *Scanner {
s, _ := NewWithOptions(Options{})
return s
}
// NewWithOptions creates a Scanner with the specified options.
func NewWithOptions(opts Options) (*Scanner, error) {
s := &Scanner{
patternSigTable: make(map[int]Signature),
extractorLookup: make(map[string]*Extractor),
}
allSigs := AllSignatures()
if opts.AdditionalSignatures != nil {
allSigs = append(allSigs, opts.AdditionalSignatures...)
}
for _, sig := range allSigs {
if !includeSig(&sig, opts.Include, opts.Exclude) {
continue
}
s.signatureCount++
s.patternCount += len(sig.Magic)
// Register extractor
var ext *Extractor
if sig.Extractor != nil {
extCopy := *sig.Extractor
ext = &extCopy
}
s.extractorLookup[sig.Name] = ext
for _, pattern := range sig.Magic {
if sig.Short && !opts.FullSearch {
s.shortSignatures = append(s.shortSignatures, sig)
break
}
s.patternSigTable[len(s.patterns)] = sig
s.patterns = append(s.patterns, pattern)
}
}
s.compile()
return s, nil
}
// ScanBytes scans raw bytes for signatures and returns all matches.
func (s *Scanner) ScanBytes(data []byte) []SignatureResult {
return s.Scan(data)
}
// ScanFile reads a file from disk and scans it for signatures.
func (s *Scanner) ScanFile(path string) ([]SignatureResult, error) {
data, err := ReadFile(path)
if err != nil {
return nil, fmt.Errorf("binsnoop: read file: %w", err)
}
return s.Scan(data), nil
}
// AnalyzeBytes scans and optionally extracts from raw bytes.
func (s *Scanner) AnalyzeBytes(data []byte, filePath string, extract bool) *AnalysisResults {
results := &AnalysisResults{
FilePath: filePath,
Extractions: make(map[string]*ExtractionResult),
}
results.FileMap = s.Scan(data)
if extract && len(results.FileMap) > 0 {
results.Extractions = s.Extract(data, filePath, results.FileMap)
}
return results
}
// AnalyzeFile reads a file from disk, scans, and optionally extracts.
func (s *Scanner) AnalyzeFile(path string, extract bool) (*AnalysisResults, error) {
data, err := ReadFile(path)
if err != nil {
return nil, fmt.Errorf("binsnoop: read file: %w", err)
}
return s.AnalyzeBytes(data, path, extract), nil
}
// CarveResult extracts raw bytes for a specific signature result to a file.
func CarveResult(data []byte, result SignatureResult, outputDir string) error {
if result.Offset < 0 || result.Offset+result.Size > len(data) {
return fmt.Errorf("binsnoop: result offset/size exceeds data bounds")
}
if err := os.MkdirAll(outputDir, 0755); err != nil {
return fmt.Errorf("binsnoop: mkdir: %w", err)
}
outPath := filepath.Join(outputDir, fmt.Sprintf("%x_%s.bin", result.Offset, result.Name))
return os.WriteFile(outPath, data[result.Offset:result.Offset+result.Size], 0644)
}
// SignatureCount returns the number of loaded signatures.
func (s *Scanner) SignatureCount() int { return s.signatureCount }
// PatternCount returns the total number of magic byte patterns.
func (s *Scanner) PatternCount() int { return s.patternCount }
// ListSignatures returns the names of all available signatures.
func ListSignatures() []string {
sigs := AllSignatures()
names := make([]string, len(sigs))
for i, s := range sigs {
names[i] = s.Name
}
return names
}
func includeSig(sig *Signature, include, exclude []string) bool {
if len(include) > 0 {
for _, inc := range include {
if strings.EqualFold(sig.Name, inc) {
return true
}
}
return false
}
if len(exclude) > 0 {
for _, exc := range exclude {
if strings.EqualFold(sig.Name, exc) {
return false
}
}
}
return true
}
internal/leviathan/binsnoop/common.go
+127
View File
@@ -0,0 +1,127 @@
package binsnoop
import (
"encoding/binary"
"fmt"
"hash/crc32"
"os"
"time"
)
const MaxFileReadBytes int64 = 64 << 20
// CRC32 calculates the CRC32 checksum using the IEEE polynomial.
func CRC32(data []byte) uint32 {
return crc32.ChecksumIEEE(data)
}
// EpochToString converts a Unix timestamp to a formatted time string.
func EpochToString(epoch uint32) string {
t := time.Unix(int64(epoch), 0).UTC()
return t.Format("2006-01-02 15:04:05")
}
// GetCString extracts a NULL-terminated C string from raw bytes.
func GetCString(data []byte) string {
for i, b := range data {
if b == 0 {
s := string(data[:i])
// Validate UTF-8 printability
for _, c := range s {
if c == 0 {
return ""
}
}
return s
}
}
return string(data)
}
// IsASCIINumber returns true if the byte is an ASCII digit (0-9).
func IsASCIINumber(b byte) bool {
return b >= '0' && b <= '9'
}
// IsPrintableASCII returns true if the byte is a printable ASCII character.
func IsPrintableASCII(b byte) bool {
return b >= 0x0A && b <= 0x7E
}
// IsOffsetSafe validates that the next offset is within bounds and progressing forward.
func IsOffsetSafe(availableData, nextOffset int, lastOffset *int) bool {
if lastOffset != nil {
if *lastOffset >= nextOffset {
return false
}
}
return nextOffset < availableData
}
// ReadFile reads a bounded file into memory for scanner APIs that require a
// byte slice. Larger callers should use a streaming document path.
func ReadFile(path string) ([]byte, error) {
return ReadFileLimited(path, MaxFileReadBytes)
}
func ReadFileLimited(path string, maxBytes int64) ([]byte, error) {
info, err := os.Stat(path)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("binsnoop: path is a directory")
}
if maxBytes > 0 && info.Size() > maxBytes {
return nil, fmt.Errorf("binsnoop: file is too large: %d bytes exceeds %d bytes", info.Size(), maxBytes)
}
return os.ReadFile(path)
}
// LE16 reads a little-endian uint16 from data at the given offset.
func LE16(data []byte, offset int) uint16 {
if offset+2 > len(data) {
return 0
}
return binary.LittleEndian.Uint16(data[offset:])
}
// BE16 reads a big-endian uint16 from data at the given offset.
func BE16(data []byte, offset int) uint16 {
if offset+2 > len(data) {
return 0
}
return binary.BigEndian.Uint16(data[offset:])
}
// LE32 reads a little-endian uint32 from data at the given offset.
func LE32(data []byte, offset int) uint32 {
if offset+4 > len(data) {
return 0
}
return binary.LittleEndian.Uint32(data[offset:])
}
// BE32 reads a big-endian uint32 from data at the given offset.
func BE32(data []byte, offset int) uint32 {
if offset+4 > len(data) {
return 0
}
return binary.BigEndian.Uint32(data[offset:])
}
// LE64 reads a little-endian uint64 from data at the given offset.
func LE64(data []byte, offset int) uint64 {
if offset+8 > len(data) {
return 0
}
return binary.LittleEndian.Uint64(data[offset:])
}
// BE64 reads a big-endian uint64 from data at the given offset.
func BE64(data []byte, offset int) uint64 {
if offset+8 > len(data) {
return 0
}
return binary.BigEndian.Uint64(data[offset:])
}
internal/leviathan/binsnoop/common_test.go
+22
View File
@@ -0,0 +1,22 @@
package binsnoop
import (
"os"
"path/filepath"
"strings"
"testing"
)
func TestReadFileLimitedRejectsOversizedFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "firmware.bin")
if err := os.WriteFile(path, []byte("abcd"), 0600); err != nil {
t.Fatalf("WriteFile() failed: %v", err)
}
_, err := ReadFileLimited(path, 3)
if err == nil {
t.Fatal("expected oversized file to be rejected")
}
if !strings.Contains(err.Error(), "file is too large") {
t.Fatalf("unexpected error: %v", err)
}
}
internal/leviathan/binsnoop/extract.go
+124
View File
@@ -0,0 +1,124 @@
package binsnoop
import (
"fmt"
"os"
"os/exec"
"path/filepath"
"strings"
)
// Carve writes a slice of data to a file on disk.
func Carve(outputPath string, data []byte, offset, size int) error {
if offset < 0 || offset+size > len(data) {
return fmt.Errorf("carve: offset %d + size %d exceeds data length %d", offset, size, len(data))
}
dir := filepath.Dir(outputPath)
if err := os.MkdirAll(dir, 0755); err != nil {
return fmt.Errorf("carve: mkdir: %w", err)
}
return os.WriteFile(outputPath, data[offset:offset+size], 0644)
}
// executeExternal runs an external extraction tool.
func executeExternal(ext *Extractor, carvedFilePath, outputDir string) *ExtractionResult {
result := &ExtractionResult{
ExtractorName: ext.Command,
OutputDirectory: outputDir,
}
if err := os.MkdirAll(outputDir, 0755); err != nil {
return result
}
// Build arguments, replacing the source file placeholder
args := make([]string, len(ext.Arguments))
for i, arg := range ext.Arguments {
args[i] = strings.ReplaceAll(arg, SourceFilePlaceholder, carvedFilePath)
}
cmd := exec.Command(ext.Command, args...)
cmd.Dir = outputDir
output, err := cmd.CombinedOutput()
_ = output
exitCode := 0
if err != nil {
if exitErr, ok := err.(*exec.ExitError); ok {
exitCode = exitErr.ExitCode()
} else {
return result
}
}
// Check if exit code is in the accepted list
for _, code := range ext.ExitCodes {
if exitCode == code {
result.Success = true
break
}
}
result.DoNotRecurse = ext.DoNotRecurse
return result
}
// executeInternal runs an internal extraction function.
func executeInternal(ext *Extractor, data []byte, offset int, outputDir string) *ExtractionResult {
if ext.Function == nil {
return &ExtractionResult{}
}
result := ext.Function(data, offset, outputDir)
if result == nil {
result = &ExtractionResult{}
}
result.DoNotRecurse = ext.DoNotRecurse
return result
}
// ExecuteExtraction performs extraction for a signature result.
func ExecuteExtraction(data []byte, filePath string, sig *SignatureResult, ext *Extractor) *ExtractionResult {
if ext == nil {
return &ExtractionResult{}
}
// Use preferred extractor if specified
actualExt := ext
if sig.PreferredExtractor != nil {
actualExt = sig.PreferredExtractor
}
// Build output directory path
baseName := filepath.Base(filePath)
outputDir := filepath.Join(filepath.Dir(filePath), baseName+".extracted",
fmt.Sprintf("%x", sig.Offset))
switch actualExt.Type {
case ExtractorTypeInternal:
return executeInternal(actualExt, data, sig.Offset, outputDir)
case ExtractorTypeExternal:
// Carve data to a temp file for external tools
if err := os.MkdirAll(outputDir, 0755); err != nil {
return &ExtractionResult{OutputDirectory: outputDir}
}
carvedFile := filepath.Join(outputDir, fmt.Sprintf("carved.%s", actualExt.Extension))
endOffset := sig.Offset + sig.Size
if endOffset > len(data) {
endOffset = len(data)
}
if err := os.WriteFile(carvedFile, data[sig.Offset:endOffset], 0644); err != nil {
return &ExtractionResult{OutputDirectory: outputDir}
}
result := executeExternal(actualExt, carvedFile, outputDir)
result.OutputDirectory = outputDir
result.ExtractorName = actualExt.Command
return result
default:
return &ExtractionResult{}
}
}
internal/leviathan/binsnoop/magic.go
+268
View File
@@ -0,0 +1,268 @@
package binsnoop
// AllSignatures returns every supported signature definition.
// This is the equivalent of magic::patterns() in the Rust binwalk codebase.
func AllSignatures() []Signature {
return []Signature{
// ===================== Archives =====================
{Name: "gzip", Magic: gzipMagic(), Parser: gzipParser, Description: gzipDescription, Extractor: extGzip()},
{Name: "deb", Magic: debMagic(), Parser: debParser, Description: debDescription},
{Name: "7zip", Magic: sevenzipMagic(), Parser: sevenzipParser, Description: sevenzipDescription, Extractor: extSevenzip()},
{Name: "xz", Magic: XZMagic(), Parser: XZParser, Description: XZDescription, Extractor: extLzma()},
{Name: "tarball", Magic: tarballMagic(), Parser: tarballParser, Description: tarballDescription, Extractor: extTar()},
{Name: "zip", Magic: zipMagic(), Parser: zipParser, Description: zipDescription, Extractor: extSevenzip()},
{Name: "cpio", Magic: cpioMagic(), Parser: cpioParser, Description: cpioDescription, Extractor: extSevenzip()},
{Name: "iso9660", Magic: iso9660Magic(), Parser: iso9660Parser, Description: iso9660Description, Extractor: extSevenzip()},
{Name: "rar", Magic: rarMagic(), Parser: rarParser, Description: rarDescription, Extractor: extRar()},
{Name: "cab", Magic: cabMagic(), Parser: cabParser, Description: cabDescription, Extractor: extCab()},
{Name: "arj", Magic: arjMagic(), Parser: arjParser, Description: arjDescription, Extractor: extSevenzip()},
{Name: "dahua_zip", Magic: dahuaZipMagic(), Parser: dahuaZipParser, Description: dahuaZipDescription, Extractor: extSevenzip()},
// ===================== Compression =====================
{Name: "lzma", Magic: LZMAMagic(), Parser: LZMAParser, Description: LZMADescription, Extractor: extLzma()},
{Name: "bzip2", Magic: Bzip2Magic(), Parser: Bzip2Parser, Description: Bzip2Description, Extractor: extBzip2()},
{Name: "zstd", Magic: ZSTDMagic(), Parser: ZSTDParser, Description: ZSTDDescription, Extractor: extZstd()},
{Name: "lz4", Magic: LZ4Magic(), Parser: LZ4Parser, Description: LZ4Description, Extractor: extLz4()},
{Name: "lzop", Magic: LZOPMagic(), Parser: LZOPParser, Description: LZOPDescription, Extractor: extLzop()},
{Name: "lzfse", Magic: LZFSEMagic(), Parser: LZFSEParser, Description: LZFSEDescription, Extractor: extLzfse()},
{Name: "zlib", Short: true, Magic: ZlibMagic(), Parser: ZlibParser, Description: ZlibDescription, Extractor: extZlib()},
{Name: "compressd", Short: true, Magic: CompressdMagic(), Parser: CompressdParser, Description: CompressdDescription, Extractor: extSevenzip()},
// ===================== Filesystems =====================
{Name: "squashfs", Magic: squashfs_magic(), Parser: squashfs_parser, Description: SquashFSDescription, Extractor: extSquashfs()},
{Name: "cramfs", Magic: cramfs_magic(), Parser: cramfs_parser, Description: CramFSDescription, Extractor: extSevenzip()},
{Name: "jffs2", Magic: jffs2_magic(), Parser: jffs2_parser, Description: JFFS2Description, Extractor: extJffs2()},
{Name: "yaffs", Magic: yaffs_magic(), Parser: yaffs_parser, Description: YAFFSDescription, Extractor: extYaffs2()},
{Name: "romfs", Magic: romfs_magic(), Parser: romfs_parser, Description: RomFSDescription, Extractor: extRomfs()},
{Name: "ext", Magic: ext_magic(), Parser: ext_parser, Description: EXTDescription, Extractor: extTsk()},
{Name: "fat", Short: true, MagicOffset: FATMagicOffset, Magic: fat_magic(), Parser: fat_parser, Description: FATDescription, Extractor: extTsk()},
{Name: "ntfs", Magic: ntfs_magic(), Parser: ntfs_parser, Description: NTFSDescription, Extractor: extTsk()},
{Name: "apfs", Magic: apfs_magic(), Parser: apfs_parser, Description: APFSDescription, Extractor: extSevenzip()},
{Name: "btrfs", AlwaysDisplay: true, Magic: btrfs_magic(), Parser: btrfs_parser, Description: BTRFSDescription},
{Name: "logfs", Magic: logfs_magic(), Parser: logfs_parser, Description: LogFSDescription},
{Name: "ubi", Magic: ubi_magic(), Parser: ubi_parser, Description: UBIImageDescription, Extractor: extUbi()},
{Name: "ubifs", Magic: ubifs_magic(), Parser: ubifs_parser, Description: UBIFSDescription, Extractor: extUbifs()},
// ===================== Images & Media =====================
{Name: "png", Magic: PNGMagic(), Parser: PNGParser, Description: PNGDescription, Extractor: extPng()},
{Name: "jpeg", Magic: JPEGMagic(), Parser: JPEGParser, Description: JPEGDescription, Extractor: extJpeg()},
{Name: "bmp", Magic: BMPMagic(), Parser: BMPParser, Description: BMPDescription, Extractor: extBmp()},
{Name: "gif", Magic: GIFMagic(), Parser: GIFParser, Description: GIFDescription, Extractor: extGif()},
{Name: "svg", Magic: SVGMagic(), Parser: SVGParser, Description: SVGDescription, Extractor: extSvg()},
{Name: "riff", Magic: RIFFMagic(), Parser: RIFFParser, Description: RIFFDescription, Extractor: extRiff()},
{Name: "pdf", Magic: PDFMagic(), Parser: PDFParser, Description: PDFDescription},
// ===================== Executables =====================
{Name: "elf", Magic: ELFMagic(), Parser: ELFParser, Description: ELFDescription},
{Name: "pe", Magic: PEMagic(), Parser: PEParser, Description: PEDescription},
// ===================== Firmware Headers =====================
{Name: "uimage", Magic: uimageMagic(), Parser: uimageParser, Description: uimageDescription, Extractor: extUimage()},
{Name: "packimg", Magic: packimgMagic(), Parser: packimgParser, Description: packimgDescription},
{Name: "uboot", AlwaysDisplay: true, Magic: ubootMagic(), Parser: ubootParser, Description: ubootDescription},
{Name: "trx", Magic: trxMagic(), Parser: trxParser, Description: trxDescription, Extractor: extTrx()},
{Name: "tplink", Magic: tplinkMagic(), Parser: tplinkParser, Description: tplinkDescription},
{Name: "tplink_rtos", Magic: tplinkRTOSMagic(), Parser: tplinkRTOSParser, Description: tplinkRTOSDescription},
{Name: "dlob", Magic: dlobMagic(), Parser: dlobParser, Description: dlobDescription},
{Name: "arcadyan", Magic: arcadyanMagic(), Parser: arcadyanParser, Description: arcadyanDescription},
{Name: "chk", Magic: chkMagic(), Parser: chkParser, Description: chkDescription},
{Name: "seama", AlwaysDisplay: true, Magic: seamaMagic(), Parser: seamaParser, Description: seamaDescription},
{Name: "cfe", AlwaysDisplay: true, Magic: cfeMagic(), Parser: cfeParser, Description: cfeDescription},
{Name: "jboot_arm", Magic: jbootARMMagic(), Parser: jbootARMParser, Description: jbootARMDescription},
{Name: "jboot_stag", Magic: jbootSTAGMagic(), Parser: jbootSTAGParser, Description: jbootSTAGDescription},
{Name: "jboot_sch2", Magic: jbootSCH2Magic(), Parser: jbootSCH2Parser, Description: jbootSCH2Description, Extractor: extJboot()},
{Name: "binhdr", Magic: binhdrMagic(), Parser: binhdrParser, Description: binhdrDescription},
{Name: "autel", Magic: autelMagic(), Parser: autelParser, Description: autelDescription, Extractor: extAutel()},
{Name: "android_sparse", Magic: androidSparseMagic(), Parser: androidSparseParser, Description: androidSparseDescription, Extractor: extAndroidSparse()},
{Name: "dtb", Magic: dtbMagic(), Parser: dtbParser, Description: dtbDescription, Extractor: extDtb()},
{Name: "android_bootimg", Magic: androidBootimgMagic(), Parser: androidBootimgParser, Description: androidBootimgDescription},
{Name: "dlink_tlv", Magic: dlinkTLVMagic(), Parser: dlinkTLVParser, Description: dlinkTLVDescription, Extractor: extEncfw()},
{Name: "dlke", Magic: dlkeMagic(), Parser: dlkeParser, Description: dlkeDescription, Extractor: extEncfw()},
{Name: "shrs", Magic: shrsMagic(), Parser: shrsParser, Description: shrsDescription, Extractor: extEncfw()},
{Name: "dkbs", Magic: dkbsMagic(), Parser: dkbsParser, Description: dkbsDescription, Extractor: extEncfw()},
{Name: "dms", Magic: dmsMagic(), Parser: dmsParser, Description: dmsDescription},
{Name: "encfw", Short: true, AlwaysDisplay: true, Magic: encfwMagic(), Parser: encfwParser, Description: encfwDescription, Extractor: extEncfw()},
{Name: "encrpted_img", Short: true, Magic: encrptedImgMagic(), Parser: encrptedImgParser, Description: encrptedImgDescription, Extractor: extEncfw()},
{Name: "wince", Magic: winceMagic(), Parser: winceParser, Description: winceDescription},
{Name: "mh01", Magic: mh01Magic(), Parser: mh01Parser, Description: mh01Description},
{Name: "csman", Short: true, Magic: csmanMagic(), Parser: csmanParser, Description: csmanDescription},
{Name: "dxbc", Magic: dxbcMagic(), Parser: dxbcParser, Description: dxbcDescription},
{Name: "matter_ota", Short: true, Magic: matterOTAMagic(), Parser: matterOTAParser, Description: matterOTADescription},
{Name: "rtk", Short: true, Magic: rtkMagic(), Parser: rtkParser, Description: rtkDescription},
{Name: "dpapi", Short: true, AlwaysDisplay: true, Magic: dpapiMagic(), Parser: dpapiParser, Description: dpapiDescription},
{Name: "qcow", Short: true, AlwaysDisplay: true, Magic: qcowMagic(), Parser: qcowParser, Description: qcowDescription},
// ===================== Crypto & Hashes =====================
{Name: "aes_sbox", Magic: aesSBoxMagic(), Parser: aesSBoxParser, Description: DescriptionAESSBox},
{Name: "aes_forward_table", Magic: aesForwardTableMagic(), Parser: aesForwardTableParser, Description: DescriptionAESFT},
{Name: "aes_reverse_table", Magic: aesReverseTableMagic(), Parser: aesReverseTableParser, Description: DescriptionAESRT},
{Name: "aes_rcon", Magic: aesRconMagic(), Parser: aesRconParser, Description: DescriptionAESRCON},
{Name: "aes_acceleration_table", Magic: aesAccelerationTableMagic(), Parser: aesAccelerationTableParser, Description: DescriptionAESAcc},
{Name: "rsa", AlwaysDisplay: true, Magic: rsaMagic(), Parser: rsaParser, Description: DescriptionRSA},
{Name: "openssl", AlwaysDisplay: true, Magic: opensslMagic(), Parser: opensslParser, Description: DescriptionOpenSSL, Extractor: extEncfw()},
{Name: "gpg_signed", Short: true, Magic: gpgSignedMagic(), Parser: gpgSignedParser, Description: DescriptionGPGSigned, Extractor: extGpg()},
{Name: "pem_certificate", Magic: pemCertificateMagic(), Parser: pemParser, Description: DescriptionPEMCertificate, Extractor: extPemCert()},
{Name: "pem_public_key", AlwaysDisplay: true, Magic: pemPublicKeyMagic(), Parser: pemParser, Description: DescriptionPEMPublicKey, Extractor: extPemKey()},
{Name: "pem_private_key", AlwaysDisplay: true, Magic: pemPrivateKeyMagic(), Parser: pemParser, Description: DescriptionPEMPrivateKey, Extractor: extPemKey()},
{Name: "pkcs_der_hash", Magic: derHashMagic(), Parser: derHashParser, Description: DescriptionDERHash},
{Name: "luks", Magic: luksMagic(), Parser: luksParser, Description: DescriptionLUKS},
{Name: "crc32", Magic: crc32ConstMagic(), Parser: crc32Parser, Description: DescriptionCRC32},
{Name: "sha256", Magic: sha256ConstMagic(), Parser: sha256Parser, Description: DescriptionSHA256},
{Name: "md5", Magic: md5ConstMagic(), Parser: md5Parser, Description: DescriptionMD5},
// ===================== Misc =====================
{Name: "copyright", Magic: copyrightMagic(), Parser: copyrightParser, Description: DescriptionCopyright},
{Name: "wind_kernel", AlwaysDisplay: true, Magic: windKernelMagic(), Parser: windKernelParser, Description: DescriptionWINDKernelVersion},
{Name: "vxworks_symtab", AlwaysDisplay: true, Magic: vxworksSymtabMagic(), Parser: vxworksSymtabParser, Description: DescriptionVxWorksSymTab},
{Name: "ecos", AlwaysDisplay: true, Magic: ecosExceptionHandlerMagic(), Parser: ecosExceptionHandlerParser, Description: DescriptionECosExceptionHandler},
{Name: "linux_kernel", AlwaysDisplay: true, Magic: linuxKernelVersionMagic(), Parser: linuxKernelVersionParser, Description: DescriptionLinuxKernelVersion, Extractor: extLinuxKernel()},
{Name: "linux_boot_image", Magic: linuxBootImageMagic(), Parser: linuxBootImageParser, Description: DescriptionLinuxBootImage},
{Name: "linux_arm_zimage", Magic: linuxARMzImageMagic(), Parser: linuxARMzImageParser, Description: DescriptionLinuxARMzImage},
{Name: "linux_arm64_boot_image", Magic: linuxARM64BootImageMagic(), Parser: linuxARM64BootImageParser, Description: DescriptionLinuxARM64BootImage},
{Name: "efigpt", Magic: efigptMagic(), Parser: efigptParser, Description: DescriptionEFIGPT, Extractor: extSevenzip()},
{Name: "mbr", Short: true, MagicOffset: MBRMagicOffset, AlwaysDisplay: true, Magic: mbrMagic(), Parser: mbrParser, Description: DescriptionMBR, Extractor: extMbr()},
{Name: "uefi_pi_volume", Magic: uefiVolumeMagic(), Parser: uefiVolumeParser, Description: DescriptionUEFIVolume, Extractor: extUefi()},
{Name: "uefi_capsule", Magic: uefiCapsuleMagic(), Parser: uefiCapsuleParser, Description: DescriptionUEFICapsule, Extractor: extUefi()},
{Name: "pchrom", Magic: pchRomMagic(), Parser: pchRomParser, Description: DescriptionPCHRom, Extractor: extUefi()},
{Name: "pcapng", Magic: pcapngMagic(), Parser: pcapngParser, Description: DescriptionPcapNG, Extractor: extPcap()},
{Name: "pjl", Magic: pjlMagic(), Parser: pjlParser, Description: DescriptionPJL},
{Name: "srecord", Magic: srecordMagic(), Parser: srecordParser, Description: DescriptionSRecord, Extractor: extSrec()},
{Name: "srecord_generic", Short: true, Magic: srecordGenericMagic(), Parser: srecordParser, Description: DescriptionSRecordGeneric, Extractor: extSrec()},
{Name: "dmg", Magic: dmgMagic(), Parser: dmgParser, Description: DescriptionDMG, Extractor: extDmg()},
{Name: "qnx_ifs", Magic: qnxIFSMagic(), Parser: qnxIFSParser, Description: DescriptionQNXIFS, Extractor: extDumpifs()},
}
}
// ============================================================================
// External extractor definitions (command-line tools)
// ============================================================================
func extSevenzip() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extGzip() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "gzip", Extension: "gz", Arguments: []string{"-d", "-k", "-f", "%e"}, ExitCodes: []int{0}}
}
func extLzma() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "xz", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extTar() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "tar", Extension: "tar", Arguments: []string{"xf", "%e"}, ExitCodes: []int{0}}
}
func extBzip2() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "bzip2", Extension: "bz2", Arguments: []string{"-d", "-k", "-f", "%e"}, ExitCodes: []int{0}}
}
func extZstd() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "zstd", Extension: "zst", Arguments: []string{"-d", "-f", "%e"}, ExitCodes: []int{0}}
}
func extLz4() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "lz4", Extension: "lz4", Arguments: []string{"-d", "-f", "%e", "decompressed.bin"}, ExitCodes: []int{0}}
}
func extLzop() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "lzop", Extension: "lzo", Arguments: []string{"-d", "-f", "%e"}, ExitCodes: []int{0}}
}
func extLzfse() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "lzfse", Extension: "lzfse", Arguments: []string{"-decode", "-i", "%e", "-o", "decompressed.bin"}, ExitCodes: []int{0}}
}
func extZlib() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extSquashfs() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "unsquashfs", Extension: "sqsh", Arguments: []string{"-f", "-d", "squashfs-root", "%e"}, ExitCodes: []int{0}}
}
func extJffs2() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "jefferson", Extension: "jffs2", Arguments: []string{"-d", "jffs2-root", "%e"}, ExitCodes: []int{0}}
}
func extYaffs2() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "unyaffs", Extension: "yaffs", Arguments: []string{"%e", "yaffs-root"}, ExitCodes: []int{0}}
}
func extRomfs() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "romfs", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extTsk() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "tsk_recover", Extension: "img", Arguments: []string{"-a", "-e", "%e", "tsk-root"}, ExitCodes: []int{0}}
}
func extCab() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cabextract", Extension: "cab", Arguments: []string{"%e"}, ExitCodes: []int{0}}
}
func extRar() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "unrar", Extension: "rar", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0}}
}
func extUbi() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "ubireader_extract_images", Extension: "ubi", Arguments: []string{"%e"}, ExitCodes: []int{0}}
}
func extUbifs() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "ubireader_extract_files", Extension: "ubifs", Arguments: []string{"%e"}, ExitCodes: []int{0}}
}
func extPng() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "png", Arguments: []string{"%e", "image.png"}, ExitCodes: []int{0}}
}
func extJpeg() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "jpg", Arguments: []string{"%e", "image.jpg"}, ExitCodes: []int{0}}
}
func extBmp() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "bmp", Arguments: []string{"%e", "image.bmp"}, ExitCodes: []int{0}}
}
func extGif() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "gif", Arguments: []string{"%e", "image.gif"}, ExitCodes: []int{0}}
}
func extSvg() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "svg", Arguments: []string{"%e", "image.svg"}, ExitCodes: []int{0}}
}
func extRiff() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "riff", Arguments: []string{"%e", "media.riff"}, ExitCodes: []int{0}}
}
func extUimage() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extTrx() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extJboot() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extAutel() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extAndroidSparse() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "simg2img", Extension: "simg", Arguments: []string{"%e", "output.img"}, ExitCodes: []int{0}}
}
func extDtb() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "dtc", Extension: "dtb", Arguments: []string{"-I", "dtb", "-O", "dts", "-o", "device_tree.dts", "%e"}, ExitCodes: []int{0}}
}
func extEncfw() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "bin", Arguments: []string{"%e", "encrypted_firmware.bin"}, ExitCodes: []int{0}}
}
func extGpg() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "gpg", Extension: "gpg", Arguments: []string{"--decrypt", "-o", "decrypted.bin", "%e"}, ExitCodes: []int{0}}
}
func extPemCert() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pem", Arguments: []string{"%e", "certificate.pem"}, ExitCodes: []int{0}}
}
func extPemKey() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pem", Arguments: []string{"%e", "key.pem"}, ExitCodes: []int{0}}
}
func extLinuxKernel() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extMbr() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extUefi() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "uefi-firmware-parser", Extension: "bin", Arguments: []string{"-e", "-O", "%e"}, ExitCodes: []int{0}}
}
func extPcap() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pcapng", Arguments: []string{"%e", "capture.pcapng"}, ExitCodes: []int{0}}
}
func extSrec() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "srec_cat", Extension: "srec", Arguments: []string{"%e", "-o", "output.bin", "-binary"}, ExitCodes: []int{0}}
}
func extDmg() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "dmg", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
func extDumpifs() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "dumpifs", Extension: "ifs", Arguments: []string{"-x", "%e"}, ExitCodes: []int{0}}
}
func extIso9660() *Extractor {
return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "iso", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}}
}
internal/leviathan/binsnoop/match.go
+134
View File
@@ -0,0 +1,134 @@
package binsnoop
// ahoCorasick implements the Aho-Corasick multi-pattern byte matching algorithm.
type ahoCorasick struct {
gotoFn []map[byte]int
failFn []int
outputFn [][]int
patterns [][]byte
}
// ahoMatch represents a single pattern match.
type ahoMatch struct {
PatternIndex int
Start int
}
// newAhoCorasick builds an Aho-Corasick automaton from the given byte patterns.
func newAhoCorasick(patterns [][]byte) *ahoCorasick {
ac := &ahoCorasick{
gotoFn: []map[byte]int{make(map[byte]int)},
failFn: []int{0},
outputFn: [][]int{nil},
patterns: patterns,
}
// Build the trie (goto function)
for i, pattern := range patterns {
state := 0
for _, b := range pattern {
if next, ok := ac.gotoFn[state][b]; ok {
state = next
} else {
newState := len(ac.gotoFn)
ac.gotoFn[state][b] = newState
ac.gotoFn = append(ac.gotoFn, make(map[byte]int))
ac.failFn = append(ac.failFn, 0)
ac.outputFn = append(ac.outputFn, nil)
state = newState
}
}
ac.outputFn[state] = append(ac.outputFn[state], i)
}
// Build failure links (BFS)
queue := make([]int, 0)
for _, next := range ac.gotoFn[0] {
queue = append(queue, next)
}
for len(queue) > 0 {
r := queue[0]
queue = queue[1:]
for b, s := range ac.gotoFn[r] {
queue = append(queue, s)
state := ac.failFn[r]
for {
if _, ok := ac.gotoFn[state][b]; ok || state == 0 {
break
}
state = ac.failFn[state]
}
if next, ok := ac.gotoFn[state][b]; ok && next != s {
ac.failFn[s] = next
} else {
ac.failFn[s] = 0
}
// Merge output from failure state
if ac.outputFn[ac.failFn[s]] != nil {
ac.outputFn[s] = append(ac.outputFn[s], ac.outputFn[ac.failFn[s]]...)
}
}
}
return ac
}
// findOverlapping searches data starting at startOffset and returns all overlapping matches.
func (ac *ahoCorasick) findOverlapping(data []byte, startOffset int) []ahoMatch {
var matches []ahoMatch
state := 0
for i := startOffset; i < len(data); i++ {
b := data[i]
for {
if _, ok := ac.gotoFn[state][b]; ok || state == 0 {
break
}
state = ac.failFn[state]
}
if next, ok := ac.gotoFn[state][b]; ok {
state = next
}
for _, patIdx := range ac.outputFn[state] {
matchStart := i - len(ac.patterns[patIdx]) + 1
matches = append(matches, ahoMatch{
PatternIndex: patIdx,
Start: matchStart,
})
}
}
return matches
}
// findOverlappingLimit is the bounded variant used by deep scans. It stops
// collecting once limit matches have been found so adversarial input cannot
// force an unbounded match slice.
func (ac *ahoCorasick) findOverlappingLimit(data []byte, startOffset, limit int) ([]ahoMatch, bool) {
if limit <= 0 {
return nil, false
}
matches := make([]ahoMatch, 0, min(limit, 256))
state := 0
for i := startOffset; i < len(data); i++ {
b := data[i]
for {
if _, ok := ac.gotoFn[state][b]; ok || state == 0 {
break
}
state = ac.failFn[state]
}
if next, ok := ac.gotoFn[state][b]; ok {
state = next
}
for _, patIdx := range ac.outputFn[state] {
matches = append(matches, ahoMatch{PatternIndex: patIdx, Start: i - len(ac.patterns[patIdx]) + 1})
if len(matches) >= limit {
return matches, true
}
}
}
return matches, false
}
internal/leviathan/binsnoop/parse.go
+107
View File
@@ -0,0 +1,107 @@
package binsnoop
import (
"errors"
"fmt"
)
// ErrParse is returned when structure parsing fails.
var ErrParse = errors.New("structure parse error")
// StructField defines a single field in a binary structure.
type StructField struct {
Name string
Type string // "u8", "u16", "u24", "u32", "u64"
}
// typeSize returns the byte size for a type string.
func typeSize(ctype string) (int, error) {
switch ctype {
case "u8":
return 1, nil
case "u16":
return 2, nil
case "u24":
return 3, nil
case "u32":
return 4, nil
case "u64":
return 8, nil
default:
return 0, fmt.Errorf("unknown type: %s", ctype)
}
}
// StructSize returns the total byte size of a structure definition.
func StructSize(fields []StructField) int {
total := 0
for _, f := range fields {
sz, err := typeSize(f.Type)
if err != nil {
continue
}
total += sz
}
return total
}
// Parse parses a binary structure from data according to the field definitions.
// Endianness must be "big" or "little". All values are returned as int.
func Parse(data []byte, fields []StructField, endianness string) (map[string]int, error) {
structSize := StructSize(fields)
if len(data) < structSize {
return nil, ErrParse
}
result := make(map[string]int, len(fields))
offset := 0
for _, f := range fields {
sz, err := typeSize(f.Type)
if err != nil {
return nil, ErrParse
}
var value int
switch sz {
case 1:
value = int(data[offset])
case 2:
if endianness == "big" {
value = int(data[offset])<<8 | int(data[offset+1])
} else {
value = int(data[offset+1])<<8 | int(data[offset])
}
case 3:
if endianness == "big" {
value = int(data[offset])<<16 | int(data[offset+1])<<8 | int(data[offset+2])
} else {
value = int(data[offset+2])<<16 | int(data[offset+1])<<8 | int(data[offset])
}
case 4:
if endianness == "big" {
value = int(data[offset])<<24 | int(data[offset+1])<<16 | int(data[offset+2])<<8 | int(data[offset+3])
} else {
value = int(data[offset+3])<<24 | int(data[offset+2])<<16 | int(data[offset+1])<<8 | int(data[offset])
}
case 8:
if endianness == "big" {
value = int(uint64(data[offset])<<56 | uint64(data[offset+1])<<48 |
uint64(data[offset+2])<<40 | uint64(data[offset+3])<<32 |
uint64(data[offset+4])<<24 | uint64(data[offset+5])<<16 |
uint64(data[offset+6])<<8 | uint64(data[offset+7]))
} else {
value = int(uint64(data[offset+7])<<56 | uint64(data[offset+6])<<48 |
uint64(data[offset+5])<<40 | uint64(data[offset+4])<<32 |
uint64(data[offset+3])<<24 | uint64(data[offset+2])<<16 |
uint64(data[offset+1])<<8 | uint64(data[offset]))
}
}
offset += sz
result[f.Name] = value
}
return result, nil
}
internal/leviathan/binsnoop/scanner.go
+352
View File
@@ -0,0 +1,352 @@
package binsnoop
import "sort"
// Scanner is the core scanning engine. It holds compiled signature patterns
// and performs Aho-Corasick based multi-pattern matching on binary data.
type Scanner struct {
signatureCount int
patternCount int
shortSignatures []Signature
patterns [][]byte
patternSigTable map[int]Signature
extractorLookup map[string]*Extractor
ac *ahoCorasick
}
// compile builds the Aho-Corasick automaton from loaded patterns.
func (s *Scanner) compile() {
if len(s.patterns) > 0 {
s.ac = newAhoCorasick(s.patterns)
}
}
// Scan searches data for all known file signatures and returns validated results.
// This implements the same algorithm as the Rust binwalk scanner:
// 1. Check short signatures at file offset 0
// 2. Run Aho-Corasick scan with restart on high-confidence sized matches
// 3. Sort and de-conflict overlapping results
// 4. Estimate sizes for signatures with unknown size
func (s *Scanner) Scan(data []byte) []SignatureResult {
var fileMap []SignatureResult
nextValidOffset := 0
availableData := len(data)
// Phase 1: Check short signatures at offset 0
for i := range s.shortSignatures {
sig := &s.shortSignatures[i]
for _, magic := range sig.Magic {
magicStart := sig.MagicOffset
magicEnd := magicStart + len(magic)
if len(data) >= magicEnd && bytesEqual(data[magicStart:magicEnd], magic) {
result, err := sig.Parser(data, magicStart)
if err != nil {
continue
}
autoPopulate(result, sig)
fileMap = append(fileMap, *result)
if result.Confidence >= ConfidenceHigh {
nextValidOffset = result.Offset + result.Size
}
break
}
}
}
// Phase 2: Aho-Corasick scan.
//
// The automaton is run exactly once over the whole buffer; we then walk
// the matches in position order. When a high/medium-confidence sized
// signature is accepted, nextValidOffset jumps past it and any later
// matches that start inside that consumed region are skipped. This is the
// same result the binwalk scanner produces, but without re-running the
// search from every new offset — a previous implementation rescanned the
// tail of the file on each accepted match, which is O(n^2) on inputs that
// contain many closely-spaced signatures (e.g. real firmware images) and
// could appear to hang.
if s.ac == nil {
return s.postProcess(data, fileMap)
}
matches := s.ac.findOverlapping(data, nextValidOffset)
for _, m := range matches {
magicOffset := m.Start
if magicOffset < nextValidOffset {
continue
}
sig, ok := s.patternSigTable[m.PatternIndex]
if !ok {
continue
}
result, err := sig.Parser(data, magicOffset)
if err != nil {
continue
}
sigEnd := result.Offset + result.Size
if sigEnd > availableData {
continue
}
autoPopulate(result, &sig)
fileMap = append(fileMap, *result)
if result.Confidence >= ConfidenceMedium && result.Size > 0 {
nextValidOffset = sigEnd
}
}
return s.postProcess(data, fileMap)
}
// postProcess sorts results, removes conflicts, and estimates unknown sizes.
func (s *Scanner) postProcess(data []byte, fileMap []SignatureResult) []SignatureResult {
if len(fileMap) == 0 {
return fileMap
}
// Sort by offset
sort.Slice(fileMap, func(i, j int) bool {
return fileMap[i].Offset < fileMap[j].Offset
})
// De-conflict overlapping signatures in a single pass, filtering in place.
// (The previous implementation removed elements with
// append(fileMap[:i], fileMap[i+1:]...) inside the loop; each removal is
// O(n), so a file with many overlapping hits was O(n^2) and could appear
// to hang.) Removed entries are dropped; on a duplicate offset the
// higher-confidence signature wins, matching the original behaviour.
out := fileMap[:0]
nextValidOffset := 0
for _, thisSig := range fileMap {
// Duplicate offset: keep only the highest-confidence signature.
if len(out) > 0 && thisSig.Offset == out[len(out)-1].Offset {
if thisSig.Confidence > out[len(out)-1].Confidence {
out[len(out)-1] = thisSig
if thisSig.Confidence >= ConfidenceMedium {
nextValidOffset = thisSig.Offset + thisSig.Size
}
}
continue
}
// Drop signatures that fall inside a previously accepted sized region.
if thisSig.Offset < nextValidOffset {
continue
}
// Drop signatures whose declared size runs past EOF or wraps.
remainingSize := len(data) - thisSig.Offset
if thisSig.Size > remainingSize || (thisSig.Offset+thisSig.Size) < 0 {
continue
}
out = append(out, thisSig)
if thisSig.Confidence >= ConfidenceMedium {
nextValidOffset = thisSig.Offset + thisSig.Size
}
}
fileMap = out
// Estimate sizes for signatures with unknown size (0): extend each to the
// offset of the next medium-or-higher confidence signature, or to EOF.
// Computed in a single right-to-left pass to stay O(n).
nextMediumOffset := len(data)
for i := len(fileMap) - 1; i >= 0; i-- {
if fileMap[i].Size == 0 {
fileMap[i].Size = nextMediumOffset - fileMap[i].Offset
}
if fileMap[i].Confidence >= ConfidenceMedium {
nextMediumOffset = fileMap[i].Offset
}
}
return fileMap
}
// ScanDeep validates signatures without suppressing hits inside an already
// identified container. Results are bounded and annotated with containment.
func (s *Scanner) ScanDeep(data []byte, maxResults int) DeepScan {
if maxResults <= 0 || maxResults > 4096 {
maxResults = 4096
}
result := DeepScan{}
if len(data) == 0 {
return result
}
results := make([]SignatureResult, 0)
seen := make(map[struct {
offset int
name string
}]int)
add := func(parsed *SignatureResult, sig *Signature) {
if parsed == nil || parsed.Offset < 0 || parsed.Offset > len(data) {
return
}
if parsed.Size < 0 || parsed.Size > len(data)-parsed.Offset {
return
}
autoPopulate(parsed, sig)
key := struct {
offset int
name string
}{parsed.Offset, parsed.Name}
if idx, ok := seen[key]; ok {
if parsed.Confidence > results[idx].Confidence {
results[idx] = *parsed
}
return
}
if len(results) >= maxResults {
result.Truncated = true
return
}
seen[key] = len(results)
results = append(results, *parsed)
}
for i := range s.shortSignatures {
sig := &s.shortSignatures[i]
for _, magic := range sig.Magic {
start := sig.MagicOffset
end := start + len(magic)
if start >= 0 && end <= len(data) && bytesEqual(data[start:end], magic) {
parsed, err := sig.Parser(data, start)
if err == nil {
add(parsed, sig)
}
break
}
}
}
if s.ac != nil && len(results) < maxResults {
matchLimit := maxResults * 16
matches, truncated := s.ac.findOverlappingLimit(data, 0, matchLimit)
result.Truncated = result.Truncated || truncated
for _, match := range matches {
sig, ok := s.patternSigTable[match.PatternIndex]
if !ok {
continue
}
parsed, err := sig.Parser(data, match.Start)
if err != nil {
continue
}
add(parsed, &sig)
if len(results) >= maxResults {
result.Truncated = true
break
}
}
}
sort.Slice(results, func(i, j int) bool {
if results[i].Offset != results[j].Offset {
return results[i].Offset < results[j].Offset
}
if results[i].Confidence != results[j].Confidence {
return results[i].Confidence > results[j].Confidence
}
return results[i].Size > results[j].Size
})
unique := results[:0]
for _, hit := range results {
if len(unique) > 0 && unique[len(unique)-1].Offset == hit.Offset {
continue
}
unique = append(unique, hit)
}
results = unique
nextOffset := len(data)
for i := len(results) - 1; i >= 0; i-- {
if results[i].Size == 0 {
results[i].Size = nextOffset - results[i].Offset
}
nextOffset = results[i].Offset
}
active := make([]int, 0)
for _, hit := range results {
end := hit.Offset + hit.Size
kept := active[:0]
for _, idx := range active {
if result.Results[idx].End > hit.Offset {
kept = append(kept, idx)
}
}
active = kept
parent := -1
overlap := false
for i := len(active) - 1; i >= 0; i-- {
candidate := result.Results[active[i]]
if candidate.End >= end && candidate.Offset < hit.Offset {
parent = active[i]
break
}
overlap = true
}
deep := DeepScanResult{SignatureResult: hit, End: end, ParentOffset: -1, Overlap: overlap}
if parent >= 0 {
deep.ParentOffset = result.Results[parent].Offset
deep.Depth = result.Results[parent].Depth + 1
}
result.Results = append(result.Results, deep)
active = append(active, len(result.Results)-1)
}
return result
}
// Extract performs extraction on all extractable signatures in the file map.
func (s *Scanner) Extract(data []byte, filePath string, fileMap []SignatureResult) map[string]*ExtractionResult {
results := make(map[string]*ExtractionResult)
for _, sig := range fileMap {
if sig.ExtractionDeclined {
continue
}
ext, ok := s.extractorLookup[sig.Name]
if !ok || ext == nil {
continue
}
result := ExecuteExtraction(data, filePath, &sig, ext)
if !result.Success && sig.Size < len(data)-sig.Offset {
// Retry with all available data
expandedSig := sig
expandedSig.Size = len(data) - sig.Offset
result = ExecuteExtraction(data, filePath, &expandedSig, ext)
}
results[sig.ID] = result
}
return results
}
// bytesEqual compares two byte slices.
func bytesEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
// autoPopulate fills in auto-generated fields on a SignatureResult.
func autoPopulate(result *SignatureResult, sig *Signature) {
result.ID = newUUID()
result.Name = sig.Name
result.AlwaysDisplay = sig.AlwaysDisplay
}
internal/leviathan/binsnoop/sig_archives.go
+1143
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/binsnoop/sig_compression.go
+1081
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/binsnoop/sig_crypto.go
+795
View File
@@ -0,0 +1,795 @@
package binsnoop
import (
"encoding/base64"
"fmt"
"strings"
)
// ---- AES descriptions ----
const (
DescriptionAESSBox = "AES S-Box"
DescriptionAESFT = "AES Forward Table"
DescriptionAESRT = "AES Reverse Table"
DescriptionAESRCON = "AES RCON"
DescriptionAESAcc = "AES Acceleration Table"
)
// aesSBoxMagic returns magic bytes for AES S-Box (forward and reverse).
func aesSBoxMagic() [][]byte {
return [][]byte{
{0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5}, // Forward S-Box
{0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38}, // Reverse S-Box
}
}
// aesSBoxParser validates AES S-Box signatures.
func aesSBoxParser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Description: DescriptionAESSBox,
Confidence: ConfidenceLow,
}, nil
}
// aesForwardTableMagic returns magic bytes for AES Forward Table.
func aesForwardTableMagic() [][]byte {
return [][]byte{
{0xC6, 0x63, 0x63, 0xA5, 0xF8, 0x7C, 0x7C, 0x84, 0xEE, 0x77, 0x77, 0x99, 0xF6, 0x7B, 0x7B, 0x8D},
}
}
// aesForwardTableParser validates AES Forward Table signatures.
func aesForwardTableParser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Description: DescriptionAESFT,
Confidence: ConfidenceLow,
}, nil
}
// aesReverseTableMagic returns magic bytes for AES Reverse Table.
func aesReverseTableMagic() [][]byte {
return [][]byte{
{0x51, 0xF4, 0xA7, 0x50, 0x7E, 0x41, 0x65, 0x53, 0x1A, 0x17, 0xA4, 0xC3, 0x3A, 0x27, 0x5E, 0x96},
}
}
// aesReverseTableParser validates AES Reverse Table signatures.
func aesReverseTableParser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Description: DescriptionAESRT,
Confidence: ConfidenceLow,
}, nil
}
// aesRconMagic returns magic bytes for AES RCON.
func aesRconMagic() [][]byte {
return [][]byte{
{0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36},
{0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00,
0x10, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00,
0x1B, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00},
}
}
// aesRconParser validates AES RCON signatures.
func aesRconParser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Description: DescriptionAESRCON,
Confidence: ConfidenceLow,
}, nil
}
// aesAccelerationTableMagic returns magic bytes for AES Acceleration Table.
func aesAccelerationTableMagic() [][]byte {
return [][]byte{
// combined sbox x2
{0xA5, 0x84, 0x99, 0x8D, 0x0D, 0xBD, 0xB1, 0x54, 0x50, 0x03, 0xA9, 0x7D, 0x19, 0x62, 0xE6, 0x9A},
// combined sbox x3
{0xC6, 0xF8, 0xEE, 0xF6, 0xFF, 0xD6, 0xDE, 0x91, 0x60, 0x02, 0xCE, 0x56, 0xE7, 0xB5, 0x4D, 0xEC},
// Galois mult 2
{0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e,
0x20, 0x22, 0x24, 0x26, 0x28, 0x2a, 0x2c, 0x2e},
// Galois mult 3
{0x00, 0x03, 0x06, 0x05, 0x0c, 0x0f, 0x0a, 0x09, 0x18, 0x1b, 0x1e, 0x1d, 0x14, 0x17, 0x12, 0x11,
0x30, 0x33, 0x36, 0x35, 0x3c, 0x3f, 0x3a, 0x39},
// Galois mult 9
{0x00, 0x09, 0x12, 0x1b, 0x24, 0x2d, 0x36, 0x3f, 0x48, 0x41, 0x5a, 0x53, 0x6c, 0x65, 0x7e, 0x77,
0x90, 0x99, 0x82, 0x8b, 0xb4, 0xbd, 0xa6, 0xaf},
// Galois mult 11
{0x00, 0x0b, 0x16, 0x1d, 0x2c, 0x27, 0x3a, 0x31, 0x58, 0x53, 0x4e, 0x45, 0x74, 0x7f, 0x62, 0x69,
0xb0, 0xbb, 0xa6, 0xad, 0x9c, 0x97, 0x8a, 0x81},
// Galois mult 13
{0x00, 0x0d, 0x1a, 0x17, 0x34, 0x39, 0x2e, 0x23, 0x68, 0x65, 0x72, 0x7f, 0x5c, 0x51, 0x46, 0x4b,
0xd0, 0xdd, 0xca, 0xc7, 0xe4, 0xe9, 0xfe, 0xf3},
// Galois mult 14
{0x00, 0x0e, 0x1c, 0x12, 0x38, 0x36, 0x24, 0x2a, 0x70, 0x7e, 0x6c, 0x62, 0x48, 0x46, 0x54, 0x5a,
0xe0, 0xee, 0xfc, 0xf2, 0xd8, 0xd6, 0xc4, 0xca},
}
}
// aesAccelerationTableParser validates AES Acceleration Table signatures.
func aesAccelerationTableParser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Description: DescriptionAESAcc,
Confidence: ConfidenceLow,
}, nil
}
// ---- RSA ----
const DescriptionRSA = "RSA encrypted session key"
type rsaKeyDefinition struct {
magic []byte
keySize int
usageOffset int
keyIDOffset int
validChecks map[int][][]byte
terminatorOffset int
}
func rsaKeyDefinitions() []rsaKeyDefinition {
return []rsaKeyDefinition{
// 1024b RSA key
{
magic: []byte{0x84, 0x8C, 0x03},
keySize: 1024,
keyIDOffset: 3,
terminatorOffset: 142,
usageOffset: 11,
validChecks: map[int][][]byte{
12: {
{0x04, 0x00}, {0x03, 0xff}, {0x03, 0xfe}, {0x03, 0xfd},
{0x03, 0xfc}, {0x03, 0xfb}, {0x03, 0xfa}, {0x03, 0xf9},
},
},
},
// 2048b RSA key
{
magic: []byte{0x85, 0x01, 0x0c, 0x03},
keySize: 2048,
keyIDOffset: 4,
terminatorOffset: 271,
usageOffset: 12,
validChecks: map[int][][]byte{
13: {
{0x08, 0x00}, {0x07, 0xff}, {0x07, 0xfe}, {0x07, 0xfd},
{0x07, 0xfc}, {0x07, 0xfb}, {0x07, 0xfa}, {0x07, 0xf9},
},
},
},
// 3072b RSA key
{
magic: []byte{0x85, 0x01, 0x8c, 0x03},
keySize: 3072,
keyIDOffset: 4,
terminatorOffset: 399,
usageOffset: 12,
validChecks: map[int][][]byte{
13: {
{0x0c, 0x00}, {0x0b, 0xff}, {0x0b, 0xfe}, {0x0b, 0xfd},
{0x0b, 0xfc}, {0x0b, 0xfb}, {0x0b, 0xfa}, {0x0b, 0xf9},
},
},
},
// 4096b RSA key
{
magic: []byte{0x85, 0x02, 0x0c, 0x03},
keySize: 4096,
keyIDOffset: 4,
terminatorOffset: 527,
usageOffset: 12,
validChecks: map[int][][]byte{
13: {
{0x10, 0x00}, {0x0f, 0xff}, {0x0f, 0xfe}, {0x0f, 0xfd},
{0x0f, 0xfc}, {0x0f, 0xfb}, {0x0f, 0xfa}, {0x0f, 0xf9},
},
},
},
// 8192b RSA key
{
magic: []byte{0x85, 0x04, 0x0c, 0x03},
keySize: 8192,
keyIDOffset: 4,
terminatorOffset: 1039,
usageOffset: 12,
validChecks: map[int][][]byte{
13: {
{0x20, 0x00}, {0x1f, 0xff}, {0x1f, 0xfe}, {0x1f, 0xfd},
{0x1f, 0xfc}, {0x1f, 0xfb}, {0x1f, 0xfa}, {0x1f, 0xf9},
},
},
},
}
}
// rsaMagic returns magic bytes for all known RSA key sizes.
func rsaMagic() [][]byte {
var magics [][]byte
for _, def := range rsaKeyDefinitions() {
m := make([]byte, len(def.magic))
copy(m, def.magic)
magics = append(magics, m)
}
return magics
}
type rsaKeyInfo struct {
bits int
canSign bool
canEncrypt bool
keyID int
dataSize int
}
func rsaKeyParse(rawData []byte, def *rsaKeyDefinition) (*rsaKeyInfo, error) {
const terminatorByte byte = 0xD2
const encryptOnly byte = 2
const signAndEncrypt byte = 1
const validBytesSize = 2
keyIDFields := []StructField{{"id", "u64"}}
keyDataLen := def.terminatorOffset + 1
if len(rawData) < keyDataLen {
return nil, fmt.Errorf("insufficient RSA key data")
}
keyData := rawData[:keyDataLen]
if keyData[def.terminatorOffset] != terminatorByte {
return nil, fmt.Errorf("invalid RSA terminator byte")
}
if def.keyIDOffset+8 > len(keyData) {
return nil, fmt.Errorf("key ID offset out of bounds")
}
keyIDParsed, err := Parse(keyData[def.keyIDOffset:], keyIDFields, "big")
if err != nil {
return nil, fmt.Errorf("failed to parse RSA key ID")
}
info := &rsaKeyInfo{keyID: keyIDParsed["id"]}
info.canEncrypt = keyData[def.usageOffset] == encryptOnly
info.canSign = keyData[def.usageOffset] == signAndEncrypt
if info.canSign {
info.canEncrypt = true
}
if !info.canSign && !info.canEncrypt {
return nil, fmt.Errorf("RSA key cannot sign or encrypt")
}
for checkOffset, expectedList := range def.validChecks {
if checkOffset+validBytesSize > len(keyData) {
continue
}
actual := keyData[checkOffset : checkOffset+validBytesSize]
for _, expected := range expectedList {
if len(expected) == validBytesSize && actual[0] == expected[0] && actual[1] == expected[1] {
info.bits = def.keySize
info.dataSize = keyDataLen
return info, nil
}
}
}
return nil, fmt.Errorf("RSA key validation bytes mismatch")
}
// rsaParser validates RSA encrypted session key signatures.
func rsaParser(data []byte, offset int) (*SignatureResult, error) {
result := &SignatureResult{
Offset: offset,
Description: DescriptionRSA,
Confidence: ConfidenceMedium,
}
for _, def := range rsaKeyDefinitions() {
magicEnd := offset + len(def.magic)
if magicEnd > len(data) {
continue
}
match := true
for i, b := range def.magic {
if data[offset+i] != b {
match = false
break
}
}
if !match {
continue
}
info, err := rsaKeyParse(data[offset:], &def)
if err != nil {
break
}
result.Size = info.dataSize
result.Description = fmt.Sprintf("%s, %d bits, can sign: %t, can encrypt: %t, key ID: %#018X",
result.Description, info.bits, info.canSign, info.canEncrypt, info.keyID)
return result, nil
}
return nil, fmt.Errorf("invalid RSA signature")
}
// ---- OpenSSL ----
const DescriptionOpenSSL = "OpenSSL encryption"
// opensslMagic returns magic bytes for OpenSSL encrypted files.
func opensslMagic() [][]byte {
return [][]byte{
[]byte("Salted__"),
}
}
func isSaltInvalid(salt int) bool {
const saltLen = 8
badByteCount := 0
for i := 0; i < saltLen; i++ {
saltByte := byte((salt >> (8 * i)) & 0xFF)
if saltByte == 0 || IsPrintableASCII(saltByte) {
badByteCount++
}
}
return badByteCount == saltLen
}
// opensslParser validates OpenSSL encryption signatures.
func opensslParser(data []byte, offset int) (*SignatureResult, error) {
sslFields := []StructField{{"magic", "u64"}, {"salt", "u64"}}
result := &SignatureResult{
Offset: offset,
Description: DescriptionOpenSSL,
Confidence: ConfidenceLow,
}
if offset+StructSize(sslFields) > len(data) {
return nil, fmt.Errorf("insufficient OpenSSL data")
}
sslHeader, err := Parse(data[offset:], sslFields, "big")
if err != nil {
return nil, fmt.Errorf("failed to parse OpenSSL header")
}
salt := sslHeader["salt"]
if salt == 0 {
return nil, fmt.Errorf("OpenSSL salt is zero")
}
if isSaltInvalid(salt) {
return nil, fmt.Errorf("OpenSSL salt is invalid")
}
if offset == 0 {
result.Confidence = ConfidenceMedium
}
result.Description = fmt.Sprintf("%s, salt: %#X", result.Description, salt)
return result, nil
}
// ---- GPG ----
const DescriptionGPGSigned = "GPG signed file"
// gpgSignedMagic returns magic bytes for GPG signed files.
func gpgSignedMagic() [][]byte {
return [][]byte{
{0xA3, 0x01},
}
}
// gpgSignedParser validates GPG signed file signatures.
// The Rust version does a zlib decompression dry run; here we do a simplified validation.
func gpgSignedParser(data []byte, offset int) (*SignatureResult, error) {
result := &SignatureResult{
Offset: offset,
Confidence: ConfidenceHigh,
Description: DescriptionGPGSigned,
}
// GPG signed files are zlib compressed with magic bytes replaced.
// Validate there is enough data and the zlib-like stream follows.
if offset+4 > len(data) {
return nil, fmt.Errorf("insufficient GPG data")
}
// The third byte should be a valid zlib CMF byte (deflate method = 8)
cmf := data[offset+2]
if cmf&0x0F != 8 {
return nil, fmt.Errorf("GPG: invalid deflate method in underlying zlib stream")
}
result.Description = fmt.Sprintf("%s, compressed data at offset %d", result.Description, offset)
return result, nil
}
// ---- PEM ----
const (
DescriptionPEMCertificate = "PEM certificate"
DescriptionPEMPublicKey = "PEM public key"
DescriptionPEMPrivateKey = "PEM private key"
)
// pemCertificateMagic returns magic bytes for PEM certificates.
func pemCertificateMagic() [][]byte {
return [][]byte{
[]byte("-----BEGIN CERTIFICATE-----"),
}
}
// pemPublicKeyMagic returns magic bytes for PEM public keys.
func pemPublicKeyMagic() [][]byte {
return [][]byte{
[]byte("-----BEGIN PUBLIC KEY-----"),
[]byte("-----BEGIN RSA PUBLIC KEY-----"),
[]byte("-----BEGIN DSA PUBLIC KEY-----"),
[]byte("-----BEGIN ECDSA PUBLIC KEY-----"),
}
}
// pemPrivateKeyMagic returns magic bytes for PEM private keys.
func pemPrivateKeyMagic() [][]byte {
return [][]byte{
[]byte("-----BEGIN PRIVATE KEY-----"),
[]byte("-----BEGIN EC PRIVATE KEY-----"),
[]byte("-----BEGIN RSA PRIVATE KEY-----"),
[]byte("-----BEGIN DSA PRIVATE KEY-----"),
[]byte("-----BEGIN OPENSSH PRIVATE KEY-----"),
[]byte("-----BEGIN ANY PRIVATE KEY-----"),
[]byte("-----BEGIN ENCRYPTED PRIVATE KEY-----"),
[]byte("-----BEGIN TSS2 PRIVATE KEY-----"),
}
}
func decodePEMData(pemData []byte) (int, error) {
const delim = "--"
pemStr := string(pemData)
delimCount := 0
var base64Str strings.Builder
for _, line := range strings.Split(pemStr, "\n") {
line = strings.TrimRight(line, "\r")
if strings.HasPrefix(line, delim) {
delimCount++
if delimCount == 2 {
break
}
continue
}
base64Str.WriteString(line)
}
if base64Str.Len() == 0 {
return 0, fmt.Errorf("no PEM base64 data found")
}
decoded, err := base64.StdEncoding.DecodeString(base64Str.String())
if err != nil {
return 0, fmt.Errorf("PEM base64 decode failed: %v", err)
}
return len(decoded), nil
}
// pemFindEnd locates the end delimiter and returns the total PEM size.
func pemFindEnd(data []byte, offset int) (int, bool) {
const endPrefix = "-----END "
pemData := string(data[offset:])
idx := strings.Index(pemData, endPrefix)
if idx < 0 {
return 0, false
}
// Find end of the delimiter line
rest := pemData[idx:]
nlIdx := strings.Index(rest, "\n")
if nlIdx < 0 {
return idx + len(rest), true
}
return idx + nlIdx + 1, true
}
// pemParser validates PEM certificate and key signatures.
func pemParser(data []byte, offset int) (*SignatureResult, error) {
const minPEMLen = 26
result := &SignatureResult{
Offset: offset,
Confidence: ConfidenceHigh,
}
if offset+minPEMLen > len(data) {
return nil, fmt.Errorf("insufficient PEM data")
}
pemMagic := data[offset : offset+minPEMLen]
pemMagicStr := string(pemMagic)
// Determine type based on first 26 bytes
matched := false
for _, m := range pemPublicKeyMagic() {
if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) {
result.Description = DescriptionPEMPublicKey
matched = true
break
}
}
if !matched {
for _, m := range pemPrivateKeyMagic() {
if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) {
result.Description = DescriptionPEMPrivateKey
matched = true
break
}
}
}
if !matched {
for _, m := range pemCertificateMagic() {
if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) {
result.Description = DescriptionPEMCertificate
matched = true
break
}
}
}
if !matched {
return nil, fmt.Errorf("PEM magic not recognised")
}
// Find the end of PEM data
pemSize, found := pemFindEnd(data, offset)
if !found || pemSize == 0 {
return nil, fmt.Errorf("PEM end delimiter not found")
}
// Validate base64 content
if offset+pemSize > len(data) {
pemSize = len(data) - offset
}
if _, err := decodePEMData(data[offset : offset+pemSize]); err != nil {
return nil, fmt.Errorf("PEM validation failed: %v", err)
}
if offset == 0 && pemSize == len(data) {
result.ExtractionDeclined = true
}
result.Size = pemSize
return result, nil
}
// ---- PKCS DER ----
const DescriptionDERHash = "PKCS DER hash"
func derHashLookups() map[string][]byte {
return map[string][]byte{
"MD5": {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10},
"SHA1": {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14},
"SHA256": {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20},
"SHA384": {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30},
"SHA512": {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04},
}
}
// derHashMagic returns magic bytes for all DER hash types.
func derHashMagic() [][]byte {
var magics [][]byte
for _, m := range derHashLookups() {
mc := make([]byte, len(m))
copy(mc, m)
magics = append(magics, mc)
}
return magics
}
// derHashParser validates PKCS DER hash signatures.
func derHashParser(data []byte, offset int) (*SignatureResult, error) {
result := &SignatureResult{
Offset: offset,
Description: DescriptionDERHash,
Confidence: ConfidenceMedium,
}
for name, magic := range derHashLookups() {
hashEnd := offset + len(magic)
if hashEnd > len(data) {
continue
}
match := true
for i, b := range magic {
if data[offset+i] != b {
match = false
break
}
}
if match {
result.Size = len(magic)
result.Description = fmt.Sprintf("%s, %s", result.Description, name)
return result, nil
}
}
return nil, fmt.Errorf("DER hash validation failed")
}
// ---- LUKS ----
const DescriptionLUKS = "LUKS header"
// luksMagic returns magic bytes for LUKS headers.
func luksMagic() [][]byte {
return [][]byte{
{0x4C, 0x55, 0x4B, 0x53, 0xBA, 0xBE}, // "LUKS\xBA\xBE"
}
}
// luksParser validates LUKS header signatures.
func luksParser(data []byte, offset int) (*SignatureResult, error) {
const cipherAlgoStart = 8
const cipherAlgoEnd = 40
const cipherModeStart = 40
const cipherModeEnd = 72
const hashFnStart = 72
const hashFnEnd = 104
const luks2MinHeaderSize = 4032
luksBaseFields := []StructField{
{"magic_1", "u32"},
{"magic_2", "u16"},
{"version", "u16"},
{"header_size", "u64"},
}
result := &SignatureResult{
Offset: offset,
Name: "luks",
Description: DescriptionLUKS,
Confidence: ConfidenceMedium,
}
if offset+StructSize(luksBaseFields) > len(data) {
return nil, fmt.Errorf("insufficient LUKS data")
}
luksBase, err := Parse(data[offset:], luksBaseFields, "big")
if err != nil {
return nil, fmt.Errorf("failed to parse LUKS header")
}
version := luksBase["version"]
// Hash function string
if offset+hashFnEnd > len(data) {
return nil, fmt.Errorf("LUKS hash function data out of bounds")
}
hashFn := GetCString(data[offset+hashFnStart : offset+hashFnEnd])
if hashFn == "" {
return nil, fmt.Errorf("LUKS hash function string empty")
}
if version == 1 {
if offset+cipherAlgoEnd > len(data) {
return nil, fmt.Errorf("LUKS cipher data out of bounds")
}
cipherAlgo := GetCString(data[offset+cipherAlgoStart : offset+cipherAlgoEnd])
cipherMode := GetCString(data[offset+cipherModeStart : offset+cipherModeEnd])
if cipherAlgo == "" || cipherMode == "" {
return nil, fmt.Errorf("LUKS v1 cipher strings empty")
}
result.Description = fmt.Sprintf("%s, version: %d, cipher algorithm: %s, cipher mode: %s, hash fn: %s",
result.Description, version, cipherAlgo, cipherMode, hashFn)
return result, nil
} else if version == 2 {
headerSize := luksBase["header_size"]
availableData := len(data) - offset
if headerSize <= luks2MinHeaderSize || headerSize >= availableData {
return nil, fmt.Errorf("LUKS v2 header size invalid")
}
result.Description = fmt.Sprintf("%s, version: %d, header size: %d bytes, hash fn: %s",
result.Description, version, headerSize, hashFn)
return result, nil
}
return nil, fmt.Errorf("unsupported LUKS version: %d", version)
}
// ---- Hash constants ----
const hashMagicLen = 16
const (
DescriptionCRC32 = "CRC32 polynomial table"
DescriptionSHA256 = "SHA256 hash constants"
DescriptionMD5 = "MD5 hash constants"
)
// crc32Magic returns magic bytes for CRC32 polynomial tables.
// Order matters: index 0 is big endian, index 1 is little endian.
func crc32ConstMagic() [][]byte {
return [][]byte{
// Big endian
{0x00, 0x00, 0x00, 0x00, 0x77, 0x07, 0x30, 0x96, 0xEE, 0x0E, 0x61, 0x2C, 0x99, 0x09, 0x51, 0xBA},
// Little endian
{0x00, 0x00, 0x00, 0x00, 0x96, 0x30, 0x07, 0x77, 0x2C, 0x61, 0x0E, 0xEE, 0xBA, 0x51, 0x09, 0x99},
}
}
// sha256Magic returns magic bytes for SHA256 hash constants.
func sha256ConstMagic() [][]byte {
return [][]byte{
// Big endian
{0x42, 0x8a, 0x2f, 0x98, 0x71, 0x37, 0x44, 0x91, 0xb5, 0xc0, 0xfb, 0xcf, 0xe9, 0xb5, 0xdb, 0xa5},
// Little endian
{0x98, 0x2f, 0x8a, 0x42, 0x91, 0x44, 0x37, 0x71, 0xcf, 0xfb, 0xc0, 0xb5, 0xa5, 0xdb, 0xb5, 0xe9},
}
}
// md5Magic returns magic bytes for MD5 hash constants.
func md5ConstMagic() [][]byte {
return [][]byte{
// Big endian
{0xd7, 0x6a, 0xa4, 0x78, 0xe8, 0xc7, 0xb7, 0x56, 0x24, 0x20, 0x70, 0xdb, 0xc1, 0xbd, 0xce, 0xee},
// Little endian
{0x78, 0xa4, 0x6a, 0xd7, 0x56, 0xb7, 0xc7, 0xe8, 0xdb, 0x70, 0x20, 0x24, 0xee, 0xce, 0xbd, 0xc1},
}
}
func hashEndianness(data []byte, offset int, magics [][]byte) string {
if offset+hashMagicLen > len(data) {
return "little"
}
thisMagic := data[offset : offset+hashMagicLen]
if len(magics) > 0 && len(magics[0]) == hashMagicLen {
match := true
for i := 0; i < hashMagicLen; i++ {
if thisMagic[i] != magics[0][i] {
match = false
break
}
}
if match {
return "big"
}
}
return "little"
}
// crc32Parser validates CRC32 polynomial table signatures.
func crc32Parser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Size: hashMagicLen,
Description: fmt.Sprintf("%s, %s endian", DescriptionCRC32, hashEndianness(data, offset, crc32ConstMagic())),
}, nil
}
// sha256Parser validates SHA256 hash constant signatures.
func sha256Parser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Size: hashMagicLen,
Description: fmt.Sprintf("%s, %s endian", DescriptionSHA256, hashEndianness(data, offset, sha256ConstMagic())),
}, nil
}
// md5Parser validates MD5 hash constant signatures.
func md5Parser(data []byte, offset int) (*SignatureResult, error) {
return &SignatureResult{
Offset: offset,
Size: hashMagicLen,
Description: fmt.Sprintf("%s, %s endian", DescriptionMD5, hashEndianness(data, offset, md5ConstMagic())),
}, nil
}
internal/leviathan/binsnoop/sig_executables.go
+485
View File
@@ -0,0 +1,485 @@
package binsnoop
import (
"fmt"
)
// ---------------------------------------------------------------------------
// ELF
// ---------------------------------------------------------------------------
const ELFDescription = "ELF binary"
func ELFMagic() [][]byte {
return [][]byte{
{0x7F, 0x45, 0x4C, 0x46}, // "\x7FELF"
}
}
func ELFParser(data []byte, offset int) (*SignatureResult, error) {
const elfIdentSize = 16
const elfInfoStructSize = 8
const expectedVersion = 1
elfIdentFields := []StructField{
{"magic", "u32"},
{"class", "u8"},
{"endianness", "u8"},
{"version", "u8"},
{"osabi", "u8"},
{"abiversion", "u8"},
{"padding_1", "u32"},
{"padding_2", "u24"},
}
elfInfoFields := []StructField{
{"type", "u16"},
{"machine", "u16"},
{"version", "u32"},
}
elfClasses := map[int]int{
1: 32,
2: 64,
}
elfEndianness := map[int]string{
1: "little",
2: "big",
}
elfOSABI := map[int]string{
0: "System-V (Unix)",
1: "HP-UX",
2: "NetBSD",
3: "Linux",
4: "GNU Hurd",
5: "86Open",
6: "Solaris",
7: "AIX",
8: "IRIX",
9: "FreeBSD",
10: "Tru64",
11: "Novell Modesto",
12: "OpenBSD",
13: "OpenVMS",
14: "NonStop Kernel",
15: "AROS",
16: "FenixOS",
17: "Nuxi CloudABI",
18: "OpenVOS",
97: "ARM ABI",
102: "Cell LV2",
202: "Cafe OS",
255: "embedded",
}
elfTypes := map[int]string{
0: "no file type",
1: "relocatable",
2: "executable",
3: "shared object",
4: "core file",
}
elfMachines := map[int]string{
0: "no machine",
1: "AT&T WE 32100",
2: "SPARC",
3: "x86",
4: "Motorola 68k",
5: "Motorola 88k",
6: "Intel MCU",
7: "Intel 80860",
8: "MIPS",
9: "IBM System/370",
10: "MIPS RS3000",
11: "RS6000",
15: "HP PA-RISC",
16: "nCUBE",
17: "Fujitsu VPP500",
18: "SPARC32PLUS",
19: "Intel 80960",
20: "PowerPC",
21: "PowerPC 64-bit",
22: "S390",
23: "IBM SPU/SPC",
24: "cisco SVIP",
25: "cisco 7200",
36: "NEC V800",
37: "Fujitsu FR20",
38: "TRW RH-32",
39: "Motorola RCE",
40: "ARM",
41: "Digital Alpha",
42: "SuperH",
43: "SPARCv9",
44: "Siemens TriCore embedded processor",
45: "Argonaut RISC Core",
46: "Hitachi H8/300",
47: "Hitachi H8/300H",
48: "Hitachi H8S",
49: "Hitachi H8/500",
50: "IA-64",
51: "Stanford MIPS-X",
52: "Motorola ColdFire",
53: "Motorola M68HC12",
54: "Fujitsu MMA Multimedia Accelerator",
55: "Siemens PCP",
56: "Sony nCPU embedded RISC processor",
57: "Denso NDR1 microprocessor",
58: "Motorola StarCore",
59: "Toyota ME16",
60: "STMicroelectronics ST100",
61: "Advanced Logic TinyJ embedded processor",
62: "AMD X86-64",
63: "Sony DSP processor",
64: "PDP-10",
65: "PDP-11",
66: "Siemens FX66",
67: "STMicroelectronics ST9+",
68: "STMicroelectronics ST7",
69: "Motorola MC68HC16",
70: "Motorola MC68HC11",
71: "Motorola MC68HC08",
72: "Motorola MC68HC05",
73: "Silicon Graphics SVx",
74: "STMicroelectonrics ST19",
75: "Digital VAX",
76: "Axis Communications 32-bit CPU",
77: "Infineon Technologies 32-bit CPU",
78: "Element 14 64-bit DSP",
79: "LSI Logic 16-bit DSP",
80: "MMIX",
81: "Harvard machine-independent",
82: "SiTera Prism",
83: "Atmel AVR 8-bit",
84: "Fujitsu FR30",
85: "Mitsubishi D10V",
86: "Mitsubishi D30V",
87: "NEC v850",
88: "Renesas M32R",
89: "Matsushita MN10300",
90: "Matsushita MN10200",
91: "picoJava",
92: "OpenRISC",
93: "Synopsys ARCompact ARC700 cores",
94: "Tensilica Xtensa",
95: "Alphamosaic VideoCore",
96: "Thompson Multimedia",
97: "NatSemi 32k",
98: "Tenor Network TPC",
99: "Trebia SNP 1000",
100: "STMicroelectronics ST200",
101: "Ubicom IP2022",
102: "MAX Processor",
103: "NatSemi CompactRISC",
104: "Fujitsu F2MC16",
105: "TI msp430",
106: "Analog Devices Blackfin",
107: "S1C33 Family of Seiko Epson",
108: "Sharp embedded",
109: "Arca RISC",
110: "PKU-Unity Ltd.",
111: "eXcess: 16/32/64-bit",
112: "Icera Deep Execution Processor",
113: "Altera Nios II",
114: "NatSemi CRX",
115: "Motorola XGATE",
116: "Infineon C16x/XC16x",
117: "Renesas M16C series",
118: "Microchip dsPIC30F",
119: "Freescale RISC core",
120: "Renesas M32C series",
131: "Altium TSK3000 core",
132: "Freescale RS08",
134: "Cyan Technology eCOG2",
135: "Sunplus S+core7 RISC",
136: "New Japan Radio (NJR) 24-bit DSP",
137: "Broadcom VideoCore III processor",
138: "LatticeMico32",
139: "Seiko Epson C17 family",
140: "TMS320C6000",
141: "TMS320C2000",
142: "TMS320C55x",
144: "TI Programmable Realtime Unit",
160: "STMicroelectronics 64bit VLIW DSP",
161: "Cypress M8C",
162: "Renesas R32C series",
163: "NXP TriMedia family",
164: "Qualcomm DSP6",
165: "Intel 8051 and variants",
166: "STMicroelectronics STxP7x family",
167: "Andes embedded RISC",
168: "Cyan eCOG1X family",
169: "Dallas MAXQ30",
170: "New Japan Radio (NJR) 16-bit DSP",
171: "M2000 Reconfigurable RISC",
172: "Cray NV2 vector architecture",
173: "Renesas RX family",
174: "META",
175: "MCST Elbrus e2k",
176: "Cyan Technology eCOG16 family",
177: "NatSemi CompactRISC",
178: "Freescale Extended Time Processing Unit",
179: "Infineon SLE9X",
180: "Intel L1OM",
181: "Intel K1OM",
183: "ARM 64-bit",
185: "Atmel 32-bit family",
186: "STMicroeletronics STM8 8-bit",
187: "Tilera TILE64",
188: "Tilera TILEPro",
189: "Xilinx MicroBlaze 32-bit RISC",
190: "NVIDIA CUDA architecture",
191: "Tilera TILE-Gx",
195: "Synopsys ARCv2/HS3x/HS4x cores",
197: "Renesas RL78 family",
199: "Renesas 78K0R",
200: "Freescale 56800EX",
201: "Beyond BA1",
202: "Beyond BA2",
203: "XMOS xCORE",
204: "Microchip 8-bit PIC(r)",
210: "KM211 KM32",
211: "KM211 KMX32",
212: "KM211 KMX16",
213: "KM211 KMX8",
214: "KM211 KVARC",
215: "Paneve CDP",
216: "Cognitive Smart Memory",
217: "iCelero CoolEngine",
218: "Nanoradio Optimized RISC",
219: "CSR Kalimba architecture family",
220: "Zilog Z80",
221: "Controls and Data Services VISIUMcore processor",
222: "FTDI Chip FT32 high performance 32-bit RISC architecture",
223: "Moxie processor family",
224: "AMD GPU architecture",
243: "RISC-V",
244: "Lanai 32-bit processor",
245: "CEVA Processor Architecture Family",
246: "CEVA X2 Processor Family",
247: "Berkeley Packet Filter",
248: "Graphcore Intelligent Processing Unit",
249: "Imagination Technologies",
250: "Netronome Flow Processor",
251: "NEC Vector Engine",
252: "C-SKY processor family",
253: "Synopsys ARCv3 64-bit ISA/HS6x cores",
254: "MOS Technology MCS 6502 processor",
255: "Synopsys ARCv3 32-bit",
256: "Kalray VLIW core of the MPPA family",
257: "WDC 65C816",
258: "LoongArch",
259: "ChipON KungFu32",
}
elfData := data[offset:]
if len(elfData) < elfIdentSize+elfInfoStructSize {
return nil, fmt.Errorf("ELF: not enough data")
}
// Parse the e_ident (endianness doesn't matter for single-byte fields)
eIdent, err := Parse(elfData, elfIdentFields, "little")
if err != nil {
return nil, fmt.Errorf("ELF: e_ident parse error: %w", err)
}
// Sanity check padding fields
if eIdent["padding_1"] != 0 || eIdent["padding_2"] != 0 {
return nil, fmt.Errorf("ELF: non-zero padding in e_ident")
}
if eIdent["version"] != expectedVersion {
return nil, fmt.Errorf("ELF: unexpected e_ident version %d", eIdent["version"])
}
classVal, classOK := elfClasses[eIdent["class"]]
if !classOK {
return nil, fmt.Errorf("ELF: unknown class %d", eIdent["class"])
}
endianStr, endianOK := elfEndianness[eIdent["endianness"]]
if !endianOK {
return nil, fmt.Errorf("ELF: unknown endianness %d", eIdent["endianness"])
}
osabiStr, osabiOK := elfOSABI[eIdent["osabi"]]
if !osabiOK {
return nil, fmt.Errorf("ELF: unknown OS/ABI %d", eIdent["osabi"])
}
// Parse the rest of the ELF header with the correct endianness
elfInfoData := elfData[elfIdentSize : elfIdentSize+elfInfoStructSize]
elfInfo, err := Parse(elfInfoData, elfInfoFields, endianStr)
if err != nil {
return nil, fmt.Errorf("ELF: info parse error: %w", err)
}
if elfInfo["version"] != expectedVersion {
return nil, fmt.Errorf("ELF: unexpected info version %d", elfInfo["version"])
}
typeStr, typeOK := elfTypes[elfInfo["type"]]
if !typeOK {
return nil, fmt.Errorf("ELF: unknown type %d", elfInfo["type"])
}
machineStr := "Unknown"
if m, ok := elfMachines[elfInfo["machine"]]; ok {
machineStr = m
}
return &SignatureResult{
Offset: offset,
Name: "elf",
Description: fmt.Sprintf("%s, %d-bit %s, %s for %s, %s endian", ELFDescription, classVal, typeStr, machineStr, osabiStr, endianStr),
Confidence: ConfidenceMedium,
}, nil
}
// ---------------------------------------------------------------------------
// PE
// ---------------------------------------------------------------------------
const PEDescription = "Windows PE binary"
func PEMagic() [][]byte {
// First 16 bytes of common DOS headers (e_magic through e_ss)
return [][]byte{
{0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00},
{0x4D, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00},
}
}
func PEParser(data []byte, offset int) (*SignatureResult, error) {
const peMagicValue = 0x00004550 // "PE\x00\x00"
dosFields := []StructField{
{"e_magic", "u16"},
{"e_cblp", "u16"},
{"e_cp", "u16"},
{"e_crlc", "u16"},
{"e_cparhdr", "u16"},
{"e_minalloc", "u16"},
{"e_maxalloc", "u16"},
{"e_ss", "u16"},
{"e_sp", "u16"},
{"e_csum", "u16"},
{"e_ip", "u16"},
{"e_cs", "u16"},
{"e_lfarlc", "u16"},
{"e_ovno", "u16"},
{"e_res_1", "u16"},
{"e_res_2", "u16"},
{"e_res_3", "u16"},
{"e_res_4", "u16"},
{"e_oemid", "u16"},
{"e_oeminfo", "u16"},
{"e_res_5", "u16"},
{"e_res_6", "u16"},
{"e_res_7", "u16"},
{"e_res_8", "u16"},
{"e_res_9", "u16"},
{"e_res_10", "u16"},
{"e_res_11", "u16"},
{"e_res_12", "u16"},
{"e_res_13", "u16"},
{"e_res_14", "u16"},
{"e_lfanew", "u32"},
}
peFields := []StructField{
{"magic", "u32"},
{"machine", "u16"},
{"number_of_sections", "u16"},
{"timestamp", "u32"},
{"symbol_table_ptr", "u32"},
{"number_of_symbols", "u32"},
{"optional_header_size", "u16"},
{"characteristics", "u16"},
}
knownMachineTypes := map[int]string{
0x0: "Unknown",
0x184: "Alpha32",
0x284: "Alpha64",
0x1D3: "Matsushita AM33",
0x8664: "Intel x86-64",
0x1C0: "ARM",
0xAA64: "ARM-64",
0x1C4: "ARM Thumb2",
0xEBC: "EFI",
0x14C: "Intel x86",
0x200: "Intel Itanium",
0x6232: "LoongArch 32-bit",
0x6264: "LoongArch 64-bit",
0x9041: "Mitsubishi M32R",
0x266: "MIPS16",
0x366: "MIPS with FPU",
0x466: "MIPS16 with FPU",
0x1F0: "PowerPC",
0x1F1: "PowerPC with FPU",
0x5032: "RISC-V 32-bit",
0x5064: "RISC-V 64-bit",
0x5128: "RISC-V 128-bit",
0x1A2: "Hitachi SH3",
0x1A3: "Hitachi SH3 DSP",
0x1A6: "Hitachi SH4",
0x1A8: "Hitachi SH5",
0x1C2: "Thumb",
0x169: "MIPS WCEv2",
}
peData := data[offset:]
dosStructSize := StructSize(dosFields)
peStructSize := StructSize(peFields)
if len(peData) < dosStructSize {
return nil, fmt.Errorf("PE: not enough data for DOS header")
}
dosHeader, err := Parse(peData, dosFields, "little")
if err != nil {
return nil, fmt.Errorf("PE: DOS header parse error: %w", err)
}
// Sanity check the reserved header fields; they should all be 0
for _, field := range []string{
"e_res_1", "e_res_2", "e_res_3", "e_res_4",
"e_res_5", "e_res_6", "e_res_7", "e_res_8",
"e_res_9", "e_res_10", "e_res_11", "e_res_12",
"e_res_13", "e_res_14",
} {
if dosHeader[field] != 0 {
return nil, fmt.Errorf("PE: reserved field %s is non-zero", field)
}
}
peHeaderStart := dosHeader["e_lfanew"]
peHeaderEnd := peHeaderStart + peStructSize
if peHeaderEnd > len(peData) {
return nil, fmt.Errorf("PE: e_lfanew points beyond available data")
}
peHeader, err := Parse(peData[peHeaderStart:peHeaderEnd], peFields, "little")
if err != nil {
return nil, fmt.Errorf("PE: PE header parse error: %w", err)
}
if peHeader["magic"] != peMagicValue {
return nil, fmt.Errorf("PE: invalid PE magic 0x%08X", peHeader["magic"])
}
machineStr, machineOK := knownMachineTypes[peHeader["machine"]]
if !machineOK {
return nil, fmt.Errorf("PE: unknown machine type 0x%X", peHeader["machine"])
}
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, machine type: %s", PEDescription, machineStr),
Confidence: ConfidenceMedium,
}, nil
}
internal/leviathan/binsnoop/sig_filesystems.go
+1289
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/binsnoop/sig_firmware.go
+1333
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/binsnoop/sig_images.go
+677
View File
@@ -0,0 +1,677 @@
package binsnoop
import (
"bytes"
"fmt"
"strings"
)
// ---------------------------------------------------------------------------
// PNG
// ---------------------------------------------------------------------------
const PNGDescription = "PNG image"
func PNGMagic() [][]byte {
// PNG magic followed by IHDR chunk size (0x0000000D) and IHDR chunk type
return [][]byte{
{0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 0x00, 0x00, 0x00, 0x0D, 0x49, 0x48, 0x44, 0x52},
}
}
func PNGParser(data []byte, offset int) (*SignatureResult, error) {
const pngHeaderLen = 8
const crcSize = 4
const iendChunkType = 0x49454E44
chunkFields := []StructField{
{"length", "u32"},
{"type", "u32"},
}
chunkHeaderSize := StructSize(chunkFields)
if offset+pngHeaderLen >= len(data) {
return nil, fmt.Errorf("PNG: not enough data for header")
}
// Walk chunks starting after the 8-byte PNG header
chunkData := data[offset+pngHeaderLen:]
available := len(chunkData)
chunkOffset := 0
var previousOffset *int
for IsOffsetSafe(available, chunkOffset, previousOffset) {
if chunkOffset+chunkHeaderSize > available {
break
}
hdr, err := Parse(chunkData[chunkOffset:], chunkFields, "big")
if err != nil {
break
}
isLast := hdr["type"] == iendChunkType
totalChunkSize := chunkHeaderSize + hdr["length"] + crcSize
prev := chunkOffset
previousOffset = &prev
chunkOffset += totalChunkSize
if isLast {
pngSize := pngHeaderLen + chunkOffset
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, total size: %d bytes", PNGDescription, pngSize),
Confidence: ConfidenceHigh,
Size: pngSize,
}, nil
}
}
return nil, fmt.Errorf("PNG: failed to walk chunks to IEND")
}
// ---------------------------------------------------------------------------
// JPEG
// ---------------------------------------------------------------------------
const JPEGDescription = "JPEG image"
func JPEGMagic() [][]byte {
return [][]byte{
{0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00}, // JFIF
{0xFF, 0xD8, 0xFF, 0xE1}, // EXIF
{0xFF, 0xD8, 0xFF, 0xDB}, // DQT
}
}
func JPEGParser(data []byte, offset int) (*SignatureResult, error) {
const markerMagic byte = 0xFF
const sosMarker byte = 0xDA
const eofMarker byte = 0xD9
noLengthMarkers := []byte{
0x00, 0x01, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, eofMarker,
}
sosSkipMarkers := []byte{
0x00, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7,
}
jpegData := data[offset:]
nextMarkerOffset := 0
for {
if nextMarkerOffset >= len(jpegData) {
break
}
magic := jpegData[nextMarkerOffset]
if magic != markerMagic {
break
}
nextMarkerOffset++
if nextMarkerOffset >= len(jpegData) {
break
}
markerID := jpegData[nextMarkerOffset]
nextMarkerOffset++
// If this marker has a length field
if !byteInSlice(markerID, noLengthMarkers) {
if nextMarkerOffset+2 > len(jpegData) {
break
}
segLen := int(jpegData[nextMarkerOffset])<<8 | int(jpegData[nextMarkerOffset+1])
nextMarkerOffset += segLen
}
// SOS marker: scan until next valid marker
if markerID == sosMarker {
for {
if nextMarkerOffset+2 > len(jpegData) {
break
}
if jpegData[nextMarkerOffset] == markerMagic && !byteInSlice(jpegData[nextMarkerOffset+1], sosSkipMarkers) {
break
}
nextMarkerOffset++
}
}
// EOF marker
if markerID == eofMarker {
jpegSize := nextMarkerOffset
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, total size: %d bytes", JPEGDescription, jpegSize),
Confidence: ConfidenceMedium,
Size: jpegSize,
}, nil
}
}
return nil, fmt.Errorf("JPEG: failed to find EOI marker")
}
// ---------------------------------------------------------------------------
// BMP
// ---------------------------------------------------------------------------
const BMPDescription = "BMP image (Bitmap)"
func BMPMagic() [][]byte {
return [][]byte{
{0x42, 0x4D}, // "BM"
}
}
func BMPParser(data []byte, offset int) (*SignatureResult, error) {
const bmpFileHeaderSize = 14
bmpHeaderFields := []StructField{
{"bfType", "u16"},
{"bfSize", "u32"},
{"bfReserved1", "u16"},
{"bfReserved2", "u16"},
{"bfOffBits", "u32"},
}
validDIBHeaderSizes := []int{12, 40, 108, 124}
bmpData := data[offset:]
if len(bmpData) < StructSize(bmpHeaderFields) {
return nil, fmt.Errorf("BMP: not enough data for header")
}
hdr, err := Parse(bmpData, bmpHeaderFields, "little")
if err != nil {
return nil, fmt.Errorf("BMP: parse error: %w", err)
}
bmpDataSize := len(bmpData)
// The BMP file size cannot be bigger than available data
if bmpDataSize < hdr["bfSize"] {
return nil, fmt.Errorf("BMP: reported size %d exceeds available data %d", hdr["bfSize"], bmpDataSize)
}
if hdr["bfSize"] == 0 {
return nil, fmt.Errorf("BMP: file size is 0")
}
if hdr["bfOffBits"] == 0 {
return nil, fmt.Errorf("BMP: bitmap bits offset is 0")
}
if hdr["bfOffBits"] > bmpDataSize {
return nil, fmt.Errorf("BMP: bitmap bits offset exceeds available data")
}
// Validate DIB header size
if offset+bmpFileHeaderSize+4 > len(data) {
return nil, fmt.Errorf("BMP: not enough data for DIB header size")
}
dibHeaderSize := int(LE32(data, offset+bmpFileHeaderSize))
validDIB := false
for _, vs := range validDIBHeaderSizes {
if dibHeaderSize == vs {
validDIB = true
break
}
}
if !validDIB {
return nil, fmt.Errorf("BMP: invalid DIB header size %d", dibHeaderSize)
}
// The offset that points to the image data cannot point into the second header
if hdr["bfOffBits"] < bmpFileHeaderSize+dibHeaderSize {
return nil, fmt.Errorf("BMP: bitmap bits offset points into header")
}
totalSize := hdr["bfSize"]
return &SignatureResult{
Offset: offset,
Name: "bmp",
Description: fmt.Sprintf("BMP image, total size: %d", totalSize),
Confidence: ConfidenceMedium,
Size: totalSize,
}, nil
}
// ---------------------------------------------------------------------------
// GIF
// ---------------------------------------------------------------------------
const GIFDescription = "GIF image"
func GIFMagic() [][]byte {
return [][]byte{
{0x47, 0x49, 0x46, 0x38, 0x37, 0x61}, // GIF87a
{0x47, 0x49, 0x46, 0x38, 0x39, 0x61}, // GIF89a
}
}
func GIFParser(data []byte, offset int) (*SignatureResult, error) {
gifHeaderFields := []StructField{
{"magic_p1", "u24"},
{"magic_p2", "u24"},
{"image_width", "u16"},
{"image_height", "u16"},
{"flags", "u8"},
{"bg_color_index", "u8"},
{"aspect_ratio", "u8"},
}
gifData := data[offset:]
if len(gifData) < StructSize(gifHeaderFields) {
return nil, fmt.Errorf("GIF: not enough data for header")
}
hdr, err := Parse(gifData, gifHeaderFields, "little")
if err != nil {
return nil, fmt.Errorf("GIF: parse error: %w", err)
}
// Calculate header size including optional global color table
headerSize := StructSize(gifHeaderFields) + gifColorTableSize(hdr["flags"])
if offset+headerSize > len(data) {
return nil, fmt.Errorf("GIF: not enough data for header + color table")
}
// Walk GIF blocks to find terminator
blockData := data[offset+headerSize:]
gifDataSize, err := getGIFDataSize(blockData)
if err != nil {
return nil, fmt.Errorf("GIF: %w", err)
}
totalSize := headerSize + gifDataSize
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, %dx%d pixels, total size: %d bytes", GIFDescription, hdr["image_width"], hdr["image_height"], totalSize),
Confidence: ConfidenceHigh,
Size: totalSize,
}, nil
}
func gifColorTableSize(flags int) int {
const hasColorTable = 0x80
const colorTableSizeMask = 0x07
if flags&hasColorTable == 0 {
return 0
}
encodedSize := (flags & colorTableSizeMask) + 1
size := 3
for i := 0; i < encodedSize; i++ {
size *= 2
}
return size
}
func getGIFDataSize(gifData []byte) (int, error) {
const extensionBlock byte = 0x21
const terminatorBlock byte = 0x3B
const imageDescriptorBlock byte = 0x2C
nextOffset := 0
var previousOffset *int
available := len(gifData)
for IsOffsetSafe(available, nextOffset, previousOffset) {
if nextOffset >= available {
break
}
blockType := gifData[nextOffset]
var blockSize int
var err error
switch blockType {
case imageDescriptorBlock:
blockSize, err = parseGIFImageDescriptor(gifData[nextOffset:])
case extensionBlock:
blockSize, err = parseGIFExtension(gifData[nextOffset:])
case terminatorBlock:
return nextOffset + 1, nil
default:
return 0, fmt.Errorf("unknown GIF block type 0x%02X at offset %d", blockType, nextOffset)
}
if err != nil {
return 0, err
}
prev := nextOffset
previousOffset = &prev
nextOffset += blockSize
}
return 0, fmt.Errorf("failed to find GIF terminator block")
}
func parseGIFImageDescriptor(gifData []byte) (int, error) {
const lzwCodeSize = 1
imgDescFields := []StructField{
{"magic", "u8"},
{"image_left", "u16"},
{"image_top", "u16"},
{"image_width", "u16"},
{"image_height", "u16"},
{"flags", "u8"},
}
if len(gifData) < StructSize(imgDescFields) {
return 0, fmt.Errorf("GIF image descriptor: not enough data")
}
hdr, err := Parse(gifData, imgDescFields, "little")
if err != nil {
return 0, err
}
totalSize := StructSize(imgDescFields) + gifColorTableSize(hdr["flags"]) + lzwCodeSize
if totalSize > len(gifData) {
return 0, fmt.Errorf("GIF image descriptor: not enough data for sub-blocks")
}
subBlocksSize, err := parseGIFSubBlocks(gifData[totalSize:])
if err != nil {
return 0, err
}
totalSize += subBlocksSize
return totalSize, nil
}
func parseGIFExtension(extensionData []byte) (int, error) {
const plainText = 1
const application = 0xFF
const headerSize = 2
extFields := []StructField{
{"magic", "u8"},
{"extension_type", "u8"},
{"sub_block_offset", "u8"},
}
if len(extensionData) < StructSize(extFields) {
return 0, fmt.Errorf("GIF extension: not enough data")
}
hdr, err := Parse(extensionData, extFields, "little")
if err != nil {
return 0, err
}
subBlocksOffset := headerSize
extType := hdr["extension_type"]
if extType == application || extType == plainText {
subBlocksOffset += hdr["sub_block_offset"] + 1
}
if subBlocksOffset > len(extensionData) {
return 0, fmt.Errorf("GIF extension: sub-blocks offset exceeds data")
}
subBlocksSize, err := parseGIFSubBlocks(extensionData[subBlocksOffset:])
if err != nil {
return 0, err
}
return subBlocksOffset + subBlocksSize, nil
}
func parseGIFSubBlocks(subBlockData []byte) (int, error) {
const subBlockTerminator byte = 0
available := len(subBlockData)
nextOffset := 0
var previousOffset *int
for IsOffsetSafe(available, nextOffset, previousOffset) {
if nextOffset >= available {
break
}
subBlockSize := subBlockData[nextOffset]
if subBlockSize == subBlockTerminator {
return nextOffset + 1, nil
}
prev := nextOffset
previousOffset = &prev
nextOffset += int(subBlockSize) + 1
}
return 0, fmt.Errorf("GIF sub-blocks: terminator not found")
}
// ---------------------------------------------------------------------------
// SVG
// ---------------------------------------------------------------------------
const SVGDescription = "SVG image"
const svgOpenTag = "<svg "
const svgCloseTag = "</svg>"
const svgHeadMagic = `xmlns="http://www.w3.org/2000/svg"`
func SVGMagic() [][]byte {
return [][]byte{
{0x3C, 0x73, 0x76, 0x67, 0x20}, // "<svg "
}
}
func SVGParser(data []byte, offset int) (*SignatureResult, error) {
// Operate directly on the byte slice. A previous implementation did
// string(data[offset:]) — copying the entire remaining file — on every
// call; since the parser runs once per "<svg " match, a buffer dense with
// "<svg " tokens made this O(n^2) in copies and could hang indefinitely.
svgData := data[offset:]
openTag := []byte(svgOpenTag)
closeTag := []byte(svgCloseTag)
headMagic := []byte(svgHeadMagic)
// The matched "<svg " must be a real SVG root: its opening tag has to
// terminate with '>' and declare the SVG xmlns (which the loop below
// requires anyway via headTagCount). Verifying this up front, with a cap
// on how far we look for the closing '>', keeps the parser O(1) per call
// regardless of file size — so a buffer densely packed with "<svg " tokens
// can't turn scanning into an O(n^2) hang.
const maxRootTagLen = 64 * 1024
rootLimit := len(svgData)
if rootLimit > maxRootTagLen {
rootLimit = maxRootTagLen
}
rootEnd := bytes.IndexByte(svgData[:rootLimit], '>')
if rootEnd < 0 {
return nil, fmt.Errorf("SVG: root tag not terminated")
}
if !bytes.Contains(svgData[:rootEnd+1], headMagic) {
return nil, fmt.Errorf("SVG: root tag missing SVG namespace")
}
headTagCount := 0
unclosedSVGTags := 0
// Search for all <svg and </svg> occurrences
searchIdx := 0
for searchIdx < len(svgData) {
// Find next occurrence of either tag
openIdx := bytes.Index(svgData[searchIdx:], openTag)
closeIdx := bytes.Index(svgData[searchIdx:], closeTag)
var tagStart int
if openIdx == -1 && closeIdx == -1 {
break
} else if openIdx == -1 {
tagStart = searchIdx + closeIdx
} else if closeIdx == -1 {
tagStart = searchIdx + openIdx
} else if openIdx <= closeIdx {
tagStart = searchIdx + openIdx
} else {
tagStart = searchIdx + closeIdx
}
tagBytes := svgData[tagStart:]
// Find the end '>' of this tag
endIdx := bytes.IndexByte(tagBytes, '>')
if endIdx == -1 {
break
}
fullTag := tagBytes[:endIdx+1]
isOpen := bytes.HasPrefix(fullTag, openTag)
isClose := bytes.HasPrefix(fullTag, closeTag)
isHead := bytes.Contains(fullTag, headMagic)
if isHead {
headTagCount++
}
if isOpen {
unclosedSVGTags++
}
if isClose {
unclosedSVGTags--
}
// More than one head tag is invalid
if headTagCount > 1 {
break
}
// If one head tag and all tags closed, that's EOF
if headTagCount == 1 && unclosedSVGTags == 0 {
totalSize := tagStart + len(svgCloseTag)
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, total size: %d bytes", SVGDescription, totalSize),
Confidence: ConfidenceMedium,
Size: totalSize,
}, nil
}
searchIdx = tagStart + 1
}
return nil, fmt.Errorf("SVG: failed to find matching closing tag")
}
// ---------------------------------------------------------------------------
// RIFF
// ---------------------------------------------------------------------------
const RIFFDescription = "RIFF image"
func RIFFMagic() [][]byte {
return [][]byte{
{0x52, 0x49, 0x46, 0x46}, // "RIFF"
}
}
func RIFFParser(data []byte, offset int) (*SignatureResult, error) {
const riffMagicValue = 0x46464952 // "RIFF" in LE
const chunkTypeStart = 8
const chunkTypeEnd = 12
const fileSizeOffset = 8
riffFields := []StructField{
{"magic", "u32"},
{"file_size", "u32"},
{"chunk_type", "u32"},
}
riffData := data[offset:]
if len(riffData) < StructSize(riffFields) {
return nil, fmt.Errorf("RIFF: not enough data for header")
}
hdr, err := Parse(riffData, riffFields, "little")
if err != nil {
return nil, fmt.Errorf("RIFF: parse error: %w", err)
}
if hdr["magic"] != riffMagicValue {
return nil, fmt.Errorf("RIFF: invalid magic")
}
if chunkTypeEnd > len(riffData) {
return nil, fmt.Errorf("RIFF: not enough data for chunk type string")
}
// Get the RIFF type string (e.g., "WAVE", "AVI ")
typeBytes := riffData[chunkTypeStart:chunkTypeEnd]
typeString := strings.TrimSpace(string(typeBytes))
// Validate that the type string is valid UTF-8 / printable
for _, b := range typeBytes {
if b < 0x20 || b > 0x7E {
return nil, fmt.Errorf("RIFF: non-printable chunk type")
}
}
totalSize := hdr["file_size"] + fileSizeOffset
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, encoding type: %s, total size: %d bytes", RIFFDescription, typeString, totalSize),
Confidence: ConfidenceMedium,
Size: totalSize,
}, nil
}
// ---------------------------------------------------------------------------
// PDF
// ---------------------------------------------------------------------------
const PDFDescription = "PDF document"
func PDFMagic() [][]byte {
return [][]byte{
{0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E}, // "%PDF-1."
}
}
func PDFParser(data []byte, offset int) (*SignatureResult, error) {
const minPDFSize = 16
const newlineOffset = 8
const minorNumberOffset = 7
const asciiZero byte = 0x30
const asciiNine byte = 0x39
const asciiNewline byte = 0x0A
const asciiPercent byte = 0x25
const asciiCarriageReturn byte = 0x0D
if offset+minPDFSize > len(data) {
return nil, fmt.Errorf("PDF: not enough data for header")
}
pdfHeader := data[offset : offset+minPDFSize]
// Get the minor version number
versionMinor := pdfHeader[minorNumberOffset]
if versionMinor < asciiZero || versionMinor > asciiNine {
return nil, fmt.Errorf("PDF: invalid minor version number")
}
// After the version line, expect newline(s) followed by '%'
for _, b := range pdfHeader[newlineOffset:] {
if b == asciiNewline || b == asciiCarriageReturn {
continue
} else if b == asciiPercent {
return &SignatureResult{
Offset: offset,
Description: fmt.Sprintf("%s, version 1.%d", PDFDescription, versionMinor-asciiZero),
Confidence: ConfidenceLow,
Size: 0,
}, nil
} else {
break
}
}
return nil, fmt.Errorf("PDF: invalid header format")
}
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
func byteInSlice(b byte, s []byte) bool {
return bytes.ContainsRune(s, rune(b))
}
internal/leviathan/binsnoop/sig_misc.go
+1399
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/binsnoop/types.go
+117
View File
@@ -0,0 +1,117 @@
package binsnoop
import (
"crypto/rand"
"fmt"
"sort"
)
// Confidence levels for SignatureResult
const (
ConfidenceLow uint8 = 0
ConfidenceMedium uint8 = 128
ConfidenceHigh uint8 = 250
)
// SignatureResult describes a validated file signature found during scanning.
type SignatureResult struct {
Offset int `json:"offset"`
ID string `json:"id"`
Size int `json:"size"`
Name string `json:"name"`
Confidence uint8 `json:"confidence"`
Description string `json:"description"`
AlwaysDisplay bool `json:"always_display"`
ExtractionDeclined bool `json:"extraction_declined"`
PreferredExtractor *Extractor `json:"-"`
}
// DeepScanResult is a validated signature retained by a non-suppressing scan.
// ParentOffset is -1 for a top-level hit.
type DeepScanResult struct {
SignatureResult
End int `json:"end"`
Depth int `json:"depth"`
ParentOffset int `json:"parent_offset"`
Overlap bool `json:"overlap"`
}
// DeepScan is a bounded nested-signature analysis result.
type DeepScan struct {
Results []DeepScanResult `json:"results"`
Truncated bool `json:"truncated"`
}
// SignatureParser validates a magic byte match and returns a SignatureResult or error.
type SignatureParser func(data []byte, offset int) (*SignatureResult, error)
// Signature defines a file signature to search for.
type Signature struct {
Name string
Short bool
Magic [][]byte
MagicOffset int
Description string
AlwaysDisplay bool
Parser SignatureParser
Extractor *Extractor
}
// ExtractorType defines the type of extractor.
type ExtractorType int
const (
ExtractorTypeNone ExtractorType = iota
ExtractorTypeInternal
ExtractorTypeExternal
)
// SourceFilePlaceholder is replaced at runtime with the actual file path.
const SourceFilePlaceholder = "%e"
// InternalExtractorFunc is the function signature for built-in extractors.
type InternalExtractorFunc func(data []byte, offset int, outputDir string) *ExtractionResult
// Extractor describes how to extract a file type.
type Extractor struct {
Type ExtractorType
Command string
Function InternalExtractorFunc
Extension string
Arguments []string
ExitCodes []int
DoNotRecurse bool
}
// ExtractionResult stores information about a completed extraction.
type ExtractionResult struct {
Size *int `json:"size,omitempty"`
Success bool `json:"success"`
ExtractorName string `json:"extractor"`
DoNotRecurse bool `json:"do_not_recurse"`
OutputDirectory string `json:"output_directory"`
}
// AnalysisResults contains all results from analysing a file or buffer.
type AnalysisResults struct {
FilePath string `json:"file_path"`
FileMap []SignatureResult `json:"file_map"`
Extractions map[string]*ExtractionResult `json:"extractions"`
}
// sortSignatureResults sorts by offset for proper file mapping.
func sortSignatureResults(results []SignatureResult) {
sort.Slice(results, func(i, j int) bool {
return results[i].Offset < results[j].Offset
})
}
// newUUID generates a v4 UUID string with no external dependencies.
func newUUID() string {
b := make([]byte, 16)
_, _ = rand.Read(b)
b[6] = (b[6] & 0x0f) | 0x40
b[8] = (b[8] & 0x3f) | 0x80
return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x",
b[0:4], b[4:6], b[6:8], b[8:10], b[10:16])
}
internal/leviathan/disasm/aarch64.go
+288
View File
@@ -0,0 +1,288 @@
package disasm
import (
"encoding/binary"
"fmt"
)
// diAArch64 is a heuristic AArch64 (ARMv8 A64) decoder. It recognises a
// subset of instructions commonly seen in disassembly and falls back to
// ".word 0x…" for everything else. 32-bit fixed-width little-endian.
func diAArch64(buf []byte, base int64, maxInstr int) []map[string]any {
out := []map[string]any{}
i := 0
for i+4 <= len(buf) && len(out) < maxInstr {
w := binary.LittleEndian.Uint32(buf[i : i+4])
mn, ops, conf := decodeA64(w, base+int64(i))
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+4]),
"mnemonic": mn,
"operands": ops,
"confident": conf,
})
i += 4
}
// Tail bytes (not multiple of 4).
for i < len(buf) && len(out) < maxInstr {
end := i + 1
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i:end]),
"mnemonic": "db",
"operands": hexCommaList(buf[i:end]),
"confident": false,
})
i++
}
return out
}
func decodeA64(w uint32, pc int64) (string, string, bool) {
// NOP
if w == 0xd503201f {
return "nop", "", true
}
// RET Xn (default x30)
if w&0xfffffc1f == 0xd65f0000 {
rn := (w >> 5) & 0x1f
if rn == 30 {
return "ret", "", true
}
return "ret", regX(rn), true
}
// BR Xn
if w&0xfffffc1f == 0xd61f0000 {
return "br", regX((w >> 5) & 0x1f), true
}
// BLR Xn
if w&0xfffffc1f == 0xd63f0000 {
return "blr", regX((w >> 5) & 0x1f), true
}
// B / BL (unconditional branch, imm26 * 4, signed)
if w&0x7c000000 == 0x14000000 {
imm := signExtend64(uint64(w&0x03ffffff)<<2, 28)
target := pc + int64(imm)
if w&0x80000000 != 0 {
return "bl", fmt.Sprintf("0x%x", target), true
}
return "b", fmt.Sprintf("0x%x", target), true
}
// B.cond (conditional branch, imm19*4)
if w&0xff000010 == 0x54000000 {
cond := int(w & 0xf)
imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21)
return "b." + condName(cond), fmt.Sprintf("0x%x", pc+int64(imm)), true
}
// CBZ / CBNZ
if w&0x7e000000 == 0x34000000 {
sf := (w >> 31) & 1
op := (w >> 24) & 1
rt := w & 0x1f
imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21)
mn := "cbz"
if op == 1 {
mn = "cbnz"
}
r := regX(rt)
if sf == 0 {
r = regW(rt)
}
return mn, fmt.Sprintf("%s, 0x%x", r, pc+int64(imm)), true
}
// MOVZ / MOVN / MOVK (wide immediate)
if w&0x1f800000 == 0x12800000 {
sf := (w >> 31) & 1
opc := (w >> 29) & 0x3
hw := (w >> 21) & 0x3
imm16 := (w >> 5) & 0xffff
rd := w & 0x1f
mn := "movn"
switch opc {
case 0b10:
mn = "movz"
case 0b11:
mn = "movk"
}
shift := hw * 16
r := regX(rd)
if sf == 0 {
r = regW(rd)
}
if shift == 0 {
return mn, fmt.Sprintf("%s, #0x%x", r, imm16), true
}
return mn, fmt.Sprintf("%s, #0x%x, lsl #%d", r, imm16, shift), true
}
// ADR / ADRP
if w&0x1f000000 == 0x10000000 {
page := (w >> 31) & 1
rd := w & 0x1f
immlo := uint64((w >> 29) & 0x3)
immhi := uint64((w >> 5) & 0x7ffff)
imm := (immhi << 2) | immlo
mn := "adr"
shift := uint(21)
signed := signExtend64(imm, shift)
if page == 1 {
mn = "adrp"
signed = signExtend64(imm<<12, 33)
// adrp target is page-aligned: pc & ~0xFFF
return mn, fmt.Sprintf("%s, 0x%x", regX(rd), (pc & ^int64(0xfff))+int64(signed)), true
}
return mn, fmt.Sprintf("%s, 0x%x", regX(rd), pc+int64(signed)), true
}
// ADD / SUB (immediate)
if w&0x1f000000 == 0x11000000 {
sf := (w >> 31) & 1
op := (w >> 30) & 1
sh := (w >> 22) & 1
imm12 := (w >> 10) & 0xfff
rn := (w >> 5) & 0x1f
rd := w & 0x1f
mn := "add"
if op == 1 {
mn = "sub"
}
lhs := regX(rd)
mid := regX(rn)
if sf == 0 {
lhs = regW(rd)
mid = regW(rn)
}
if sh == 1 {
return mn, fmt.Sprintf("%s, %s, #0x%x, lsl #12", lhs, mid, imm12), true
}
return mn, fmt.Sprintf("%s, %s, #0x%x", lhs, mid, imm12), true
}
// LDR / STR (unsigned offset, 32/64-bit int)
if w&0x3fc00000 == 0x39400000 || w&0x3fc00000 == 0x39000000 {
size := (w >> 30) & 0x3
l := (w >> 22) & 1
imm12 := (w >> 10) & 0xfff
rn := (w >> 5) & 0x1f
rt := w & 0x1f
var mn string
if l == 1 {
mn = "ldr"
} else {
mn = "str"
}
scale := int(1 << size)
imm := int(imm12) * scale
var reg string
if size == 3 {
reg = regX(rt)
} else {
reg = regW(rt)
}
base := regXorSP(rn)
if imm == 0 {
return mn, fmt.Sprintf("%s, [%s]", reg, base), true
}
return mn, fmt.Sprintf("%s, [%s, #0x%x]", reg, base, imm), true
}
// LDR (literal / PC-rel)
if w&0xbf000000 == 0x18000000 {
sf := (w >> 30) & 1
imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21)
rt := w & 0x1f
reg := regW(rt)
if sf == 1 {
reg = regX(rt)
}
return "ldr", fmt.Sprintf("%s, 0x%x", reg, pc+int64(imm)), true
}
// ORR / AND / EOR (shifted reg) — simplified detection, no shifts reported.
if w&0x7f200000 == 0x2a000000 { // ORR (32/64) w/ shift=0
sf := (w >> 31) & 1
rm := (w >> 16) & 0x1f
rn := (w >> 5) & 0x1f
rd := w & 0x1f
// MOV (register) alias: ORR Xd, XZR, Xm → MOV Xd, Xm (when Rn==31 and shift==0)
if rn == 31 && ((w>>10)&0x3f) == 0 && ((w>>22)&0x3) == 0 {
if sf == 1 {
return "mov", fmt.Sprintf("%s, %s", regX(rd), regX(rm)), true
}
return "mov", fmt.Sprintf("%s, %s", regW(rd), regW(rm)), true
}
if sf == 1 {
return "orr", fmt.Sprintf("%s, %s, %s", regX(rd), regX(rn), regX(rm)), true
}
return "orr", fmt.Sprintf("%s, %s, %s", regW(rd), regW(rn), regW(rm)), true
}
// SVC #imm16
if w&0xffe0001f == 0xd4000001 {
imm16 := (w >> 5) & 0xffff
return "svc", fmt.Sprintf("#0x%x", imm16), true
}
// BRK #imm16
if w&0xffe0001f == 0xd4200000 {
imm16 := (w >> 5) & 0xffff
return "brk", fmt.Sprintf("#0x%x", imm16), true
}
// HINT family → treat common ones
if w&0xfffff01f == 0xd503201f {
op := (w >> 5) & 0x7f
switch op {
case 0:
return "nop", "", true
case 1:
return "yield", "", true
case 2:
return "wfe", "", true
case 3:
return "wfi", "", true
case 4:
return "sev", "", true
case 5:
return "sevl", "", true
}
return "hint", fmt.Sprintf("#0x%x", op), true
}
// Fallback
return ".word", fmt.Sprintf("0x%08x", w), false
}
func regX(r uint32) string {
r &= 0x1f
if r == 31 {
return "xzr"
}
return fmt.Sprintf("x%d", r)
}
func regW(r uint32) string {
r &= 0x1f
if r == 31 {
return "wzr"
}
return fmt.Sprintf("w%d", r)
}
// regXorSP is used for the "Rn" slot in memory ops where 31 means SP, not XZR.
func regXorSP(r uint32) string {
r &= 0x1f
if r == 31 {
return "sp"
}
return fmt.Sprintf("x%d", r)
}
func condName(c int) string {
names := []string{
"eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc",
"hi", "ls", "ge", "lt", "gt", "le", "al", "nv",
}
if c >= 0 && c < 16 {
return names[c]
}
return fmt.Sprintf("c%d", c)
}
func signExtend64(v uint64, bits uint) int64 {
mask := uint64(1) << (bits - 1)
if v&mask != 0 {
v |= ^uint64(0) << bits
}
return int64(v)
}
internal/leviathan/disasm/arm32.go
+209
View File
@@ -0,0 +1,209 @@
package disasm
import (
"encoding/binary"
"fmt"
"strings"
)
// diARM32 is a heuristic ARM state decoder for 32-bit little-endian ARM
// instructions. Thumb/T32 is intentionally not decoded here.
func diARM32(buf []byte, base int64, maxInstr int) []map[string]any {
out := []map[string]any{}
i := 0
for i+4 <= len(buf) && len(out) < maxInstr {
w := binary.LittleEndian.Uint32(buf[i : i+4])
mn, ops, conf := decodeARM32(w, base+int64(i))
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+4]),
"mnemonic": mn,
"operands": ops,
"confident": conf,
})
i += 4
}
for i < len(buf) && len(out) < maxInstr {
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+1]),
"mnemonic": "db",
"operands": hexCommaList(buf[i : i+1]),
"confident": false,
})
i++
}
return out
}
func decodeARM32(w uint32, pc int64) (string, string, bool) {
cond := (w >> 28) & 0xf
suf := armCondSuffix(cond)
if cond == 0xf {
suf = ".nv"
}
// BX Rn
if w&0x0ffffff0 == 0x012fff10 {
return "bx" + suf, armReg(w & 0xf), true
}
// SVC #imm24
if w&0x0f000000 == 0x0f000000 {
return "svc" + suf, fmt.Sprintf("#0x%x", w&0x00ffffff), true
}
// B / BL, offset is sign-extended imm24 << 2 from PC+8.
if w&0x0e000000 == 0x0a000000 {
imm := signExtend64(uint64(w&0x00ffffff)<<2, 26)
target := pc + 8 + int64(imm)
if w&0x01000000 != 0 {
return "bl" + suf, fmt.Sprintf("0x%x", target), true
}
return "b" + suf, fmt.Sprintf("0x%x", target), true
}
// PUSH/POP common aliases.
if w&0xffff0000 == 0xe92d0000 {
return "push", armRegList(w & 0xffff), true
}
if w&0xffff0000 == 0xe8bd0000 {
return "pop", armRegList(w & 0xffff), true
}
// LDR/STR immediate offset.
if w&0x0c000000 == 0x04000000 {
p := (w >> 24) & 1
u := (w >> 23) & 1
b := (w >> 22) & 1
wb := (w >> 21) & 1
l := (w >> 20) & 1
rn := (w >> 16) & 0xf
rd := (w >> 12) & 0xf
imm := w & 0xfff
mn := "str"
if l == 1 {
mn = "ldr"
}
if b == 1 {
mn += "b"
}
mn += suf
sign := ""
if u == 0 {
sign = "-"
}
addr := ""
switch {
case p == 1 && imm == 0 && wb == 0:
addr = fmt.Sprintf("[%s]", armReg(rn))
case p == 1:
addr = fmt.Sprintf("[%s, #%s0x%x]", armReg(rn), sign, imm)
if wb == 1 {
addr += "!"
}
default:
addr = fmt.Sprintf("[%s], #%s0x%x", armReg(rn), sign, imm)
}
return mn, fmt.Sprintf("%s, %s", armReg(rd), addr), true
}
// Data processing immediate/register subset.
if w&0x0c000000 == 0x00000000 || w&0x0e000000 == 0x02000000 {
immediate := w&0x02000000 != 0
opcode := (w >> 21) & 0xf
s := (w >> 20) & 1
rn := (w >> 16) & 0xf
rd := (w >> 12) & 0xf
mn := armDataProcName(opcode)
if mn == "" {
return ".word", fmt.Sprintf("0x%08x", w), false
}
mn += suf
if s == 1 && opcode != 8 && opcode != 9 && opcode != 10 && opcode != 11 {
mn += "s"
}
op2 := ""
if immediate {
imm8 := w & 0xff
rot := ((w >> 8) & 0xf) * 2
imm := (imm8 >> rot) | (imm8 << ((32 - rot) & 31))
op2 = fmt.Sprintf("#0x%x", imm)
} else {
rm := w & 0xf
shift := (w >> 4) & 0xff
if shift == 0 {
op2 = armReg(rm)
} else {
op2 = fmt.Sprintf("%s, %s", armReg(rm), armShift(shift))
}
}
switch opcode {
case 8, 9, 10, 11: // tst/teq/cmp/cmn
return mn, fmt.Sprintf("%s, %s", armReg(rn), op2), true
case 13, 15: // mov/mvn
return mn, fmt.Sprintf("%s, %s", armReg(rd), op2), true
default:
return mn, fmt.Sprintf("%s, %s, %s", armReg(rd), armReg(rn), op2), true
}
}
return ".word", fmt.Sprintf("0x%08x", w), false
}
func armCondSuffix(cond uint32) string {
return [...]string{".eq", ".ne", ".cs", ".cc", ".mi", ".pl", ".vs", ".vc", ".hi", ".ls", ".ge", ".lt", ".gt", ".le", "", ""}[cond&0xf]
}
func armDataProcName(op uint32) string {
return [...]string{"and", "eor", "sub", "rsb", "add", "adc", "sbc", "rsc", "tst", "teq", "cmp", "cmn", "orr", "mov", "bic", "mvn"}[op&0xf]
}
func armReg(r uint32) string {
switch r & 0xf {
case 13:
return "sp"
case 14:
return "lr"
case 15:
return "pc"
default:
return fmt.Sprintf("r%d", r&0xf)
}
}
func armRegList(mask uint32) string {
regs := []string{}
start := -1
flush := func(end int) {
if start < 0 {
return
}
if start == end {
regs = append(regs, armReg(uint32(start)))
} else {
regs = append(regs, fmt.Sprintf("%s-%s", armReg(uint32(start)), armReg(uint32(end))))
}
start = -1
}
for i := 0; i < 16; i++ {
if mask&(1<<uint(i)) != 0 {
if start < 0 {
start = i
}
continue
}
flush(i - 1)
}
flush(15)
return "{" + strings.Join(regs, ", ") + "}"
}
func armShift(v uint32) string {
typ := (v >> 1) & 0x3
imm := (v >> 3) & 0x1f
names := [...]string{"lsl", "lsr", "asr", "ror"}
if imm == 0 {
if typ == 0 {
return "lsl #0"
}
if typ == 3 {
return "rrx"
}
imm = 32
}
return fmt.Sprintf("%s #%d", names[typ], imm)
}
internal/leviathan/disasm/cfg.go
+541
View File
@@ -0,0 +1,541 @@
package disasm
import (
"sort"
"strconv"
"strings"
)
// cfg.go builds control-flow graphs and cross-references on top of the
// per-architecture decoders. It performs a recursive-descent sweep from an
// entry point: it follows direct branches and the fall-through after calls
// (calls leave the function, so their targets are recorded as xrefs rather
// than followed), splits the decoded instructions into basic blocks, and
// records every direct code/data reference it can resolve statically.
// Block is one basic block: a maximal straight-line instruction run.
type Block struct {
Start int64 `json:"start"`
End int64 `json:"end"` // address after the last instruction
Instrs []map[string]any `json:"instrs"`
Succs []int64 `json:"succs"` // successor block start addresses
Kind string `json:"kind"` // ret | jmp | cond | call | fallthrough
}
// Xref is a single static reference from one address to another.
type Xref struct {
From int64 `json:"from"`
To int64 `json:"to"`
Kind string `json:"kind"` // call | jump | data
}
// CFG is the analysis result for one entry point.
type CFG struct {
Entry int64 `json:"entry"`
Arch string `json:"arch"`
Blocks []*Block `json:"blocks"`
Calls []int64 `json:"calls"` // distinct call targets (in or out of range)
Xrefs []Xref `json:"xrefs"` // all resolved references
Unknown int `json:"unknown"` // indirect/unresolved branch+call count
}
// Function is a discovered function and its structural metrics.
type Function struct {
Name string `json:"name"`
Entry int64 `json:"entry"`
End int64 `json:"end"`
Blocks []*Block `json:"blocks"`
Instructions int `json:"instructions"`
Complexity int `json:"complexity"`
Calls []int64 `json:"calls"`
CalledBy []int64 `json:"calledBy"`
Unknown int `json:"unknown"`
}
// CallEdge is one direct call from a discovered function.
type CallEdge struct {
Caller int64 `json:"caller"`
CallSite int64 `json:"callSite"`
Callee int64 `json:"callee"`
External bool `json:"external"`
}
// Seed is a known or suspected function entry supplied by executable metadata.
type Seed struct {
Name string `json:"name"`
Entry int64 `json:"entry"`
}
// Program is a bounded recursive function-discovery result for one byte range.
type Program struct {
Entry int64 `json:"entry"`
Base int64 `json:"base"`
End int64 `json:"end"`
Arch string `json:"arch"`
Functions []*Function `json:"functions"`
Edges []CallEdge `json:"edges"`
ExternalCalls []int64 `json:"externalCalls"`
Instructions int `json:"instructions"`
Unknown int `json:"unknown"`
Truncated bool `json:"truncated"`
}
type rawInstr struct {
addr int64
length int
mnem string
class string
target int64
hasTgt bool
data int64
hasDat bool
m map[string]any
}
// BuildCFG decodes from entry (within buf, which starts at address base) and
// returns the control-flow graph plus discovered references.
func BuildCFG(buf []byte, base int64, arch string, entry int64, maxInstr int) *CFG {
if maxInstr <= 0 {
maxInstr = 8192
}
cfg := &CFG{Entry: entry, Arch: arch}
decoded := map[int64]*rawInstr{}
inRange := func(a int64) bool { return a >= base && a < base+int64(len(buf)) }
var stack []int64
push := func(a int64) {
if inRange(a) {
stack = append(stack, a)
}
}
push(entry)
for len(stack) > 0 && len(decoded) < maxInstr {
addr := stack[len(stack)-1]
stack = stack[:len(stack)-1]
for inRange(addr) && len(decoded) < maxInstr {
if _, seen := decoded[addr]; seen {
break
}
ri := decodeAt(buf, addr, base, arch)
if ri == nil {
break
}
decoded[addr] = ri
next := addr + int64(ri.length)
switch ri.class {
case "ret":
addr = -1
case "jmp":
if ri.hasTgt {
addr = ri.target
} else {
cfg.Unknown++
addr = -1
}
case "cond":
if ri.hasTgt {
push(ri.target)
} else {
cfg.Unknown++
}
addr = next
case "call":
if !ri.hasTgt {
cfg.Unknown++
}
addr = next
default:
addr = next
}
if addr < 0 {
break
}
}
}
// Collect references and call targets.
callSet := map[int64]bool{}
for _, ri := range decoded {
if ri.hasTgt {
switch ri.class {
case "call":
cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.target, Kind: "call"})
if !callSet[ri.target] {
callSet[ri.target] = true
cfg.Calls = append(cfg.Calls, ri.target)
}
case "jmp", "cond":
cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.target, Kind: "jump"})
}
}
if ri.hasDat {
cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.data, Kind: "data"})
}
}
sort.Slice(cfg.Calls, func(i, j int) bool { return cfg.Calls[i] < cfg.Calls[j] })
sort.Slice(cfg.Xrefs, func(i, j int) bool {
if cfg.Xrefs[i].From != cfg.Xrefs[j].From {
return cfg.Xrefs[i].From < cfg.Xrefs[j].From
}
return cfg.Xrefs[i].To < cfg.Xrefs[j].To
})
cfg.Blocks = buildBlocks(decoded, inRange)
return cfg
}
// DiscoverFunctions recursively follows direct calls from entry, building a
// bounded call graph over buf. Indirect calls are counted but cannot be
// discovered statically. maxInstr is a total budget across all functions.
func DiscoverFunctions(buf []byte, base int64, arch string, entry int64, maxFunctions, maxInstr int) *Program {
return DiscoverFunctionsFromSeeds(buf, base, arch, []Seed{{Entry: entry}}, maxFunctions, maxInstr)
}
// DiscoverFunctionsFromSeeds analyzes named and unnamed roots plus every
// direct in-range call target reached from them.
func DiscoverFunctionsFromSeeds(buf []byte, base int64, arch string, seeds []Seed, maxFunctions, maxInstr int) *Program {
if maxFunctions <= 0 || maxFunctions > 4096 {
maxFunctions = 256
}
if maxInstr <= 0 || maxInstr > 1_000_000 {
maxInstr = 65_536
}
end := base + int64(len(buf))
entry := base
if len(seeds) > 0 {
entry = seeds[0].Entry
}
program := &Program{Entry: entry, Base: base, End: end, Arch: arch}
inRange := func(addr int64) bool { return addr >= base && addr < end }
queue := make([]int64, 0, len(seeds))
queued := map[int64]bool{}
names := map[int64]string{}
for _, seed := range seeds {
if !inRange(seed.Entry) {
continue
}
if !queued[seed.Entry] {
queued[seed.Entry] = true
queue = append(queue, seed.Entry)
}
if seed.Name != "" {
names[seed.Entry] = seed.Name
}
}
if len(queue) == 0 {
program.Truncated = true
return program
}
discovered := map[int64]*Function{}
callerSets := map[int64]map[int64]bool{}
externalSet := map[int64]bool{}
for len(queue) > 0 && len(discovered) < maxFunctions && program.Instructions < maxInstr {
fnEntry := queue[0]
queue = queue[1:]
if discovered[fnEntry] != nil {
continue
}
remaining := maxInstr - program.Instructions
perFunction := remaining
if perFunction > 8192 {
perFunction = 8192
}
cfg := BuildCFG(buf, base, arch, fnEntry, perFunction)
fn := summarizeFunction(cfg, names[fnEntry])
discovered[fnEntry] = fn
program.Functions = append(program.Functions, fn)
program.Instructions += fn.Instructions
program.Unknown += fn.Unknown
for _, xref := range cfg.Xrefs {
if xref.Kind != "call" {
continue
}
external := !inRange(xref.To)
program.Edges = append(program.Edges, CallEdge{
Caller: fnEntry, CallSite: xref.From, Callee: xref.To, External: external,
})
if external {
externalSet[xref.To] = true
continue
}
if callerSets[xref.To] == nil {
callerSets[xref.To] = map[int64]bool{}
}
callerSets[xref.To][fnEntry] = true
if !queued[xref.To] {
queued[xref.To] = true
queue = append(queue, xref.To)
}
}
}
if len(queue) > 0 || program.Instructions >= maxInstr {
program.Truncated = true
}
for _, fn := range program.Functions {
for caller := range callerSets[fn.Entry] {
fn.CalledBy = append(fn.CalledBy, caller)
}
sort.Slice(fn.CalledBy, func(i, j int) bool { return fn.CalledBy[i] < fn.CalledBy[j] })
}
for target := range externalSet {
program.ExternalCalls = append(program.ExternalCalls, target)
}
sort.Slice(program.Functions, func(i, j int) bool { return program.Functions[i].Entry < program.Functions[j].Entry })
sort.Slice(program.Edges, func(i, j int) bool {
if program.Edges[i].Caller != program.Edges[j].Caller {
return program.Edges[i].Caller < program.Edges[j].Caller
}
if program.Edges[i].CallSite != program.Edges[j].CallSite {
return program.Edges[i].CallSite < program.Edges[j].CallSite
}
return program.Edges[i].Callee < program.Edges[j].Callee
})
sort.Slice(program.ExternalCalls, func(i, j int) bool { return program.ExternalCalls[i] < program.ExternalCalls[j] })
return program
}
func summarizeFunction(cfg *CFG, name string) *Function {
if name == "" {
name = unknownFunctionName(cfg.Entry)
}
fn := &Function{
Name: name, Entry: cfg.Entry, Blocks: cfg.Blocks,
Calls: append([]int64(nil), cfg.Calls...), Unknown: cfg.Unknown,
}
edges := 0
for _, block := range cfg.Blocks {
if block.End > fn.End {
fn.End = block.End
}
fn.Instructions += len(block.Instrs)
edges += len(block.Succs)
}
fn.Complexity = edges - len(cfg.Blocks) + 2
if fn.Complexity < 1 {
fn.Complexity = 1
}
return fn
}
func unknownFunctionName(entry int64) string {
return "sub_" + strconv.FormatInt(entry, 16)
}
func buildBlocks(decoded map[int64]*rawInstr, inRange func(int64) bool) []*Block {
if len(decoded) == 0 {
return nil
}
addrs := make([]int64, 0, len(decoded))
for a := range decoded {
addrs = append(addrs, a)
}
sort.Slice(addrs, func(i, j int) bool { return addrs[i] < addrs[j] })
// Leaders: first instr, every branch target, and the instruction after any
// branch/call/ret.
leaders := map[int64]bool{addrs[0]: true}
for _, a := range addrs {
ri := decoded[a]
next := a + int64(ri.length)
switch ri.class {
case "ret":
if _, ok := decoded[next]; ok {
leaders[next] = true
}
case "jmp":
if ri.hasTgt {
if _, ok := decoded[ri.target]; ok {
leaders[ri.target] = true
}
}
if _, ok := decoded[next]; ok {
leaders[next] = true
}
case "cond", "call":
if ri.hasTgt {
if _, ok := decoded[ri.target]; ok {
leaders[ri.target] = true
}
}
if _, ok := decoded[next]; ok {
leaders[next] = true
}
}
}
var blocks []*Block
var cur *Block
flush := func() {
if cur != nil {
blocks = append(blocks, cur)
cur = nil
}
}
for idx, a := range addrs {
ri := decoded[a]
if leaders[a] {
flush()
cur = &Block{Start: a}
}
if cur == nil {
cur = &Block{Start: a}
}
cur.Instrs = append(cur.Instrs, ri.m)
next := a + int64(ri.length)
cur.End = next
// Decide whether this instruction terminates the block.
isLast := idx == len(addrs)-1
nextIsLeader := false
if !isLast {
if leaders[addrs[idx+1]] {
nextIsLeader = true
}
}
terminator := ri.class == "ret" || ri.class == "jmp" || ri.class == "cond"
if terminator || nextIsLeader || isLast {
cur.Kind = blockKind(ri.class)
switch ri.class {
case "ret":
// no successors
case "jmp":
if ri.hasTgt && inRange(ri.target) {
cur.Succs = []int64{ri.target}
}
case "cond":
if ri.hasTgt && inRange(ri.target) {
cur.Succs = append(cur.Succs, ri.target)
}
if _, ok := decoded[next]; ok {
cur.Succs = append(cur.Succs, next)
}
default:
if _, ok := decoded[next]; ok {
cur.Succs = []int64{next}
}
}
flush()
}
}
flush()
return blocks
}
func blockKind(class string) string {
switch class {
case "ret", "jmp", "cond", "call":
return class
default:
return "fallthrough"
}
}
// decodeAt decodes a single instruction at address addr (buf starts at base).
func decodeAt(buf []byte, addr, base int64, arch string) *rawInstr {
off := int(addr - base)
if off < 0 || off >= len(buf) {
return nil
}
end := off + 16 // max x86 instruction length; fixed-width arches need <= 4
if end > len(buf) {
end = len(buf)
}
res := Disassemble(buf[off:end], addr, arch, 1)
if len(res) == 0 {
return nil
}
m := res[0]
ri := &rawInstr{addr: addr, m: m}
ri.length = byteCount(m["bytes"].(string))
if ri.length <= 0 {
ri.length = 1
}
ri.mnem, _ = m["mnemonic"].(string)
ops, _ := m["operands"].(string)
ri.class = classify(ri.mnem)
if t, ok := directTarget(ops); ok {
ri.target, ri.hasTgt = t, true
}
if dataReferencing(ri.class) {
if d, ok := dataTarget(ops); ok {
ri.data, ri.hasDat = d, true
}
}
return ri
}
func byteCount(s string) int {
if s == "" {
return 0
}
return len(strings.Fields(s))
}
func classify(mnem string) string {
switch mnem {
case "ret", "retf", "iret":
return "ret"
case "jmp", "jmpf", "b", "br":
return "jmp"
case "call", "callf", "bl", "blr", "jal", "jalr":
return "call"
case "cbz", "cbnz", "tbz", "tbnz", "loop", "loope", "loopne", "jecxz", "jrcxz":
return "cond"
}
if strings.HasPrefix(mnem, "b.") { // AArch64 conditional branch
return "cond"
}
if strings.HasPrefix(mnem, "j") { // x86 Jcc (jmp already handled above)
return "cond"
}
if strings.HasPrefix(mnem, "beq") || strings.HasPrefix(mnem, "bne") { // MIPS
return "cond"
}
return "normal"
}
func dataReferencing(class string) bool {
// RIP-relative resolved targets show up as "; 0x.." comments on any
// instruction; only mine them on non-control-flow ops as data xrefs.
return class == "normal" || class == "call"
}
// directTarget extracts a branch/call target when the final operand is a bare
// absolute address (a direct, statically known target). Register/memory
// indirect operands return false.
func directTarget(operands string) (int64, bool) {
if i := strings.Index(operands, ";"); i >= 0 {
operands = operands[:i]
}
if c := strings.LastIndex(operands, ","); c >= 0 {
operands = operands[c+1:]
}
operands = strings.TrimSpace(operands)
return parseHexAddr(operands)
}
// dataTarget extracts a resolved address from a "; 0x.." trailing comment
// (e.g. an x86 RIP-relative memory operand).
func dataTarget(operands string) (int64, bool) {
i := strings.Index(operands, ";")
if i < 0 {
return 0, false
}
return parseHexAddr(strings.TrimSpace(operands[i+1:]))
}
func parseHexAddr(s string) (int64, bool) {
if !strings.HasPrefix(s, "0x") {
return 0, false
}
v, err := strconv.ParseUint(s[2:], 16, 64)
if err != nil {
return 0, false
}
return int64(v), true
}
internal/leviathan/disasm/image.go
+74
View File
@@ -0,0 +1,74 @@
package disasm
import (
"github.com/andydixon/nexus/internal/leviathan/document"
"github.com/andydixon/nexus/internal/leviathan/parsers"
)
// ImageAnalysis is a metadata-seeded analysis across file-backed executable
// regions in an ELF or PE image.
type ImageAnalysis struct {
Info *parsers.ExecutableInfo `json:"info"`
Programs []*Program `json:"programs"`
Truncated bool `json:"truncated"`
}
// DiscoverExecutable analyzes executable regions using header entries,
// symbols and exports as roots. Budgets apply across the whole image.
func DiscoverExecutable(d *document.Document, maxBytes int64, maxFunctions, maxInstr int) (*ImageAnalysis, error) {
info, err := parsers.InspectExecutable(d)
if err != nil {
return nil, err
}
if maxBytes <= 0 || maxBytes > 32<<20 {
maxBytes = 8 << 20
}
if maxFunctions <= 0 || maxFunctions > 4096 {
maxFunctions = 512
}
if maxInstr <= 0 || maxInstr > 1_000_000 {
maxInstr = 131_072
}
analysis := &ImageAnalysis{Info: info}
remainingBytes := maxBytes
remainingFunctions := maxFunctions
remainingInstr := maxInstr
for _, region := range info.Regions {
if remainingBytes <= 0 || remainingFunctions <= 0 || remainingInstr <= 0 {
analysis.Truncated = true
break
}
length := region.Size
if length > 4<<20 {
length = 4 << 20
analysis.Truncated = true
}
if length > remainingBytes {
length = remainingBytes
analysis.Truncated = true
}
if length <= 0 {
continue
}
buf, err := d.Range(region.Offset, length)
if err != nil {
return nil, err
}
seeds := make([]Seed, 0)
for _, seed := range info.Seeds {
if seed.Offset >= region.Offset && seed.Offset < region.Offset+length {
seeds = append(seeds, Seed{Name: seed.Name, Entry: seed.Offset})
}
}
if len(seeds) == 0 {
seeds = append(seeds, Seed{Name: region.Name, Entry: region.Offset})
}
program := DiscoverFunctionsFromSeeds(buf, region.Offset, info.Arch, seeds, remainingFunctions, remainingInstr)
analysis.Programs = append(analysis.Programs, program)
remainingBytes -= length
remainingFunctions -= len(program.Functions)
remainingInstr -= program.Instructions
analysis.Truncated = analysis.Truncated || program.Truncated
}
return analysis, nil
}
internal/leviathan/disasm/mips.go
+144
View File
@@ -0,0 +1,144 @@
package disasm
import (
"encoding/binary"
"fmt"
)
func diMIPS32(buf []byte, base int64, little bool, maxInstr int) []map[string]any {
out := []map[string]any{}
for i := 0; i+4 <= len(buf) && len(out) < maxInstr; i += 4 {
var w uint32
if little {
w = binary.LittleEndian.Uint32(buf[i : i+4])
} else {
w = binary.BigEndian.Uint32(buf[i : i+4])
}
mn, ops, conf := decodeMIPS32(w, base+int64(i))
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+4]),
"mnemonic": mn,
"operands": ops,
"confident": conf,
})
}
for i := len(buf) - len(buf)%4; i < len(buf) && len(out) < maxInstr; i++ {
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+1]),
"mnemonic": "db",
"operands": hexCommaList(buf[i : i+1]),
"confident": false,
})
}
return out
}
func decodeMIPS32(w uint32, pc int64) (string, string, bool) {
if w == 0 {
return "nop", "", true
}
op := (w >> 26) & 0x3f
rs := (w >> 21) & 0x1f
rt := (w >> 16) & 0x1f
rd := (w >> 11) & 0x1f
shamt := (w >> 6) & 0x1f
funct := w & 0x3f
imm := uint16(w)
simm := int16(imm)
target26 := w & 0x03ffffff
switch op {
case 0x00:
switch funct {
case 0x00:
return "sll", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true
case 0x02:
return "srl", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true
case 0x03:
return "sra", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true
case 0x08:
return "jr", mipsReg(rs), true
case 0x09:
return "jalr", fmt.Sprintf("%s, %s", mipsReg(rd), mipsReg(rs)), true
case 0x0c:
return "syscall", fmt.Sprintf("0x%x", (w>>6)&0xfffff), true
case 0x20:
return "add", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x21:
return "addu", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x22:
return "sub", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x23:
return "subu", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x24:
return "and", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x25:
return "or", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x26:
return "xor", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x27:
return "nor", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
case 0x2a:
return "slt", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true
}
case 0x02:
return "j", fmt.Sprintf("0x%x", mipsJumpTarget(pc, target26)), true
case 0x03:
return "jal", fmt.Sprintf("0x%x", mipsJumpTarget(pc, target26)), true
case 0x04:
return "beq", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rs), mipsReg(rt), mipsBranchTarget(pc, simm)), true
case 0x05:
return "bne", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rs), mipsReg(rt), mipsBranchTarget(pc, simm)), true
case 0x08:
return "addi", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true
case 0x09:
return "addiu", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true
case 0x0a:
return "slti", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true
case 0x0c:
return "andi", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true
case 0x0d:
return "ori", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true
case 0x0e:
return "xori", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true
case 0x0f:
return "lui", fmt.Sprintf("%s, 0x%x", mipsReg(rt), imm), true
case 0x20:
return "lb", mipsMem(rt, rs, simm), true
case 0x21:
return "lh", mipsMem(rt, rs, simm), true
case 0x23:
return "lw", mipsMem(rt, rs, simm), true
case 0x28:
return "sb", mipsMem(rt, rs, simm), true
case 0x29:
return "sh", mipsMem(rt, rs, simm), true
case 0x2b:
return "sw", mipsMem(rt, rs, simm), true
}
return ".word", fmt.Sprintf("0x%08x", w), false
}
func mipsJumpTarget(pc int64, target uint32) int64 {
upper := uint64(pc+4) & 0xfffffffff0000000
return int64(upper | uint64(target<<2))
}
func mipsBranchTarget(pc int64, imm int16) int64 {
return pc + 4 + int64(imm)*4
}
func mipsMem(rt, rs uint32, off int16) string {
return fmt.Sprintf("%s, %d(%s)", mipsReg(rt), off, mipsReg(rs))
}
func mipsReg(r uint32) string {
return [...]string{
"$zero", "$at", "$v0", "$v1", "$a0", "$a1", "$a2", "$a3",
"$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7",
"$s0", "$s1", "$s2", "$s3", "$s4", "$s5", "$s6", "$s7",
"$t8", "$t9", "$k0", "$k1", "$gp", "$sp", "$fp", "$ra",
}[r&0x1f]
}
internal/leviathan/disasm/thumb.go
+117
View File
@@ -0,0 +1,117 @@
package disasm
import (
"encoding/binary"
"fmt"
)
func diThumb16(buf []byte, base int64, maxInstr int) []map[string]any {
out := []map[string]any{}
for i := 0; i+2 <= len(buf) && len(out) < maxInstr; i += 2 {
h := binary.LittleEndian.Uint16(buf[i : i+2])
mn, ops, conf := decodeThumb16(h, base+int64(i))
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+2]),
"mnemonic": mn,
"operands": ops,
"confident": conf,
})
}
for i := len(buf) - len(buf)%2; i < len(buf) && len(out) < maxInstr; i++ {
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i : i+1]),
"mnemonic": "db",
"operands": hexCommaList(buf[i : i+1]),
"confident": false,
})
}
return out
}
func decodeThumb16(h uint16, pc int64) (string, string, bool) {
switch {
case h == 0xbf00:
return "nop", "", true
case h&0xff87 == 0x4700:
return "bx", thumbHighReg((h >> 3) & 0xf), true
case h&0xff87 == 0x4780:
return "blx", thumbHighReg((h >> 3) & 0xf), true
case h&0xf800 == 0xe000:
imm := signExtend64(uint64(h&0x07ff)<<1, 12)
return "b", fmt.Sprintf("0x%x", pc+4+imm), true
case h&0xf000 == 0xd000 && h&0x0f00 != 0x0f00:
cond := uint32((h >> 8) & 0xf)
imm := signExtend64(uint64(h&0x00ff)<<1, 9)
return "b" + armCondSuffix(cond), fmt.Sprintf("0x%x", pc+4+imm), true
case h&0xff00 == 0xdf00:
return "svc", fmt.Sprintf("#0x%x", h&0x00ff), true
case h&0xfe00 == 0xb400:
mask := uint32(h & 0x00ff)
if h&0x0100 != 0 {
mask |= 1 << 14
}
return "push", armRegList(mask), true
case h&0xfe00 == 0xbc00:
mask := uint32(h & 0x00ff)
if h&0x0100 != 0 {
mask |= 1 << 15
}
return "pop", armRegList(mask), true
case h&0xf800 == 0x2000:
rd := uint32((h >> 8) & 0x7)
return "movs", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true
case h&0xf800 == 0x2800:
rn := uint32((h >> 8) & 0x7)
return "cmp", fmt.Sprintf("%s, #0x%x", armReg(rn), h&0x00ff), true
case h&0xf800 == 0x3000:
rd := uint32((h >> 8) & 0x7)
return "adds", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true
case h&0xf800 == 0x3800:
rd := uint32((h >> 8) & 0x7)
return "subs", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true
case h&0xfc00 == 0x4000:
op := (h >> 6) & 0xf
rs := uint32((h >> 3) & 0x7)
rd := uint32(h & 0x7)
names := [...]string{"ands", "eors", "lsls", "lsrs", "asrs", "adcs", "sbcs", "rors", "tst", "rsbs", "cmp", "cmn", "orrs", "muls", "bics", "mvns"}
mn := names[op]
if op == 8 || op == 10 || op == 11 {
return mn, fmt.Sprintf("%s, %s", armReg(rd), armReg(rs)), true
}
return mn, fmt.Sprintf("%s, %s", armReg(rd), armReg(rs)), true
case h&0xfe00 == 0x1800:
rm := uint32((h >> 6) & 0x7)
rn := uint32((h >> 3) & 0x7)
rd := uint32(h & 0x7)
mn := "adds"
if h&0x0200 != 0 {
mn = "subs"
}
return mn, fmt.Sprintf("%s, %s, %s", armReg(rd), armReg(rn), armReg(rm)), true
case h&0xe000 == 0x6000:
imm := uint32((h>>6)&0x1f) * 4
rn := uint32((h >> 3) & 0x7)
rd := uint32(h & 0x7)
mn := "str"
if h&0x0800 != 0 {
mn = "ldr"
}
return mn, fmt.Sprintf("%s, [%s, #0x%x]", armReg(rd), armReg(rn), imm), true
}
return ".hword", fmt.Sprintf("0x%04x", h), false
}
func thumbHighReg(r uint16) string {
if r == 13 {
return "sp"
}
if r == 14 {
return "lr"
}
if r == 15 {
return "pc"
}
return fmt.Sprintf("r%d", r)
}
internal/leviathan/disasm/x86.go
+91
View File
@@ -0,0 +1,91 @@
// Package disasm provides a honest, limited heuristic disassembler.
// It does NOT aim to be a full x86/ARM decoder. It recognises a
// subset of common opcodes so the UI can show plausible disassembly
// for inspection, and is explicit in the output when instructions
// are unknown.
package disasm
import "fmt"
// Instr is one decoded instruction.
type Instr struct {
Offset int64
Bytes []byte
Mnemonic string
Operands string
Confident bool
}
// Disassemble returns a slice of Instr for buf starting at `base`.
// arch is one of "x86", "x86_64", "arm", "thumb", "aarch64", "mips".
// For unsupported arches we emit a byte stream with "db" entries.
func Disassemble(buf []byte, base int64, arch string, maxInstr int) []map[string]any {
if maxInstr <= 0 {
maxInstr = 4096
}
switch arch {
case "x86", "x86_64":
return diX86(buf, base, arch == "x86_64", maxInstr)
case "aarch64", "arm64":
return diAArch64(buf, base, maxInstr)
case "arm", "arm32":
return diARM32(buf, base, maxInstr)
case "thumb", "thumb16", "t16":
return diThumb16(buf, base, maxInstr)
case "mips", "mipsbe", "mips32":
return diMIPS32(buf, base, false, maxInstr)
case "mipsle", "mipsel":
return diMIPS32(buf, base, true, maxInstr)
}
return diRaw(buf, base, arch, maxInstr)
}
// diRaw emits `db` lines 16 bytes at a time for unsupported arches.
func diRaw(buf []byte, base int64, arch string, maxInstr int) []map[string]any {
out := []map[string]any{}
step := 16
for i := 0; i < len(buf) && len(out) < maxInstr; i += step {
end := i + step
if end > len(buf) {
end = len(buf)
}
out = append(out, map[string]any{
"offset": base + int64(i),
"bytes": hexStr(buf[i:end]),
"mnemonic": "db",
"operands": hexCommaList(buf[i:end]),
"confident": false,
"note": arch + " decode not implemented; raw bytes only",
})
}
return out
}
// diX86 decodes x86 / x86-64 instructions. The full decoder lives in
// x86_decode.go; this dispatcher only forwards to it.
func diX86(buf []byte, base int64, is64 bool, maxInstr int) []map[string]any {
return decodeX86(buf, base, is64, maxInstr)
}
func hexStr(b []byte) string {
const h = "0123456789abcdef"
out := make([]byte, 0, len(b)*3)
for i, c := range b {
if i > 0 {
out = append(out, ' ')
}
out = append(out, h[c>>4], h[c&0x0f])
}
return string(out)
}
func hexCommaList(b []byte) string {
out := ""
for i, c := range b {
if i > 0 {
out += ","
}
out += fmt.Sprintf("0x%02x", c)
}
return out
}
internal/leviathan/disasm/x86_decode.go
+1172
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/document/document.go
+439
View File
@@ -0,0 +1,439 @@
// Package document implements the chunked binary document model.
//
// The document holds three conceptual layers per slot:
//
// source — immutable original bytes, loaded once.
// patch — sparse overlay of modified bytes keyed by offset.
// working — logical view = source with patch applied on read.
//
// Reads happen through windowed Range() calls so neither the editor
// nor analysis services need to materialise the whole file.
package document
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sync"
)
// ChunkSize is the granularity at which source bytes are exposed to
// analysis services. It is independent of the view row width.
const ChunkSize = 64 * 1024
// Document represents one loaded artefact. A Session may hold many.
type Document struct {
mu sync.RWMutex
name string
source []byte // original immutable content
path string // optional immutable file-backed source
size int64
patch map[int64]byte
history []EditOp
redo []EditOp
cursor int
}
// EditOp records a single byte-level mutation for undo/redo.
type EditOp struct {
Offset int64
Old byte
New byte
Kind string // "set", "fill", "xor"
}
// New creates a Document from raw source bytes. The slice is not
// copied; callers must not mutate it after handoff.
func New(name string, src []byte) *Document {
return &Document{
name: name,
source: src,
size: int64(len(src)),
patch: make(map[int64]byte),
}
}
// Open creates a Document backed by an immutable file on disk. Source
// bytes are read on demand through Range rather than loaded eagerly.
func Open(path string) (*Document, error) {
st, err := os.Stat(path)
if err != nil {
return nil, err
}
if st.IsDir() {
return nil, errors.New("document path is a directory")
}
return &Document{
name: filepath.Base(path),
path: path,
size: st.Size(),
patch: make(map[int64]byte),
}, nil
}
// Name returns the document's display name.
func (d *Document) Name() string {
d.mu.RLock()
defer d.mu.RUnlock()
return d.name
}
// Size returns the logical document length in bytes.
func (d *Document) Size() int64 {
d.mu.RLock()
defer d.mu.RUnlock()
return d.size
}
// At returns the logical byte at offset (patch takes precedence).
func (d *Document) At(offset int64) (byte, error) {
d.mu.RLock()
defer d.mu.RUnlock()
if offset < 0 || offset >= d.size {
return 0, errors.New("offset out of range")
}
if b, ok := d.patch[offset]; ok {
return b, nil
}
return d.sourceByteLocked(offset)
}
// Range copies [start, start+length) into dst. It applies the patch
// overlay. Callers get a fresh slice they are free to retain.
func (d *Document) Range(start, length int64) ([]byte, error) {
d.mu.RLock()
defer d.mu.RUnlock()
n := d.size
if start < 0 || start > n {
return nil, errors.New("range start out of bounds")
}
if length < 0 {
return nil, errors.New("negative range")
}
if length > n-start {
length = n - start
}
out, err := d.sourceRangeLocked(start, length)
if err != nil {
return nil, err
}
if len(d.patch) > 0 {
end := start + length
for off, b := range d.patch {
if off >= start && off < end {
out[off-start] = b
}
}
}
return out, nil
}
// IsDirty reports whether any byte has been patched.
func (d *Document) IsDirty() bool {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.patch) > 0
}
// Set patches a single byte. Returns the previous value.
func (d *Document) Set(offset int64, b byte) (byte, error) {
d.mu.Lock()
defer d.mu.Unlock()
if offset < 0 || offset >= d.size {
return 0, errors.New("offset out of range")
}
orig, err := d.sourceByteLocked(offset)
if err != nil {
return 0, err
}
var old byte
if v, ok := d.patch[offset]; ok {
old = v
} else {
old = orig
}
if b == orig {
delete(d.patch, offset)
} else {
d.patch[offset] = b
}
d.history = append(d.history, EditOp{Offset: offset, Old: old, New: b, Kind: "set"})
d.redo = nil
return old, nil
}
// Fill patches [start,start+length) with b.
func (d *Document) Fill(start, length int64, b byte) error {
d.mu.Lock()
defer d.mu.Unlock()
if err := d.validateRangeLocked("fill", start, length); err != nil {
return err
}
orig, err := d.sourceRangeLocked(start, length)
if err != nil {
return err
}
for i := int64(0); i < length; i++ {
off := start + i
origB := orig[i]
var old byte
if v, ok := d.patch[off]; ok {
old = v
} else {
old = origB
}
if b == origB {
delete(d.patch, off)
} else {
d.patch[off] = b
}
d.history = append(d.history, EditOp{Offset: off, Old: old, New: b, Kind: "fill"})
}
d.redo = nil
return nil
}
// XOR patches [start,start+length) with a repeating key.
func (d *Document) XOR(start, length int64, key []byte) error {
if len(key) == 0 {
return errors.New("empty key")
}
d.mu.Lock()
defer d.mu.Unlock()
if err := d.validateRangeLocked("xor", start, length); err != nil {
return err
}
orig, err := d.sourceRangeLocked(start, length)
if err != nil {
return err
}
for i := int64(0); i < length; i++ {
off := start + i
origB := orig[i]
var cur byte
if v, ok := d.patch[off]; ok {
cur = v
} else {
cur = origB
}
newB := cur ^ key[i%int64(len(key))]
if newB == origB {
delete(d.patch, off)
} else {
d.patch[off] = newB
}
d.history = append(d.history, EditOp{Offset: off, Old: cur, New: newB, Kind: "xor"})
}
d.redo = nil
return nil
}
// Copy copies [src,src+length) onto [dst,dst+length) as a patch.
func (d *Document) Copy(src, length, dst int64) error {
d.mu.Lock()
defer d.mu.Unlock()
if err := d.validateRangeLocked("copy source", src, length); err != nil {
return err
}
if err := d.validateRangeLocked("copy destination", dst, length); err != nil {
return err
}
buf, err := d.workingRangeLocked(src, length)
if err != nil {
return err
}
dstOrig, err := d.sourceRangeLocked(dst, length)
if err != nil {
return err
}
for i := int64(0); i < length; i++ {
off := dst + i
origB := dstOrig[i]
var cur byte
if v, ok := d.patch[off]; ok {
cur = v
} else {
cur = origB
}
b := buf[i]
if b == origB {
delete(d.patch, off)
} else {
d.patch[off] = b
}
d.history = append(d.history, EditOp{Offset: off, Old: cur, New: b, Kind: "fill"})
}
d.redo = nil
return nil
}
// Undo reverts the last edit. Returns false if there is no history.
func (d *Document) Undo() bool {
d.mu.Lock()
defer d.mu.Unlock()
if len(d.history) == 0 {
return false
}
op := d.history[len(d.history)-1]
d.history = d.history[:len(d.history)-1]
orig, err := d.sourceByteLocked(op.Offset)
if err != nil {
return false
}
if op.Old == orig {
delete(d.patch, op.Offset)
} else {
d.patch[op.Offset] = op.Old
}
d.redo = append(d.redo, op)
return true
}
// Redo reapplies the last undone edit. Returns false if there is no redo.
func (d *Document) Redo() bool {
d.mu.Lock()
defer d.mu.Unlock()
if len(d.redo) == 0 {
return false
}
op := d.redo[len(d.redo)-1]
d.redo = d.redo[:len(d.redo)-1]
orig, err := d.sourceByteLocked(op.Offset)
if err != nil {
return false
}
if op.New == orig {
delete(d.patch, op.Offset)
} else {
d.patch[op.Offset] = op.New
}
d.history = append(d.history, op)
return true
}
// Revert drops all patches and history.
func (d *Document) Revert() {
d.mu.Lock()
defer d.mu.Unlock()
d.patch = make(map[int64]byte)
d.history = nil
d.redo = nil
}
// Patches returns a snapshot of the patch overlay.
func (d *Document) Patches() map[int64]byte {
d.mu.RLock()
defer d.mu.RUnlock()
out := make(map[int64]byte, len(d.patch))
for k, v := range d.patch {
out[k] = v
}
return out
}
// HistoryLen returns the number of recorded edits.
func (d *Document) HistoryLen() int {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.history)
}
// RedoLen returns the number of redoable edits.
func (d *Document) RedoLen() int {
d.mu.RLock()
defer d.mu.RUnlock()
return len(d.redo)
}
// ForEachChunk yields chunks covering [start,start+length) with the
// working-copy bytes visible. It is suitable for analysis passes that
// want bounded memory use.
func (d *Document) ForEachChunk(start, length int64, fn func(off int64, chunk []byte) error) error {
d.mu.RLock()
n := d.size
d.mu.RUnlock()
if start < 0 {
start = 0
}
end := start + length
if length < 0 || end > n {
end = n
}
off := start
for off < end {
chunkEnd := off + ChunkSize
if chunkEnd > end {
chunkEnd = end
}
buf, err := d.Range(off, chunkEnd-off)
if err != nil {
return err
}
if err := fn(off, buf); err != nil {
return err
}
off = chunkEnd
}
return nil
}
func (d *Document) sourceByteLocked(offset int64) (byte, error) {
buf, err := d.sourceRangeLocked(offset, 1)
if err != nil {
return 0, err
}
if len(buf) == 0 {
return 0, errors.New("offset out of range")
}
return buf[0], nil
}
func (d *Document) workingRangeLocked(start, length int64) ([]byte, error) {
buf, err := d.sourceRangeLocked(start, length)
if err != nil {
return nil, err
}
for off, b := range d.patch {
if off >= start && off < start+length {
buf[off-start] = b
}
}
return buf, nil
}
func (d *Document) sourceRangeLocked(start, length int64) ([]byte, error) {
if err := d.validateRangeLocked("range", start, length); err != nil {
return nil, err
}
if length == 0 {
return []byte{}, nil
}
if d.path == "" {
out := make([]byte, length)
copy(out, d.source[start:start+length])
return out, nil
}
f, err := os.Open(d.path)
if err != nil {
return nil, err
}
defer f.Close()
out := make([]byte, length)
_, err = f.ReadAt(out, start)
if errors.Is(err, io.EOF) {
err = nil
}
if err != nil {
return nil, err
}
return out, nil
}
func (d *Document) validateRangeLocked(label string, start, length int64) error {
if start < 0 || start > d.size || length < 0 || length > d.size-start {
return fmt.Errorf("%s range invalid", label)
}
return nil
}
internal/leviathan/document/session.go
+62
View File
@@ -0,0 +1,62 @@
package document
import (
"errors"
"sync"
)
// Slot identifies one of the documents in a session.
type Slot string
const (
SlotOriginal Slot = "original"
SlotCompare Slot = "compare"
)
// Session tracks loaded documents and addresses them by slot.
// The original slot is the active working document; compare is
// used by the diff tooling.
type Session struct {
mu sync.RWMutex
documents map[Slot]*Document
}
// NewSession returns an empty session.
func NewSession() *Session {
return &Session{documents: make(map[Slot]*Document)}
}
// Load replaces the document in the given slot.
func (s *Session) Load(slot Slot, name string, src []byte) *Document {
s.mu.Lock()
defer s.mu.Unlock()
d := New(name, src)
s.documents[slot] = d
return d
}
// Get returns the document for a slot, or an error if not loaded.
func (s *Session) Get(slot Slot) (*Document, error) {
s.mu.RLock()
defer s.mu.RUnlock()
d, ok := s.documents[slot]
if !ok {
return nil, errors.New("slot not loaded: " + string(slot))
}
return d, nil
}
// Has reports whether a slot has a document loaded.
func (s *Session) Has(slot Slot) bool {
s.mu.RLock()
defer s.mu.RUnlock()
_, ok := s.documents[slot]
return ok
}
// Drop removes a slot.
func (s *Session) Drop(slot Slot) {
s.mu.Lock()
defer s.mu.Unlock()
delete(s.documents, slot)
}
internal/leviathan/parsers/asn1.go
+321
View File
@@ -0,0 +1,321 @@
package parsers
import (
"encoding/asn1"
"encoding/pem"
"errors"
"fmt"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ParseASN1 produces a field tree for the DER TLV structure starting
// at `at`, spanning at most `length` bytes. Nested SEQUENCE / SET
// structures are decoded recursively; primitive content is rendered
// as best-effort.
func ParseASN1(d *document.Document, at, length int64) (*Field, error) {
size := d.Size()
if length < 0 || at+length > size {
length = size - at
}
if length <= 0 {
return nil, errors.New("asn1: empty range")
}
buf, err := d.Range(at, length)
if err != nil {
return nil, err
}
// If the buffer looks like PEM, decode first.
if pemBlock, _ := pem.Decode(buf); pemBlock != nil {
root := &Field{Name: "PEM " + pemBlock.Type, Type: "container", Offset: at, Length: length}
inner, err := decodeASN1(pemBlock.Bytes, at)
if err == nil {
root.Children = append(root.Children, inner)
} else {
root.Note = "pem decoded but asn1 failed: " + err.Error()
}
return root, nil
}
return decodeASN1(buf, at)
}
// decodeASN1 walks buf recursively, treating it as one or more DER TLVs.
func decodeASN1(buf []byte, base int64) (*Field, error) {
root := &Field{Name: "ASN.1", Type: "container", Offset: base, Length: int64(len(buf))}
off := 0
for off < len(buf) {
raw, consumed, err := readTLV(buf[off:])
if err != nil {
if off == 0 {
return nil, err
}
break
}
node := asn1Node(raw, base+int64(off))
root.Children = append(root.Children, node)
off += consumed
}
if len(root.Children) == 0 {
return nil, errors.New("asn1: no TLV decoded")
}
if len(root.Children) == 1 {
return root.Children[0], nil
}
return root, nil
}
// tlv is one DER TLV we've split out of the byte stream.
type tlv struct {
Class int // 0=universal, 1=application, 2=context, 3=private
Constructed bool
Tag int
Content []byte // the V of the TLV
HeaderLen int // length of T+L bytes
}
func readTLV(buf []byte) (tlv, int, error) {
if len(buf) < 2 {
return tlv{}, 0, errors.New("short TLV")
}
b0 := buf[0]
t := tlv{
Class: int(b0 >> 6),
Constructed: b0&0x20 != 0,
}
tagVal := int(b0 & 0x1f)
i := 1
if tagVal == 0x1f {
// multi-byte tag
tagVal = 0
for i < len(buf) {
c := buf[i]
i++
tagVal = (tagVal << 7) | int(c&0x7f)
if c&0x80 == 0 {
break
}
}
}
t.Tag = tagVal
if i >= len(buf) {
return tlv{}, 0, errors.New("truncated length")
}
ln := int(buf[i])
i++
if ln >= 0x80 {
n := ln & 0x7f
if n == 0 || n > 8 || i+n > len(buf) {
return tlv{}, 0, errors.New("bad length")
}
ln = 0
for k := 0; k < n; k++ {
ln = (ln << 8) | int(buf[i+k])
}
i += n
}
if i+ln > len(buf) {
return tlv{}, 0, errors.New("length exceeds buffer")
}
t.HeaderLen = i
t.Content = buf[i : i+ln]
return t, i + ln, nil
}
func asn1Node(t tlv, off int64) *Field {
name, kind := asn1TagName(t)
total := int64(t.HeaderLen + len(t.Content))
node := &Field{Name: name, Type: kind, Offset: off, Length: total}
if t.Class == 2 {
node.Name = fmt.Sprintf("[%d]", t.Tag)
}
if t.Constructed {
// Recurse into children.
inner := t.Content
childOff := off + int64(t.HeaderLen)
i := 0
for i < len(inner) {
child, consumed, err := readTLV(inner[i:])
if err != nil {
break
}
node.Children = append(node.Children, asn1Node(child, childOff+int64(i)))
i += consumed
}
} else {
node.Value = renderPrimitive(t)
}
return node
}
func asn1TagName(t tlv) (string, string) {
if t.Class != 0 {
prefix := []string{"", "APP ", "CTX ", "PRIV "}[t.Class]
return fmt.Sprintf("%s%d", prefix, t.Tag), "tag"
}
names := map[int]string{
1: "BOOLEAN",
2: "INTEGER",
3: "BIT STRING",
4: "OCTET STRING",
5: "NULL",
6: "OBJECT IDENTIFIER",
10: "ENUMERATED",
12: "UTF8String",
16: "SEQUENCE",
17: "SET",
19: "PrintableString",
20: "T61String",
22: "IA5String",
23: "UTCTime",
24: "GeneralizedTime",
}
if n, ok := names[t.Tag]; ok {
return n, strings.ToLower(n)
}
return fmt.Sprintf("tag %d", t.Tag), "tag"
}
func renderPrimitive(t tlv) string {
switch t.Tag {
case 1:
if len(t.Content) == 1 {
if t.Content[0] == 0 {
return "false"
}
return "true"
}
case 2:
return fmt.Sprintf("%s (%d bytes)", bigIntString(t.Content), len(t.Content))
case 3:
if len(t.Content) == 0 {
return "empty"
}
return fmt.Sprintf("unused=%d bits=%x", t.Content[0], t.Content[1:])
case 4:
if isPrintable(t.Content) {
return fmt.Sprintf("%q (%d bytes)", string(t.Content), len(t.Content))
}
return fmt.Sprintf("%x (%d bytes)", t.Content, len(t.Content))
case 5:
return "null"
case 6:
oid, err := parseOID(t.Content)
if err != nil {
return err.Error()
}
return oid + oidNote(oid)
case 12, 19, 20, 22:
return fmt.Sprintf("%q", string(t.Content))
case 23, 24:
return fmt.Sprintf("%q (time)", string(t.Content))
}
if isPrintable(t.Content) {
return fmt.Sprintf("%q", string(t.Content))
}
if len(t.Content) > 32 {
return fmt.Sprintf("%x… (%d bytes)", t.Content[:32], len(t.Content))
}
return fmt.Sprintf("%x", t.Content)
}
func bigIntString(b []byte) string {
if len(b) == 0 {
return "0"
}
if len(b) <= 8 {
var v int64
negative := b[0]&0x80 != 0
for _, c := range b {
v = (v << 8) | int64(c)
}
if negative {
// sign-extend
v -= 1 << uint(8*len(b))
}
return fmt.Sprintf("%d", v)
}
// large — hex is more useful than a decimal approximation.
return fmt.Sprintf("0x%x", b)
}
func parseOID(b []byte) (string, error) {
// Use Go's asn1 parser for correctness.
var oid asn1.ObjectIdentifier
rest, err := asn1.Unmarshal(append([]byte{0x06, byte(len(b))}, b...), &oid)
if err != nil || len(rest) != 0 {
// Fallback manual decode.
return manualOID(b), nil
}
return oid.String(), nil
}
func manualOID(b []byte) string {
if len(b) == 0 {
return ""
}
parts := []string{}
first := int(b[0])
parts = append(parts, fmt.Sprintf("%d.%d", first/40, first%40))
v := 0
for _, c := range b[1:] {
v = (v << 7) | int(c&0x7f)
if c&0x80 == 0 {
parts = append(parts, fmt.Sprintf("%d", v))
v = 0
}
}
return strings.Join(parts, ".")
}
// Known-OID table for human-readable hints in cert dumps.
var oidNames = map[string]string{
"1.2.840.113549.1.1.1": " rsaEncryption",
"1.2.840.113549.1.1.11": " sha256WithRSAEncryption",
"1.2.840.113549.1.1.5": " sha1WithRSAEncryption",
"1.2.840.113549.1.1.13": " sha512WithRSAEncryption",
"1.2.840.10045.2.1": " ecPublicKey",
"1.2.840.10045.4.3.2": " ecdsaWithSHA256",
"1.3.101.112": " Ed25519",
"2.5.4.3": " commonName",
"2.5.4.6": " countryName",
"2.5.4.7": " localityName",
"2.5.4.8": " stateOrProvinceName",
"2.5.4.10": " organizationName",
"2.5.4.11": " organizationalUnitName",
"2.5.4.5": " serialNumber",
"1.2.840.113549.1.9.1": " emailAddress",
"2.5.29.14": " subjectKeyIdentifier",
"2.5.29.15": " keyUsage",
"2.5.29.17": " subjectAltName",
"2.5.29.19": " basicConstraints",
"2.5.29.31": " cRLDistributionPoints",
"2.5.29.32": " certificatePolicies",
"2.5.29.35": " authorityKeyIdentifier",
"2.5.29.37": " extKeyUsage",
"1.3.6.1.5.5.7.3.1": " serverAuth",
"1.3.6.1.5.5.7.3.2": " clientAuth",
"1.3.6.1.5.5.7.1.1": " authorityInfoAccess",
}
func oidNote(oid string) string {
if n, ok := oidNames[oid]; ok {
return n
}
return ""
}
func isPrintable(b []byte) bool {
if len(b) == 0 {
return false
}
for _, c := range b {
if c == 0x09 || c == 0x0a || c == 0x0d {
continue
}
if c < 0x20 || c > 0x7e {
return false
}
}
return true
}
internal/leviathan/parsers/cert.go
+196
View File
@@ -0,0 +1,196 @@
package parsers
import (
"crypto/sha256"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ParseCerts scans the whole document for PEM-encoded certificates
// and for DER-encoded X.509 certificates appearing at arbitrary
// offsets (heuristic: SEQUENCE 0x30 0x82 <len>, then tbsCertificate
// SEQUENCE, then version INTEGER). Returns a list of decoded certs.
func ParseCerts(d *document.Document) (*Field, error) {
size := d.Size()
root := &Field{Name: "Certificates", Type: "container", Offset: 0, Length: size}
// Read whole doc — bounded by caller's judgment. For huge firmware
// blobs this might be expensive but certs are small; most real
// cases are <few MB of text or firmware tables.
if size > 32*1024*1024 {
return nil, errors.New("certs scan skipped: file > 32 MiB")
}
all, err := d.Range(0, size)
if err != nil {
return nil, err
}
// PEM pass.
rest := all
pemCount := 0
var pemOff int64 = 0
for {
block, next := pem.Decode(rest)
if block == nil {
break
}
// Approximate offset: the header "-----BEGIN" will be somewhere in `rest`.
idx := indexOfHeader(rest)
if idx < 0 {
break
}
start := pemOff + int64(idx)
end := pemOff + int64(len(rest)-len(next))
pemCount++
child := &Field{
Name: fmt.Sprintf("PEM %s #%d", block.Type, pemCount),
Type: "pem",
Offset: start,
Length: end - start,
}
if block.Type == "CERTIFICATE" {
if cert, err := x509.ParseCertificate(block.Bytes); err == nil {
populateCert(child, cert)
} else {
child.Note = "x509 parse failed: " + err.Error()
}
}
root.Children = append(root.Children, child)
pemOff += int64(len(rest) - len(next))
rest = next
}
// DER pass — scan for SEQUENCE 0x30 0x82 xx yy (definite long-form, 2 length bytes).
//
// Each candidate runs a full x509.ParseCertificate over a blob up to 64 KiB.
// Without guards a buffer dense with "30 82" byte pairs (random/firmware
// data, or a hostile file) would attempt that parse at nearly every offset
// — O(n) expensive parses, enough to freeze the UI. Two guards keep it
// bounded: a cheap structural pre-filter (a DER certificate body begins
// with the tbsCertificate SEQUENCE, 0x30) rejects almost all false
// candidates without parsing, and a hard cap on parse attempts bounds even
// adversarial input that defeats the pre-filter.
const maxDERAttempts = 1 << 16
derCount := 0
derAttempts := 0
for i := 0; i+4 < len(all); i++ {
if all[i] != 0x30 || all[i+1] != 0x82 {
continue
}
if all[i+4] != 0x30 {
continue
}
totalLen := int(all[i+2])<<8 | int(all[i+3])
if totalLen < 100 || i+4+totalLen > len(all) {
continue
}
if derAttempts++; derAttempts > maxDERAttempts {
break
}
blob := all[i : i+4+totalLen]
cert, err := x509.ParseCertificate(blob)
if err != nil {
continue
}
derCount++
child := &Field{
Name: fmt.Sprintf("DER cert #%d", derCount),
Type: "der",
Offset: int64(i),
Length: int64(4 + totalLen),
}
populateCert(child, cert)
root.Children = append(root.Children, child)
i += 4 + totalLen - 1
}
if len(root.Children) == 0 {
return nil, errors.New("no certificates found")
}
root.Value = fmt.Sprintf("PEM=%d DER=%d", pemCount, derCount)
return root, nil
}
func populateCert(f *Field, c *x509.Certificate) {
f.Value = fmt.Sprintf("subject=%q issuer=%q serial=%x",
c.Subject.String(), c.Issuer.String(), c.SerialNumber)
add := func(name, value string) {
f.Children = append(f.Children, &Field{Name: name, Type: "field", Offset: f.Offset, Length: 0, Value: value})
}
add("subject", c.Subject.String())
add("issuer", c.Issuer.String())
add("serial", fmt.Sprintf("%x", c.SerialNumber))
add("valid_from", c.NotBefore.UTC().Format("2006-01-02 15:04:05 UTC"))
add("valid_until", c.NotAfter.UTC().Format("2006-01-02 15:04:05 UTC"))
add("pubkey_alg", c.PublicKeyAlgorithm.String())
add("sig_alg", c.SignatureAlgorithm.String())
if len(c.DNSNames) > 0 {
add("dns_names", strings.Join(c.DNSNames, ", "))
}
if len(c.IPAddresses) > 0 {
ips := make([]string, 0, len(c.IPAddresses))
for _, ip := range c.IPAddresses {
ips = append(ips, ip.String())
}
add("ip_addresses", strings.Join(ips, ", "))
}
if len(c.EmailAddresses) > 0 {
add("emails", strings.Join(c.EmailAddresses, ", "))
}
if c.IsCA {
add("ca", "true")
}
if c.KeyUsage != 0 {
add("key_usage", fmt.Sprintf("0x%x", int(c.KeyUsage)))
}
if len(c.ExtKeyUsage) > 0 {
parts := make([]string, 0, len(c.ExtKeyUsage))
for _, u := range c.ExtKeyUsage {
parts = append(parts, extKeyUsageName(u))
}
add("ext_key_usage", strings.Join(parts, ", "))
}
// SHA-256 fingerprint of DER
add("sha256_fingerprint", sha256Hex(c.Raw))
}
func extKeyUsageName(u x509.ExtKeyUsage) string {
switch u {
case x509.ExtKeyUsageAny:
return "any"
case x509.ExtKeyUsageServerAuth:
return "serverAuth"
case x509.ExtKeyUsageClientAuth:
return "clientAuth"
case x509.ExtKeyUsageCodeSigning:
return "codeSigning"
case x509.ExtKeyUsageEmailProtection:
return "emailProtection"
case x509.ExtKeyUsageTimeStamping:
return "timeStamping"
case x509.ExtKeyUsageOCSPSigning:
return "OCSPSigning"
}
return fmt.Sprintf("%d", int(u))
}
func sha256Hex(b []byte) string {
h := sha256.Sum256(b)
return fmt.Sprintf("%x", h[:])
}
// indexOfHeader returns the byte offset of the next PEM BEGIN line.
func indexOfHeader(b []byte) int {
const tag = "-----BEGIN"
for i := 0; i+len(tag) <= len(b); i++ {
if string(b[i:i+len(tag)]) == tag {
return i
}
}
return -1
}
internal/leviathan/parsers/executable.go
+412
View File
@@ -0,0 +1,412 @@
package parsers
import (
"encoding/binary"
"errors"
"fmt"
"sort"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ExecutableRegion is a file-backed executable code range.
type ExecutableRegion struct {
Name string `json:"name"`
Offset int64 `json:"offset"`
Size int64 `json:"size"`
Address uint64 `json:"address"`
}
// FunctionSeed is a named function entry mapped to a file offset.
type FunctionSeed struct {
Name string `json:"name"`
Offset int64 `json:"offset"`
Address uint64 `json:"address"`
Size uint64 `json:"size"`
Source string `json:"source"`
}
// ExecutableInfo contains disassembly-oriented metadata for an ELF or PE.
type ExecutableInfo struct {
Format string `json:"format"`
Arch string `json:"arch"`
Entry int64 `json:"entry"`
Regions []ExecutableRegion `json:"regions"`
Seeds []FunctionSeed `json:"seeds"`
}
// InspectExecutable extracts executable ranges, entry point and named function
// seeds without scraping the display-oriented parser tree.
func InspectExecutable(d *document.Document) (*ExecutableInfo, error) {
head, err := d.Range(0, min64(64, d.Size()))
if err != nil {
return nil, err
}
if len(head) >= 4 && string(head[:4]) == "\x7fELF" {
return inspectELFExecutable(d, head)
}
if len(head) >= 2 && string(head[:2]) == "MZ" {
return inspectPEExecutable(d)
}
return nil, errors.New("not a supported ELF or PE executable")
}
func inspectELFExecutable(d *document.Document, hdr []byte) (*ExecutableInfo, error) {
if len(hdr) < 52 || (hdr[4] != 1 && hdr[4] != 2) || (hdr[5] != 1 && hdr[5] != 2) {
return nil, errors.New("invalid ELF header")
}
class, data := hdr[4], hdr[5]
bo := elfByteOrder(data)
var machine uint16
var entry, phoff, shoff uint64
var phentsize, phnum, shentsize, shnum uint16
if class == 2 {
if len(hdr) < 64 {
return nil, errors.New("ELF64 header too short")
}
machine = bo.Uint16(hdr[18:20])
entry = bo.Uint64(hdr[24:32])
phoff = bo.Uint64(hdr[32:40])
shoff = bo.Uint64(hdr[40:48])
phentsize, phnum = bo.Uint16(hdr[54:56]), bo.Uint16(hdr[56:58])
shentsize, shnum = bo.Uint16(hdr[58:60]), bo.Uint16(hdr[60:62])
} else {
machine = bo.Uint16(hdr[18:20])
entry = uint64(bo.Uint32(hdr[24:28]))
phoff = uint64(bo.Uint32(hdr[28:32]))
shoff = uint64(bo.Uint32(hdr[32:36]))
phentsize, phnum = bo.Uint16(hdr[42:44]), bo.Uint16(hdr[44:46])
shentsize, shnum = bo.Uint16(hdr[46:48]), bo.Uint16(hdr[48:50])
}
sections, _ := executableELFSections(d, bo, class, shoff, shentsize, shnum)
info := &ExecutableInfo{Format: "elf", Arch: elfDisasmArch(machine, data), Entry: -1}
if info.Arch == "" {
return nil, fmt.Errorf("unsupported ELF machine 0x%x", machine)
}
for i, section := range sections {
if section.flags&0x4 != 0 && validFileRange(d, section.fileOff, section.secSize) {
info.Regions = append(info.Regions, ExecutableRegion{
Name: fmt.Sprintf("section[%d]", i), Offset: int64(section.fileOff),
Size: int64(section.secSize), Address: section.addr,
})
}
}
if len(info.Regions) == 0 {
info.Regions = executableELFSegments(d, bo, class, phoff, phentsize, phnum)
}
if off, ok := elfAddressToOffset(sections, entry); ok {
info.Entry = off
info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: off, Address: entry, Source: "header"})
} else if off, ok := regionAddressToOffset(info.Regions, entry); ok {
info.Entry = off
info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: off, Address: entry, Source: "header"})
}
for _, symtab := range sections {
if symtab.typ == 2 || symtab.typ == 11 {
info.Seeds = append(info.Seeds, executableELFFunctions(d, bo, class, sections, symtab)...)
}
}
finishExecutableInfo(info)
return info, nil
}
func executableELFSegments(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entrySize, count uint16) []ExecutableRegion {
if off == 0 || entrySize == 0 || count == 0 || count > 8192 {
return nil
}
regions := make([]ExecutableRegion, 0)
for i := uint16(0); i < count; i++ {
buf, err := d.Range(int64(off)+int64(i)*int64(entrySize), int64(entrySize))
if err != nil {
break
}
var typ, flags uint32
var fileOff, address, fileSize uint64
if class == 2 {
if len(buf) < 56 {
break
}
typ, flags = bo.Uint32(buf[0:4]), bo.Uint32(buf[4:8])
fileOff, address, fileSize = bo.Uint64(buf[8:16]), bo.Uint64(buf[16:24]), bo.Uint64(buf[32:40])
} else {
if len(buf) < 32 {
break
}
typ, flags = bo.Uint32(buf[0:4]), bo.Uint32(buf[24:28])
fileOff, address, fileSize = uint64(bo.Uint32(buf[4:8])), uint64(bo.Uint32(buf[8:12])), uint64(bo.Uint32(buf[16:20]))
}
if typ == 1 && flags&0x1 != 0 && validFileRange(d, fileOff, fileSize) {
regions = append(regions, ExecutableRegion{
Name: fmt.Sprintf("PT_LOAD[%d]", i), Offset: int64(fileOff), Size: int64(fileSize), Address: address,
})
}
}
return regions
}
func executableELFSections(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entrySize, count uint16) ([]elfSectionInfo, error) {
if off == 0 || entrySize == 0 || count == 0 || count > 8192 {
return nil, errors.New("ELF has no usable section table")
}
sections := make([]elfSectionInfo, 0, count)
for i := uint16(0); i < count; i++ {
buf, err := d.Range(int64(off)+int64(i)*int64(entrySize), int64(entrySize))
if err != nil {
return nil, err
}
var section elfSectionInfo
if class == 2 {
if len(buf) < 64 {
return nil, errors.New("truncated ELF64 section")
}
section.typ, section.flags = bo.Uint32(buf[4:8]), bo.Uint64(buf[8:16])
section.addr, section.fileOff, section.secSize = bo.Uint64(buf[16:24]), bo.Uint64(buf[24:32]), bo.Uint64(buf[32:40])
section.link, section.entrySize = bo.Uint32(buf[40:44]), bo.Uint64(buf[56:64])
} else {
if len(buf) < 40 {
return nil, errors.New("truncated ELF32 section")
}
section.typ, section.flags = bo.Uint32(buf[4:8]), uint64(bo.Uint32(buf[8:12]))
section.addr = uint64(bo.Uint32(buf[12:16]))
section.fileOff, section.secSize = uint64(bo.Uint32(buf[16:20])), uint64(bo.Uint32(buf[20:24]))
section.link, section.entrySize = bo.Uint32(buf[24:28]), uint64(bo.Uint32(buf[36:40]))
}
sections = append(sections, section)
}
return sections, nil
}
func executableELFFunctions(d *document.Document, bo binary.ByteOrder, class byte, sections []elfSectionInfo, symtab elfSectionInfo) []FunctionSeed {
entrySize := symtab.entrySize
if entrySize == 0 {
entrySize = 16
if class == 2 {
entrySize = 24
}
}
if !validFileRange(d, symtab.fileOff, symtab.secSize) || int(symtab.link) >= len(sections) {
return nil
}
strsec := sections[symtab.link]
if !validFileRange(d, strsec.fileOff, strsec.secSize) || strsec.secSize > 8<<20 {
return nil
}
strtab, err := d.Range(int64(strsec.fileOff), int64(strsec.secSize))
if err != nil {
return nil
}
count := int(symtab.secSize / entrySize)
if count > 16384 {
count = 16384
}
out := make([]FunctionSeed, 0)
for i := 0; i < count; i++ {
buf, err := d.Range(int64(symtab.fileOff)+int64(i)*int64(entrySize), int64(entrySize))
if err != nil || len(buf) < int(entrySize) {
break
}
var nameOff uint32
var info byte
var sectionIndex uint16
var value, size uint64
if class == 2 {
nameOff, info, sectionIndex = bo.Uint32(buf[0:4]), buf[4], bo.Uint16(buf[6:8])
value, size = bo.Uint64(buf[8:16]), bo.Uint64(buf[16:24])
} else {
nameOff, value, size = bo.Uint32(buf[0:4]), uint64(bo.Uint32(buf[4:8])), uint64(bo.Uint32(buf[8:12]))
info, sectionIndex = buf[12], bo.Uint16(buf[14:16])
}
if info&0xf != 2 || sectionIndex == 0 || value == 0 {
continue
}
name := elfStringAt(strtab, nameOff)
offset, ok := elfAddressToOffset(sections, value)
if name == "" || !ok {
continue
}
source := "symtab"
if symtab.typ == 11 {
source = "dynsym"
}
out = append(out, FunctionSeed{Name: name, Offset: offset, Address: value, Size: size, Source: source})
}
return out
}
func inspectPEExecutable(d *document.Document) (*ExecutableInfo, error) {
sections, dirs, is64, err := peLayout(d)
if err != nil {
return nil, err
}
hdr, _ := d.Range(0, 64)
lfanew := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40]))
peHdr, _ := d.Range(lfanew, 24)
machine := binary.LittleEndian.Uint16(peHdr[4:6])
optSize := int64(binary.LittleEndian.Uint16(peHdr[20:22]))
opt, err := d.Range(lfanew+24, optSize)
if err != nil || len(opt) < 20 {
return nil, errors.New("PE optional header too short")
}
entryRVA := binary.LittleEndian.Uint32(opt[16:20])
info := &ExecutableInfo{Format: "pe", Arch: peDisasmArch(machine, is64), Entry: -1}
if info.Arch == "" {
return nil, fmt.Errorf("unsupported PE machine 0x%x", machine)
}
for _, section := range sections {
if section.Characteristics&0x20000000 != 0 && validFileRange(d, uint64(section.RawOffset), uint64(section.RawSize)) {
info.Regions = append(info.Regions, ExecutableRegion{
Name: section.Name, Offset: int64(section.RawOffset), Size: int64(section.RawSize), Address: uint64(section.VirtualAddress),
})
}
}
if off, ok := peRVAToOffset(sections, entryRVA); ok {
info.Entry = int64(off)
info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: int64(off), Address: uint64(entryRVA), Source: "header"})
}
if len(dirs) > 0 && dirs[0].RVA != 0 {
info.Seeds = append(info.Seeds, executablePEExports(d, sections, dirs[0])...)
}
finishExecutableInfo(info)
return info, nil
}
func executablePEExports(d *document.Document, sections []peSectionInfo, dir peDataDirectory) []FunctionSeed {
off, ok := peRVAToOffset(sections, dir.RVA)
if !ok {
return nil
}
buf, err := d.Range(int64(off), 40)
if err != nil || len(buf) < 40 {
return nil
}
functionCount, nameCount := binary.LittleEndian.Uint32(buf[20:24]), binary.LittleEndian.Uint32(buf[24:28])
functionsOff, funcsOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[28:32]))
namesOff, namesOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[32:36]))
ordinalsOff, ordOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[36:40]))
if !funcsOK || !namesOK || !ordOK {
return nil
}
if nameCount > 16384 {
nameCount = 16384
}
out := make([]FunctionSeed, 0, nameCount)
for i := uint32(0); i < nameCount; i++ {
namePtr, nameErr := d.Range(int64(namesOff)+int64(i)*4, 4)
ordBuf, ordErr := d.Range(int64(ordinalsOff)+int64(i)*2, 2)
if nameErr != nil || ordErr != nil || len(namePtr) < 4 || len(ordBuf) < 2 {
break
}
index := uint32(binary.LittleEndian.Uint16(ordBuf))
if index >= functionCount {
continue
}
funcBuf, err := d.Range(int64(functionsOff)+int64(index)*4, 4)
if err != nil || len(funcBuf) < 4 {
break
}
funcRVA := binary.LittleEndian.Uint32(funcBuf)
if funcRVA >= dir.RVA && funcRVA < dir.RVA+dir.Size {
continue
}
nameOff, nameOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(namePtr))
funcOff, funcOK := peRVAToOffset(sections, funcRVA)
if !nameOK || !funcOK {
continue
}
name := readCString(d, int64(nameOff), 512)
if name != "" {
out = append(out, FunctionSeed{Name: name, Offset: int64(funcOff), Address: uint64(funcRVA), Source: "export"})
}
}
return out
}
func validFileRange(d *document.Document, offset, size uint64) bool {
return size > 0 && offset <= uint64(d.Size()) && size <= uint64(d.Size())-offset
}
func elfAddressToOffset(sections []elfSectionInfo, address uint64) (int64, bool) {
for _, section := range sections {
if section.secSize > 0 && address >= section.addr && address < section.addr+section.secSize {
return int64(section.fileOff + address - section.addr), true
}
}
return 0, false
}
func regionAddressToOffset(regions []ExecutableRegion, address uint64) (int64, bool) {
for _, region := range regions {
if address >= region.Address && address < region.Address+uint64(region.Size) {
return region.Offset + int64(address-region.Address), true
}
}
return 0, false
}
func elfDisasmArch(machine uint16, data byte) string {
switch machine {
case 3:
return "x86"
case 8:
if data == 1 {
return "mipsle"
}
return "mips"
case 40:
return "arm32"
case 62:
return "x86_64"
case 183:
return "aarch64"
}
return ""
}
func peDisasmArch(machine uint16, _ bool) string {
switch machine {
case 0x14c:
return "x86"
case 0x8664:
return "x86_64"
case 0x1c0, 0x1c4:
return "arm32"
case 0xaa64:
return "aarch64"
case 0x166, 0x266, 0x366, 0x466:
return "mipsle"
}
return ""
}
func finishExecutableInfo(info *ExecutableInfo) {
sort.Slice(info.Regions, func(i, j int) bool { return info.Regions[i].Offset < info.Regions[j].Offset })
sort.Slice(info.Seeds, func(i, j int) bool {
if info.Seeds[i].Offset != info.Seeds[j].Offset {
return info.Seeds[i].Offset < info.Seeds[j].Offset
}
return info.Seeds[i].Name < info.Seeds[j].Name
})
unique := info.Seeds[:0]
for _, seed := range info.Seeds {
if seed.Offset < 0 || !offsetInRegions(info.Regions, seed.Offset) {
continue
}
if len(unique) > 0 && unique[len(unique)-1].Offset == seed.Offset {
if unique[len(unique)-1].Name == "entry" && seed.Name != "entry" {
unique[len(unique)-1] = seed
}
continue
}
unique = append(unique, seed)
}
info.Seeds = unique
}
func offsetInRegions(regions []ExecutableRegion, offset int64) bool {
for _, region := range regions {
if offset >= region.Offset && offset < region.Offset+region.Size {
return true
}
}
return false
}
internal/leviathan/parsers/formats.go
+2182
View File
File diff suppressed because it is too large Load Diff
internal/leviathan/parsers/formats_more.go
+662
View File
@@ -0,0 +1,662 @@
// Additional structure parsers that round out the structures panel into a
// forensic-grade set: gzip, tar, SQLite, TIFF, pcap/pcapng and the OLE/CFB
// compound-file container used by legacy Office documents and MSI installers.
//
// Each parser follows the same contract as the parsers in formats.go: it
// returns a *Field tree anchored at absolute document offsets, never panics on
// truncated input, and bounds every loop so a hostile file cannot wedge the UI.
package parsers
import (
"encoding/binary"
"errors"
"fmt"
"strconv"
"strings"
"time"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// leaf builds a terminal Field with no children.
func leaf(name, typ string, off, length int64, value string) *Field {
return &Field{Name: name, Type: typ, Offset: off, Length: length, Value: value}
}
// ---------- gzip (RFC 1952) ----------
// ParseGZIP decodes the gzip member header, optional name/comment/extra fields
// and the trailing CRC-32 / ISIZE footer.
func ParseGZIP(d *document.Document) (*Field, error) {
size := d.Size()
hdr, err := d.Range(0, min64(10, size))
if err != nil || len(hdr) < 10 {
return nil, errors.New("gzip too short")
}
if hdr[0] != 0x1f || hdr[1] != 0x8b {
return nil, errors.New("not gzip")
}
root := &Field{Name: "GZIP", Type: "container", Offset: 0, Length: size}
method := hdr[2]
methodName := "deflate"
if method != 8 {
methodName = fmt.Sprintf("unknown(%d)", method)
}
flg := hdr[3]
mtime := binary.LittleEndian.Uint32(hdr[4:8])
root.Children = append(root.Children,
leaf("id", "bytes", 0, 2, "\\x1f\\x8b"),
leaf("method", "u8", 2, 1, methodName),
leaf("flags", "u8", 3, 1, gzipFlags(flg)),
leaf("mtime", "u32le", 4, 4, gzipTime(mtime)),
leaf("extra_flags", "u8", 8, 1, fmt.Sprintf("%d", hdr[8])),
leaf("os", "u8", 9, 1, gzipOS(hdr[9])),
)
off := int64(10)
// FEXTRA
if flg&0x04 != 0 && off+2 <= size {
xb, _ := d.Range(off, 2)
xlen := int64(binary.LittleEndian.Uint16(xb))
root.Children = append(root.Children, leaf("extra", "bytes", off, 2+xlen, fmt.Sprintf("%d bytes", xlen)))
off += 2 + xlen
}
// FNAME
if flg&0x08 != 0 {
s, n := readCStringField(d, off, size)
root.Children = append(root.Children, leaf("name", "asciiz", off, n, s))
off += n
}
// FCOMMENT
if flg&0x10 != 0 {
s, n := readCStringField(d, off, size)
root.Children = append(root.Children, leaf("comment", "asciiz", off, n, s))
off += n
}
// FHCRC
if flg&0x02 != 0 && off+2 <= size {
hb, _ := d.Range(off, 2)
root.Children = append(root.Children, leaf("header_crc16", "u16le", off, 2, fmt.Sprintf("0x%04x", binary.LittleEndian.Uint16(hb))))
off += 2
}
if size >= off+8 {
root.Children = append(root.Children, leaf("compressed_data", "bytes", off, size-off-8, fmt.Sprintf("%d bytes", size-off-8)))
ft, _ := d.Range(size-8, 8)
root.Children = append(root.Children,
leaf("crc32", "u32le", size-8, 4, fmt.Sprintf("0x%08x", binary.LittleEndian.Uint32(ft[0:4]))),
leaf("isize", "u32le", size-4, 4, fmt.Sprintf("%d bytes (mod 2^32)", binary.LittleEndian.Uint32(ft[4:8]))),
)
}
return root, nil
}
func gzipFlags(f byte) string {
var on []string
for _, b := range []struct {
mask byte
name string
}{{0x01, "TEXT"}, {0x02, "HCRC"}, {0x04, "EXTRA"}, {0x08, "NAME"}, {0x10, "COMMENT"}} {
if f&b.mask != 0 {
on = append(on, b.name)
}
}
if len(on) == 0 {
return fmt.Sprintf("0x%02x", f)
}
return fmt.Sprintf("0x%02x (%s)", f, strings.Join(on, "|"))
}
func gzipOS(b byte) string {
names := map[byte]string{0: "FAT", 1: "Amiga", 2: "VMS", 3: "Unix", 6: "HPFS", 7: "Macintosh", 11: "NTFS", 255: "unknown"}
if n, ok := names[b]; ok {
return fmt.Sprintf("%d (%s)", b, n)
}
return fmt.Sprintf("%d", b)
}
func gzipTime(t uint32) string {
if t == 0 {
return "0 (no timestamp)"
}
return fmt.Sprintf("%d (%s)", t, time.Unix(int64(t), 0).UTC().Format("2006-01-02 15:04:05 UTC"))
}
// ---------- tar (POSIX ustar) ----------
// ParseTAR walks the 512-byte record headers of a ustar/GNU tar archive.
func ParseTAR(d *document.Document) (*Field, error) {
size := d.Size()
root := &Field{Name: "TAR", Type: "container", Offset: 0}
off := int64(0)
entries := 0
for off+512 <= size && entries < 10000 {
blk, err := d.Range(off, 512)
if err != nil {
break
}
if allZero(blk) {
root.Children = append(root.Children, leaf("end_of_archive", "bytes", off, 512, "zero block"))
off += 512
break
}
name := trimZeroStr(blk[0:100])
prefix := trimZeroStr(blk[345:500])
full := name
if prefix != "" {
full = prefix + "/" + name
}
fsize := octal(blk[124:136])
mtime := octal(blk[136:148])
typ := blk[156]
e := &Field{
Name: full, Type: "entry", Offset: off,
Length: 512 + roundUp(fsize, 512),
Value: fmt.Sprintf("%s size=%d", tarType(typ), fsize),
Children: []*Field{
leaf("name", "asciiz", off, 100, name),
leaf("mode", "octal", off+100, 8, fmt.Sprintf("0%o", octal(blk[100:108]))),
leaf("uid", "octal", off+108, 8, fmt.Sprintf("%d", octal(blk[108:116]))),
leaf("gid", "octal", off+116, 8, fmt.Sprintf("%d", octal(blk[116:124]))),
leaf("size", "octal", off+124, 12, fmt.Sprintf("%d", fsize)),
leaf("mtime", "octal", off+136, 12, tarTime(mtime)),
leaf("typeflag", "u8", off+156, 1, tarType(typ)),
leaf("magic", "ascii", off+257, 8, trimZeroStr(blk[257:265])),
leaf("uname", "asciiz", off+265, 32, trimZeroStr(blk[265:297])),
leaf("gname", "asciiz", off+297, 32, trimZeroStr(blk[297:329])),
},
}
if prefix != "" {
e.Children = append(e.Children, leaf("prefix", "asciiz", off+345, 155, prefix))
}
root.Children = append(root.Children, e)
off += 512 + roundUp(fsize, 512)
entries++
}
root.Length = off
if len(root.Children) == 0 {
return nil, errors.New("no tar entries")
}
return root, nil
}
func tarType(t byte) string {
switch t {
case 0, '0':
return "file"
case '1':
return "hardlink"
case '2':
return "symlink"
case '3':
return "char-device"
case '4':
return "block-device"
case '5':
return "directory"
case '6':
return "fifo"
case '7':
return "contiguous"
case 'x':
return "pax-header"
case 'g':
return "pax-global"
case 'L':
return "gnu-longname"
case 'K':
return "gnu-longlink"
default:
return fmt.Sprintf("type-%q", t)
}
}
func tarTime(t int64) string {
if t == 0 {
return "0"
}
return fmt.Sprintf("%d (%s)", t, time.Unix(t, 0).UTC().Format("2006-01-02 15:04:05 UTC"))
}
// ---------- SQLite 3 ----------
// ParseSQLite decodes the 100-byte SQLite database header.
func ParseSQLite(d *document.Document) (*Field, error) {
hdr, err := d.Range(0, min64(100, d.Size()))
if err != nil || len(hdr) < 100 {
return nil, errors.New("sqlite header too short")
}
if string(hdr[0:16]) != "SQLite format 3\x00" {
return nil, errors.New("not a SQLite database")
}
be := binary.BigEndian
pageSize := int64(be.Uint16(hdr[16:18]))
if pageSize == 1 {
pageSize = 65536
}
enc := map[uint32]string{1: "UTF-8", 2: "UTF-16le", 3: "UTF-16be"}[be.Uint32(hdr[56:60])]
if enc == "" {
enc = "unknown"
}
root := &Field{
Name: "SQLite", Type: "container", Offset: 0, Length: d.Size(),
Value: fmt.Sprintf("page=%d pages=%d %s", pageSize, be.Uint32(hdr[28:32]), enc),
Children: []*Field{
leaf("magic", "ascii", 0, 16, "SQLite format 3"),
leaf("page_size", "u16be", 16, 2, fmt.Sprintf("%d", pageSize)),
leaf("write_version", "u8", 18, 1, sqliteFmtVer(hdr[18])),
leaf("read_version", "u8", 19, 1, sqliteFmtVer(hdr[19])),
leaf("reserved_space", "u8", 20, 1, fmt.Sprintf("%d", hdr[20])),
leaf("change_counter", "u32be", 24, 4, fmt.Sprintf("%d", be.Uint32(hdr[24:28]))),
leaf("page_count", "u32be", 28, 4, fmt.Sprintf("%d", be.Uint32(hdr[28:32]))),
leaf("freelist_head", "u32be", 32, 4, fmt.Sprintf("%d", be.Uint32(hdr[32:36]))),
leaf("freelist_pages", "u32be", 36, 4, fmt.Sprintf("%d", be.Uint32(hdr[36:40]))),
leaf("schema_cookie", "u32be", 40, 4, fmt.Sprintf("%d", be.Uint32(hdr[40:44]))),
leaf("schema_format", "u32be", 44, 4, fmt.Sprintf("%d", be.Uint32(hdr[44:48]))),
leaf("text_encoding", "u32be", 56, 4, enc),
leaf("user_version", "u32be", 60, 4, fmt.Sprintf("%d", be.Uint32(hdr[60:64]))),
leaf("vacuum_mode", "u32be", 64, 4, fmt.Sprintf("%d", be.Uint32(hdr[64:68]))),
leaf("application_id", "u32be", 68, 4, fmt.Sprintf("0x%08x", be.Uint32(hdr[68:72]))),
leaf("version_valid_for", "u32be", 92, 4, fmt.Sprintf("%d", be.Uint32(hdr[92:96]))),
leaf("sqlite_version", "u32be", 96, 4, sqliteVersion(be.Uint32(hdr[96:100]))),
},
}
return root, nil
}
func sqliteFmtVer(b byte) string {
switch b {
case 1:
return "1 (legacy/rollback)"
case 2:
return "2 (WAL)"
default:
return fmt.Sprintf("%d", b)
}
}
func sqliteVersion(v uint32) string {
// SQLITE_VERSION_NUMBER = major*1_000_000 + minor*1000 + patch.
return fmt.Sprintf("%d (%d.%d.%d)", v, v/1000000, (v/1000)%1000, v%1000)
}
// ---------- TIFF ----------
// ParseTIFF decodes the TIFF header and the entries of each image file
// directory (IFD), following the next-IFD chain.
func ParseTIFF(d *document.Document) (*Field, error) {
size := d.Size()
hdr, err := d.Range(0, min64(8, size))
if err != nil || len(hdr) < 8 {
return nil, errors.New("tiff too short")
}
var bo binary.ByteOrder
switch string(hdr[0:2]) {
case "II":
bo = binary.LittleEndian
case "MM":
bo = binary.BigEndian
default:
return nil, errors.New("not TIFF")
}
if bo.Uint16(hdr[2:4]) != 42 {
return nil, errors.New("bad TIFF magic")
}
order := "little-endian"
if string(hdr[0:2]) == "MM" {
order = "big-endian"
}
root := &Field{
Name: "TIFF", Type: "container", Offset: 0, Length: size, Value: order,
Children: []*Field{
leaf("byte_order", "ascii", 0, 2, order),
leaf("magic", "u16", 2, 2, "42"),
leaf("first_ifd", "u32", 4, 4, fmt.Sprintf("0x%x", bo.Uint32(hdr[4:8]))),
},
}
ifdOff := int64(bo.Uint32(hdr[4:8]))
for n := 0; ifdOff > 0 && ifdOff+2 <= size && n < 64; n++ {
cb, err := d.Range(ifdOff, 2)
if err != nil {
break
}
count := int64(bo.Uint16(cb))
ifd := &Field{
Name: fmt.Sprintf("IFD[%d]", n), Type: "directory", Offset: ifdOff,
Length: 2 + count*12 + 4, Value: fmt.Sprintf("%d entries", count),
}
for i := int64(0); i < count && i < 4096; i++ {
eoff := ifdOff + 2 + i*12
eb, err := d.Range(eoff, 12)
if err != nil || len(eb) < 12 {
break
}
tag := bo.Uint16(eb[0:2])
typ := bo.Uint16(eb[2:4])
cnt := bo.Uint32(eb[4:8])
val := bo.Uint32(eb[8:12])
ifd.Children = append(ifd.Children, leaf(
fmt.Sprintf("tag 0x%04x %s", tag, tiffTag(tag)), "ifd_entry", eoff, 12,
fmt.Sprintf("type=%s count=%d value=%d", tiffType(typ), cnt, val)))
}
root.Children = append(root.Children, ifd)
nb, err := d.Range(ifdOff+2+count*12, 4)
if err != nil || len(nb) < 4 {
break
}
ifdOff = int64(bo.Uint32(nb))
}
return root, nil
}
func tiffType(t uint16) string {
names := map[uint16]string{1: "BYTE", 2: "ASCII", 3: "SHORT", 4: "LONG", 5: "RATIONAL", 6: "SBYTE", 7: "UNDEFINED", 8: "SSHORT", 9: "SLONG", 10: "SRATIONAL", 11: "FLOAT", 12: "DOUBLE"}
if n, ok := names[t]; ok {
return n
}
return fmt.Sprintf("type%d", t)
}
func tiffTag(t uint16) string {
names := map[uint16]string{
256: "ImageWidth", 257: "ImageLength", 258: "BitsPerSample", 259: "Compression",
262: "PhotometricInterpretation", 273: "StripOffsets", 277: "SamplesPerPixel",
278: "RowsPerStrip", 279: "StripByteCounts", 282: "XResolution", 283: "YResolution",
296: "ResolutionUnit", 305: "Software", 306: "DateTime", 315: "Artist",
271: "Make", 272: "Model", 274: "Orientation", 34665: "ExifIFD", 34853: "GPSIFD",
}
if n, ok := names[t]; ok {
return n
}
return ""
}
// ---------- pcap / pcapng ----------
// ParsePCAP decodes a classic libpcap global header and walks packet records.
func ParsePCAP(d *document.Document) (*Field, error) {
size := d.Size()
hdr, err := d.Range(0, min64(24, size))
if err != nil || len(hdr) < 24 {
return nil, errors.New("pcap too short")
}
magic := binary.LittleEndian.Uint32(hdr[0:4])
var bo binary.ByteOrder
nano := false
switch magic {
case 0xa1b2c3d4:
bo = binary.LittleEndian
case 0xd4c3b2a1:
bo = binary.BigEndian
case 0xa1b23c4d:
bo, nano = binary.LittleEndian, true
case 0x4d3cb2a1:
bo, nano = binary.BigEndian, true
default:
return nil, errors.New("not pcap")
}
res := "microsecond"
if nano {
res = "nanosecond"
}
linktype := bo.Uint32(hdr[20:24])
root := &Field{
Name: "PCAP", Type: "container", Offset: 0, Length: size,
Value: fmt.Sprintf("%s, %s", res, pcapLinkType(linktype)),
Children: []*Field{
leaf("magic", "u32", 0, 4, fmt.Sprintf("0x%08x (%s)", magic, res)),
leaf("version", "u16", 4, 4, fmt.Sprintf("%d.%d", bo.Uint16(hdr[4:6]), bo.Uint16(hdr[6:8]))),
leaf("thiszone", "i32", 8, 4, fmt.Sprintf("%d", int32(bo.Uint32(hdr[8:12])))),
leaf("sigfigs", "u32", 12, 4, fmt.Sprintf("%d", bo.Uint32(hdr[12:16]))),
leaf("snaplen", "u32", 16, 4, fmt.Sprintf("%d", bo.Uint32(hdr[16:20]))),
leaf("linktype", "u32", 20, 4, fmt.Sprintf("%d (%s)", linktype, pcapLinkType(linktype))),
},
}
off := int64(24)
for n := 0; off+16 <= size && n < 100000; n++ {
rb, err := d.Range(off, 16)
if err != nil {
break
}
ts := bo.Uint32(rb[0:4])
us := bo.Uint32(rb[4:8])
inclLen := int64(bo.Uint32(rb[8:12]))
origLen := bo.Uint32(rb[12:16])
root.Children = append(root.Children, leaf(
fmt.Sprintf("packet[%d]", n), "record", off, 16+inclLen,
fmt.Sprintf("ts=%d.%06d incl=%d orig=%d", ts, us, inclLen, origLen)))
off += 16 + inclLen
}
return root, nil
}
// ParsePCAPNG decodes the pcapng block structure (section header + block walk).
func ParsePCAPNG(d *document.Document) (*Field, error) {
size := d.Size()
hb, err := d.Range(0, min64(12, size))
if err != nil || len(hb) < 12 {
return nil, errors.New("pcapng too short")
}
if binary.LittleEndian.Uint32(hb[0:4]) != 0x0a0d0d0a {
return nil, errors.New("not pcapng")
}
// Byte-order magic lives in the SHB body at offset 8.
var bo binary.ByteOrder
switch binary.LittleEndian.Uint32(hb[8:12]) {
case 0x1a2b3c4d:
bo = binary.LittleEndian
case 0x4d3c2b1a:
bo = binary.BigEndian
default:
return nil, errors.New("bad pcapng byte-order magic")
}
root := &Field{Name: "PCAPNG", Type: "container", Offset: 0, Length: size}
off := int64(0)
for n := 0; off+12 <= size && n < 100000; n++ {
bh, err := d.Range(off, 12)
if err != nil {
break
}
btype := bo.Uint32(bh[0:4])
blen := int64(bo.Uint32(bh[4:8]))
if blen < 12 || off+blen > size {
break
}
root.Children = append(root.Children, leaf(
fmt.Sprintf("block[%d] %s", n, pcapngBlock(btype)), "block", off, blen,
fmt.Sprintf("type=0x%08x length=%d", btype, blen)))
off += blen
}
return root, nil
}
func pcapngBlock(t uint32) string {
switch t {
case 0x0a0d0d0a:
return "SectionHeader"
case 0x00000001:
return "InterfaceDescription"
case 0x00000003:
return "SimplePacket"
case 0x00000004:
return "NameResolution"
case 0x00000005:
return "InterfaceStatistics"
case 0x00000006:
return "EnhancedPacket"
default:
return "block"
}
}
func pcapLinkType(t uint32) string {
names := map[uint32]string{0: "NULL", 1: "ETHERNET", 6: "TOKEN_RING", 9: "PPP", 101: "RAW", 105: "IEEE802_11", 113: "LINUX_SLL", 127: "IEEE802_11_RADIOTAP", 228: "IPV4", 229: "IPV6", 276: "LINUX_SLL2"}
if n, ok := names[t]; ok {
return n
}
return fmt.Sprintf("linktype%d", t)
}
// ---------- OLE / Compound File Binary (MS-CFB) ----------
var cfbMagic = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1}
// ParseCFB decodes the compound-file header and lists the entries in the first
// directory sector (legacy .doc/.xls/.ppt/.msi container).
func ParseCFB(d *document.Document) (*Field, error) {
size := d.Size()
hdr, err := d.Range(0, min64(512, size))
if err != nil || len(hdr) < 76 {
return nil, errors.New("cfb too short")
}
if string(hdr[0:8]) != string(cfbMagic) {
return nil, errors.New("not a compound file")
}
le := binary.LittleEndian
major := le.Uint16(hdr[26:28])
sectorShift := le.Uint16(hdr[30:32])
sectorSize := int64(1) << sectorShift
firstDir := le.Uint32(hdr[48:52])
root := &Field{
Name: "CFB/OLE", Type: "container", Offset: 0, Length: size,
Value: fmt.Sprintf("v%d.%d sector=%d", major, le.Uint16(hdr[24:26]), sectorSize),
Children: []*Field{
leaf("signature", "bytes", 0, 8, "D0 CF 11 E0 A1 B1 1A E1"),
leaf("minor_version", "u16le", 24, 2, fmt.Sprintf("%d", le.Uint16(hdr[24:26]))),
leaf("major_version", "u16le", 26, 2, fmt.Sprintf("%d", major)),
leaf("byte_order", "u16le", 28, 2, fmt.Sprintf("0x%04x", le.Uint16(hdr[28:30]))),
leaf("sector_shift", "u16le", 30, 2, fmt.Sprintf("%d (%d bytes)", sectorShift, sectorSize)),
leaf("mini_sector_shift", "u16le", 32, 2, fmt.Sprintf("%d (%d bytes)", le.Uint16(hdr[32:34]), int64(1)<<le.Uint16(hdr[32:34]))),
leaf("num_dir_sectors", "u32le", 40, 4, fmt.Sprintf("%d", le.Uint32(hdr[40:44]))),
leaf("num_fat_sectors", "u32le", 44, 4, fmt.Sprintf("%d", le.Uint32(hdr[44:48]))),
leaf("first_dir_sector", "u32le", 48, 4, fmt.Sprintf("%d", firstDir)),
leaf("mini_stream_cutoff", "u32le", 56, 4, fmt.Sprintf("%d", le.Uint32(hdr[56:60]))),
leaf("first_minifat_sector", "u32le", 60, 4, fmt.Sprintf("%d", le.Uint32(hdr[60:64]))),
leaf("num_minifat_sectors", "u32le", 64, 4, fmt.Sprintf("%d", le.Uint32(hdr[64:68]))),
leaf("first_difat_sector", "u32le", 68, 4, fmt.Sprintf("%d", le.Uint32(hdr[68:72]))),
leaf("num_difat_sectors", "u32le", 72, 4, fmt.Sprintf("%d", le.Uint32(hdr[72:76]))),
},
}
// First directory sector holds up to sectorSize/128 directory entries.
dirOff := int64(firstDir+1) * sectorSize
if dir := cfbDirectory(d, dirOff, sectorSize, size); dir != nil {
root.Children = append(root.Children, dir)
}
return root, nil
}
func cfbDirectory(d *document.Document, off, sectorSize, size int64) *Field {
if off <= 0 || off+128 > size {
return nil
}
dir := &Field{Name: "directory", Type: "directory", Offset: off}
per := sectorSize / 128
count := int64(0)
for i := int64(0); i < per; i++ {
eoff := off + i*128
eb, err := d.Range(eoff, 128)
if err != nil || len(eb) < 128 {
break
}
nameLen := int64(binary.LittleEndian.Uint16(eb[64:66]))
objType := eb[66]
if objType == 0 { // unallocated entry
continue
}
name := ""
if nameLen >= 2 && nameLen <= 64 {
name = decodeUTF16LE(eb[0 : nameLen-2])
}
streamSize := binary.LittleEndian.Uint64(eb[120:128])
dir.Children = append(dir.Children, leaf(
cfbEntryLabel(name, count), "dir_entry", eoff, 128,
fmt.Sprintf("%s size=%d", cfbObjType(objType), streamSize)))
count++
}
dir.Length = sectorSize
dir.Value = fmt.Sprintf("%d entries", count)
if count == 0 {
return nil
}
return dir
}
func cfbEntryLabel(name string, i int64) string {
if name == "" {
return fmt.Sprintf("entry[%d]", i)
}
// Strip the control prefix Office uses on some stream names for display.
clean := strings.Map(func(r rune) rune {
if r < 0x20 {
return -1
}
return r
}, name)
if clean == "" {
clean = name
}
return fmt.Sprintf("entry[%d] %s", i, clean)
}
func cfbObjType(t byte) string {
switch t {
case 1:
return "storage"
case 2:
return "stream"
case 5:
return "root"
default:
return fmt.Sprintf("type%d", t)
}
}
// ---------- shared helpers ----------
func readCStringField(d *document.Document, off, size int64) (string, int64) {
var sb strings.Builder
n := int64(0)
for off+n < size && n < 4096 {
b, err := d.At(off + n)
if err != nil {
break
}
n++
if b == 0 {
break
}
sb.WriteByte(b)
}
return sb.String(), n
}
func allZero(b []byte) bool {
for _, x := range b {
if x != 0 {
return false
}
}
return true
}
func trimZeroStr(b []byte) string {
return strings.TrimRight(string(b), "\x00 ")
}
// octal parses a space/NUL-padded octal ASCII field (tar number encoding).
func octal(b []byte) int64 {
s := strings.TrimSpace(strings.Trim(string(b), "\x00"))
if s == "" {
return 0
}
v, err := strconv.ParseInt(s, 8, 64)
if err != nil {
return 0
}
return v
}
func roundUp(n, to int64) int64 {
if n <= 0 {
return 0
}
return ((n + to - 1) / to) * to
}
internal/leviathan/parsers/macho.go
+408
View File
@@ -0,0 +1,408 @@
package parsers
import (
"encoding/binary"
"errors"
"fmt"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Mach-O magic values (32/64, LE/BE) plus fat (universal) magic.
const (
machoMagic32 = 0xfeedface
machoCigam32 = 0xcefaedfe
machoMagic64 = 0xfeedfacf
machoCigam64 = 0xcffaedfe
machoFatMagic = 0xcafebabe // also Java .class; disambiguate by position + nfat_arch
machoFatCigam = 0xbebafeca
machoFatMagic64 = 0xcafebabf
machoFatCigam64 = 0xbfbafeca
)
// IsMachO reports whether the given prefix looks like a Mach-O header.
// Fat magic 0xcafebabe is shared with Java class files; we accept it
// only when nfat_arch is plausible and the first arch's cputype points
// somewhere in-document.
func IsMachO(head []byte, size int64) bool {
if len(head) < 8 {
return false
}
m := binary.LittleEndian.Uint32(head[:4])
switch m {
case machoMagic32, machoMagic64, machoCigam32, machoCigam64:
return true
}
mBE := binary.BigEndian.Uint32(head[:4])
switch mBE {
case machoFatMagic, machoFatMagic64:
// nfat_arch is next BE u32; plausibly small.
n := binary.BigEndian.Uint32(head[4:8])
return n > 0 && n < 64 && size >= 8+int64(n)*20
}
_ = mBE
return false
}
// ParseMachO returns a field tree for a Mach-O (fat or thin) file.
func ParseMachO(d *document.Document) (*Field, error) {
head, err := d.Range(0, min64(8, d.Size()))
if err != nil {
return nil, err
}
if len(head) < 8 {
return nil, errors.New("file too small for Mach-O")
}
mLE := binary.LittleEndian.Uint32(head[:4])
mBE := binary.BigEndian.Uint32(head[:4])
switch mBE {
case machoFatMagic, machoFatMagic64:
return parseMachOFat(d, mBE == machoFatMagic64)
}
// Thin header.
switch mLE {
case machoMagic32:
return parseMachOThin(d, 0, binary.LittleEndian, false)
case machoMagic64:
return parseMachOThin(d, 0, binary.LittleEndian, true)
case machoCigam32:
return parseMachOThin(d, 0, binary.BigEndian, false)
case machoCigam64:
return parseMachOThin(d, 0, binary.BigEndian, true)
}
return nil, errors.New("not a Mach-O file")
}
func parseMachOFat(d *document.Document, is64 bool) (*Field, error) {
root := &Field{Name: "Mach-O fat", Type: "container", Offset: 0}
hdr, err := d.Range(0, 8)
if err != nil {
return nil, err
}
n := int64(binary.BigEndian.Uint32(hdr[4:8]))
root.Children = append(root.Children, &Field{
Name: "magic", Type: "u32", Offset: 0, Length: 4,
Value: fmt.Sprintf("0x%08x", binary.BigEndian.Uint32(hdr[:4])),
})
root.Children = append(root.Children, &Field{
Name: "nfat_arch", Type: "u32", Offset: 4, Length: 4,
Value: fmt.Sprintf("%d", n),
})
archSize := int64(20)
if is64 {
archSize = 32
}
off := int64(8)
for i := int64(0); i < n && off+archSize <= d.Size(); i++ {
arch, err := d.Range(off, archSize)
if err != nil {
break
}
cpuType := binary.BigEndian.Uint32(arch[0:4])
cpuSub := binary.BigEndian.Uint32(arch[4:8])
var archOff, archLen int64
if is64 {
archOff = int64(binary.BigEndian.Uint64(arch[8:16]))
archLen = int64(binary.BigEndian.Uint64(arch[16:24]))
} else {
archOff = int64(binary.BigEndian.Uint32(arch[8:12]))
archLen = int64(binary.BigEndian.Uint32(arch[12:16]))
}
node := &Field{
Name: fmt.Sprintf("arch[%d]", i),
Type: "fat_arch",
Offset: off,
Length: archSize,
Value: fmt.Sprintf("cpu=%s sub=0x%x offset=%d size=%d", cpuTypeName(cpuType), cpuSub, archOff, archLen),
}
// Parse the nested Mach-O at archOff if it fits.
if archOff > 0 && archOff+8 <= d.Size() {
if nested, err := parseMachOInline(d, archOff); err == nil {
node.Children = append(node.Children, nested)
}
}
root.Children = append(root.Children, node)
off += archSize
}
return root, nil
}
// parseMachOInline is like ParseMachO but anchored at a given offset
// (used for each arch inside a fat binary).
func parseMachOInline(d *document.Document, at int64) (*Field, error) {
head, err := d.Range(at, 8)
if err != nil || len(head) < 8 {
return nil, errors.New("short arch payload")
}
mLE := binary.LittleEndian.Uint32(head[:4])
switch mLE {
case machoMagic32:
return parseMachOThin(d, at, binary.LittleEndian, false)
case machoMagic64:
return parseMachOThin(d, at, binary.LittleEndian, true)
case machoCigam32:
return parseMachOThin(d, at, binary.BigEndian, false)
case machoCigam64:
return parseMachOThin(d, at, binary.BigEndian, true)
}
return nil, errors.New("arch payload is not Mach-O")
}
func parseMachOThin(d *document.Document, base int64, bo binary.ByteOrder, is64 bool) (*Field, error) {
headerLen := int64(28)
if is64 {
headerLen = 32
}
if base+headerLen > d.Size() {
return nil, errors.New("truncated Mach-O header")
}
hdr, err := d.Range(base, headerLen)
if err != nil {
return nil, err
}
magic := bo.Uint32(hdr[0:4])
cpuType := bo.Uint32(hdr[4:8])
cpuSub := bo.Uint32(hdr[8:12])
fileType := bo.Uint32(hdr[12:16])
ncmds := bo.Uint32(hdr[16:20])
sizeOfCmds := bo.Uint32(hdr[20:24])
flags := bo.Uint32(hdr[24:28])
name := "Mach-O (32)"
if is64 {
name = "Mach-O (64)"
}
root := &Field{Name: name, Type: "container", Offset: base, Length: headerLen + int64(sizeOfCmds)}
root.Children = append(root.Children,
&Field{Name: "magic", Type: "u32", Offset: base, Length: 4, Value: fmt.Sprintf("0x%08x", magic)},
&Field{Name: "cputype", Type: "u32", Offset: base + 4, Length: 4, Value: cpuTypeName(cpuType)},
&Field{Name: "cpusubtype", Type: "u32", Offset: base + 8, Length: 4, Value: fmt.Sprintf("0x%x", cpuSub)},
&Field{Name: "filetype", Type: "u32", Offset: base + 12, Length: 4, Value: machoFileType(fileType)},
&Field{Name: "ncmds", Type: "u32", Offset: base + 16, Length: 4, Value: fmt.Sprintf("%d", ncmds)},
&Field{Name: "sizeofcmds", Type: "u32", Offset: base + 20, Length: 4, Value: fmt.Sprintf("%d", sizeOfCmds)},
&Field{Name: "flags", Type: "u32", Offset: base + 24, Length: 4, Value: fmt.Sprintf("0x%08x", flags)},
)
// Load commands.
cmdOff := base + headerLen
cmdEnd := cmdOff + int64(sizeOfCmds)
if cmdEnd > d.Size() {
cmdEnd = d.Size()
}
cmds := &Field{Name: "load_commands", Type: "list", Offset: cmdOff, Length: int64(sizeOfCmds)}
for i := uint32(0); i < ncmds && cmdOff+8 <= cmdEnd; i++ {
h, err := d.Range(cmdOff, 8)
if err != nil {
break
}
cmd := bo.Uint32(h[0:4])
sz := int64(bo.Uint32(h[4:8]))
if sz < 8 || cmdOff+sz > cmdEnd {
break
}
buf, _ := d.Range(cmdOff, sz)
node := decodeLoadCommand(base, cmdOff, cmd, sz, buf, bo, is64)
cmds.Children = append(cmds.Children, node)
cmdOff += sz
}
root.Children = append(root.Children, cmds)
return root, nil
}
func decodeLoadCommand(base, off int64, cmd uint32, sz int64, buf []byte, bo binary.ByteOrder, is64 bool) *Field {
name, mask := loadCommandName(cmd)
node := &Field{
Name: name, Type: "load_command", Offset: off, Length: sz,
Value: fmt.Sprintf("cmd=0x%08x size=%d", cmd, sz),
}
if mask != 0 {
node.Note = "required"
}
switch {
case (cmd&^0x80000000) == 0x19 && is64 && sz >= 72:
// LC_SEGMENT_64
segName := cstring(buf[8:24])
vmAddr := bo.Uint64(buf[24:32])
vmSize := bo.Uint64(buf[32:40])
fileOff := bo.Uint64(buf[40:48])
fileSize := bo.Uint64(buf[48:56])
nsects := bo.Uint32(buf[64:68])
node.Value = fmt.Sprintf("name=%q vm=%#x+%#x file=%#x+%#x nsects=%d",
segName, vmAddr, vmSize, fileOff, fileSize, nsects)
sectOff := int64(72)
for i := uint32(0); i < nsects && sectOff+80 <= sz; i++ {
s := buf[sectOff:]
sn := cstring(s[0:16])
seg := cstring(s[16:32])
addr := bo.Uint64(s[32:40])
size := bo.Uint64(s[40:48])
foff := bo.Uint32(s[48:52])
node.Children = append(node.Children, &Field{
Name: sn, Type: "section64", Offset: off + sectOff, Length: 80,
Value: fmt.Sprintf("seg=%q addr=%#x size=%#x file=%#x", seg, addr, size, foff),
})
sectOff += 80
}
case cmd == 0x01 && sz >= 56:
// LC_SEGMENT (32)
segName := cstring(buf[8:24])
vmAddr := bo.Uint32(buf[24:28])
vmSize := bo.Uint32(buf[28:32])
fileOff := bo.Uint32(buf[32:36])
fileSize := bo.Uint32(buf[36:40])
nsects := bo.Uint32(buf[48:52])
node.Value = fmt.Sprintf("name=%q vm=%#x+%#x file=%#x+%#x nsects=%d",
segName, vmAddr, vmSize, fileOff, fileSize, nsects)
case cmd == 0x1b && sz >= 24:
// LC_UUID
u := buf[8:24]
node.Value = fmt.Sprintf("UUID=%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:16])
case (cmd&^0x80000000) == 0x0c && sz > 24:
// LC_LOAD_DYLIB / similar
strOff := bo.Uint32(buf[8:12])
if int(strOff) < len(buf) {
node.Value = fmt.Sprintf("lib=%q", cstring(buf[strOff:]))
}
case cmd == 0x0e && sz > 12:
// LC_LOAD_DYLINKER
strOff := bo.Uint32(buf[8:12])
if int(strOff) < len(buf) {
node.Value = fmt.Sprintf("dylinker=%q", cstring(buf[strOff:]))
}
case cmd == 0x21 && sz >= 16:
// LC_ENCRYPTION_INFO / _64
node.Value = "encryption info"
case (cmd &^ 0x80000000) == 0x22:
node.Value = "LC_DYLD_INFO"
case cmd == 0x25:
node.Value = "LC_VERSION_MIN_MACOSX"
case cmd == 0x28 || cmd == 0x29:
node.Value = "LC_MAIN / entry point"
}
return node
}
func cstring(b []byte) string {
for i, c := range b {
if c == 0 {
return string(b[:i])
}
}
return string(b)
}
func cpuTypeName(c uint32) string {
switch c {
case 0x7:
return "x86"
case 0x1000007:
return "x86_64"
case 0xc:
return "arm"
case 0x100000c:
return "arm64"
case 0x12:
return "ppc"
case 0x1000012:
return "ppc64"
}
return fmt.Sprintf("0x%x", c)
}
func machoFileType(t uint32) string {
switch t {
case 1:
return "OBJECT"
case 2:
return "EXECUTE"
case 3:
return "FVMLIB"
case 4:
return "CORE"
case 5:
return "PRELOAD"
case 6:
return "DYLIB"
case 7:
return "DYLINKER"
case 8:
return "BUNDLE"
case 9:
return "DYLIB_STUB"
case 10:
return "DSYM"
case 11:
return "KEXT_BUNDLE"
}
return fmt.Sprintf("%d", t)
}
// loadCommandName returns a human name and whether the required bit is set.
func loadCommandName(cmd uint32) (string, uint32) {
req := cmd & 0x80000000
v := cmd &^ 0x80000000
names := map[uint32]string{
0x01: "LC_SEGMENT",
0x02: "LC_SYMTAB",
0x03: "LC_SYMSEG",
0x04: "LC_THREAD",
0x05: "LC_UNIXTHREAD",
0x06: "LC_LOADFVMLIB",
0x07: "LC_IDFVMLIB",
0x08: "LC_IDENT",
0x09: "LC_FVMFILE",
0x0a: "LC_PREPAGE",
0x0b: "LC_DYSYMTAB",
0x0c: "LC_LOAD_DYLIB",
0x0d: "LC_ID_DYLIB",
0x0e: "LC_LOAD_DYLINKER",
0x0f: "LC_ID_DYLINKER",
0x10: "LC_PREBOUND_DYLIB",
0x11: "LC_ROUTINES",
0x12: "LC_SUB_FRAMEWORK",
0x13: "LC_SUB_UMBRELLA",
0x14: "LC_SUB_CLIENT",
0x15: "LC_SUB_LIBRARY",
0x16: "LC_TWOLEVEL_HINTS",
0x17: "LC_PREBIND_CKSUM",
0x18: "LC_LOAD_WEAK_DYLIB",
0x19: "LC_SEGMENT_64",
0x1a: "LC_ROUTINES_64",
0x1b: "LC_UUID",
0x1c: "LC_RPATH",
0x1d: "LC_CODE_SIGNATURE",
0x1e: "LC_SEGMENT_SPLIT_INFO",
0x1f: "LC_REEXPORT_DYLIB",
0x20: "LC_LAZY_LOAD_DYLIB",
0x21: "LC_ENCRYPTION_INFO",
0x22: "LC_DYLD_INFO",
0x23: "LC_LOAD_UPWARD_DYLIB",
0x24: "LC_VERSION_MIN_MACOSX",
0x25: "LC_VERSION_MIN_IPHONEOS",
0x26: "LC_FUNCTION_STARTS",
0x27: "LC_DYLD_ENVIRONMENT",
0x28: "LC_MAIN",
0x29: "LC_DATA_IN_CODE",
0x2a: "LC_SOURCE_VERSION",
0x2b: "LC_DYLIB_CODE_SIGN_DRS",
0x2c: "LC_ENCRYPTION_INFO_64",
0x2d: "LC_LINKER_OPTION",
0x2e: "LC_LINKER_OPTIMIZATION_HINT",
0x2f: "LC_VERSION_MIN_TVOS",
0x30: "LC_VERSION_MIN_WATCHOS",
0x31: "LC_NOTE",
0x32: "LC_BUILD_VERSION",
0x33: "LC_DYLD_EXPORTS_TRIE",
0x34: "LC_DYLD_CHAINED_FIXUPS",
0x35: "LC_FILESET_ENTRY",
}
if n, ok := names[v]; ok {
return n, req
}
return fmt.Sprintf("LC_0x%x", v), req
}
internal/leviathan/parsers/peimports.go
+150
View File
@@ -0,0 +1,150 @@
package parsers
import (
"encoding/binary"
"errors"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ImportSym is one resolved PE import (a function name, or an ordinal).
type ImportSym struct {
DLL string `json:"dll"`
Func string `json:"func"`
Ordinal int `json:"ordinal"`
ByOrdinal bool `json:"byOrdinal"`
}
// PEImports walks the import directory of a PE file and returns the ordered
// list of imported symbols. It reuses the same RVA mapping and header parsing
// as ParsePE but yields a clean typed result (for imphash, capability
// analysis, etc.) rather than a display tree.
func PEImports(d *document.Document) ([]ImportSym, error) {
sections, dataDirs, is64, err := peLayout(d)
if err != nil {
return nil, err
}
if len(dataDirs) < 2 || dataDirs[1].RVA == 0 {
return nil, errors.New("no import directory")
}
off, ok := peRVAToOffset(sections, dataDirs[1].RVA)
if !ok {
return nil, errors.New("import RVA unmapped")
}
var out []ImportSym
for i := 0; i < 1024; i++ {
at := int64(off) + int64(i*20)
buf, err := d.Range(at, 20)
if err != nil || len(buf) < 20 {
break
}
oft := binary.LittleEndian.Uint32(buf[0:4])
nameRVA := binary.LittleEndian.Uint32(buf[12:16])
firstThunk := binary.LittleEndian.Uint32(buf[16:20])
if oft == 0 && nameRVA == 0 && firstThunk == 0 {
break
}
dll := ""
if nameOff, ok := peRVAToOffset(sections, nameRVA); ok {
dll = readCString(d, int64(nameOff), 256)
}
if dll == "" {
continue
}
out = append(out, peThunkSyms(d, sections, dll, oft, firstThunk, is64)...)
if len(out) > 100000 {
break
}
}
return out, nil
}
func peThunkSyms(d *document.Document, sections []peSectionInfo, dll string, oft, firstThunk uint32, is64 bool) []ImportSym {
thunkRVA := oft
if thunkRVA == 0 {
thunkRVA = firstThunk
}
thunkOff, ok := peRVAToOffset(sections, thunkRVA)
if !ok {
return nil
}
width := int64(4)
ordinalMask := uint64(0x80000000)
if is64 {
width = 8
ordinalMask = 0x8000000000000000
}
var out []ImportSym
for i := 0; i < 8192; i++ {
at := int64(thunkOff) + int64(i)*width
buf, err := d.Range(at, width)
if err != nil || int64(len(buf)) < width {
break
}
var thunk uint64
if is64 {
thunk = binary.LittleEndian.Uint64(buf)
} else {
thunk = uint64(binary.LittleEndian.Uint32(buf))
}
if thunk == 0 {
break
}
if thunk&ordinalMask != 0 {
out = append(out, ImportSym{DLL: dll, Ordinal: int(thunk & 0xffff), ByOrdinal: true})
} else if nameOff, ok := peRVAToOffset(sections, uint32(thunk)); ok {
name := readImportByName(d, int64(nameOff))
if name != "" {
out = append(out, ImportSym{DLL: dll, Func: name})
}
}
}
return out
}
// peLayout extracts the section table, data directories and 32/64-bitness from
// a PE without building the display field tree.
func peLayout(d *document.Document) ([]peSectionInfo, []peDataDirectory, bool, error) {
hdr, err := d.Range(0, 64)
if err != nil || len(hdr) < 64 {
return nil, nil, false, errors.New("too short")
}
lfanew := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40]))
peHdr, err := d.Range(lfanew, 24)
if err != nil || len(peHdr) < 24 || string(peHdr[:4]) != "PE\x00\x00" {
return nil, nil, false, errors.New("no PE signature")
}
numSections := int(binary.LittleEndian.Uint16(peHdr[6:8]))
sizeOptHdr := int64(binary.LittleEndian.Uint16(peHdr[20:22]))
optOff := lfanew + 24
var dataDirs []peDataDirectory
is64 := false
if sizeOptHdr > 0 {
opt, err := d.Range(optOff, sizeOptHdr)
if err == nil && len(opt) >= 2 {
_, dirs := parsePEOptionalHeader(opt, optOff)
dataDirs = dirs
is64 = binary.LittleEndian.Uint16(opt[0:2]) == 0x20b
}
}
sectOff := lfanew + 24 + sizeOptHdr
sections := make([]peSectionInfo, 0, numSections)
for i := 0; i < numSections; i++ {
sh, err := d.Range(sectOff, 40)
if err != nil || len(sh) < 40 {
break
}
sections = append(sections, peSectionInfo{
Name: string(trimZero(sh[0:8])),
VirtualSize: binary.LittleEndian.Uint32(sh[8:12]),
VirtualAddress: binary.LittleEndian.Uint32(sh[12:16]),
RawSize: binary.LittleEndian.Uint32(sh[16:20]),
RawOffset: binary.LittleEndian.Uint32(sh[20:24]),
Characteristics: binary.LittleEndian.Uint32(sh[36:40]),
})
sectOff += 40
}
return sections, dataDirs, is64, nil
}
internal/leviathan/patch/script.go
+232
View File
@@ -0,0 +1,232 @@
// Package patch implements the simple Leviathan patch script language.
//
// set OFFSET HEX... — write hex bytes at OFFSET
// fill START LEN BYTE — fill a range with a single byte
// xor START LEN KEY — XOR a range with an ASCII or hex: key
// copy SRC LEN DST — copy bytes from SRC to DST
// # comments and blank lines are allowed.
package patch
import (
"encoding/hex"
"errors"
"fmt"
"strconv"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// Step is one parsed instruction.
type Step struct {
Op string
Args []string
Line int
Source string
}
// Parse parses the script into steps.
func Parse(src string) ([]Step, error) {
out := []Step{}
for i, raw := range strings.Split(src, "\n") {
line := strings.TrimSpace(raw)
if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") {
continue
}
fields := strings.Fields(line)
op := strings.ToLower(fields[0])
out = append(out, Step{
Op: op, Args: fields[1:], Line: i + 1, Source: raw,
})
}
return out, nil
}
// Plan validates all steps against document size without applying.
// Returns per-step info or an error on validation failure.
func Plan(d *document.Document, steps []Step) ([]map[string]any, error) {
size := d.Size()
info := make([]map[string]any, 0, len(steps))
for _, s := range steps {
switch s.Op {
case "set":
if len(s.Args) < 2 {
return info, fmt.Errorf("line %d: set needs OFFSET and HEX", s.Line)
}
off, err := parseOffset(s.Args[0])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
b, err := parseHexRest(s.Args[1:])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
if !validRange(size, off, int64(len(b))) {
return info, fmt.Errorf("line %d: out of range", s.Line)
}
info = append(info, map[string]any{
"op": "set", "line": s.Line, "offset": off, "length": len(b),
})
case "fill":
if len(s.Args) != 3 {
return info, fmt.Errorf("line %d: fill needs START LEN BYTE", s.Line)
}
off, err := parseOffset(s.Args[0])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
n, err := parseOffset(s.Args[1])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
by, err := parseHexByte(s.Args[2])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
if !validRange(size, off, n) {
return info, fmt.Errorf("line %d: out of range", s.Line)
}
info = append(info, map[string]any{
"op": "fill", "line": s.Line, "offset": off, "length": n, "byte": by,
})
case "xor":
if len(s.Args) != 3 {
return info, fmt.Errorf("line %d: xor needs START LEN KEY", s.Line)
}
off, err := parseOffset(s.Args[0])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
n, err := parseOffset(s.Args[1])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
key, err := parseKey(s.Args[2])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
if !validRange(size, off, n) {
return info, fmt.Errorf("line %d: out of range", s.Line)
}
info = append(info, map[string]any{
"op": "xor", "line": s.Line, "offset": off, "length": n, "keyLen": len(key),
})
case "copy":
if len(s.Args) != 3 {
return info, fmt.Errorf("line %d: copy needs SRC LEN DST", s.Line)
}
src, err := parseOffset(s.Args[0])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
n, err := parseOffset(s.Args[1])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
dst, err := parseOffset(s.Args[2])
if err != nil {
return info, fmt.Errorf("line %d: %v", s.Line, err)
}
if !validRange(size, src, n) || !validRange(size, dst, n) {
return info, fmt.Errorf("line %d: out of range", s.Line)
}
info = append(info, map[string]any{
"op": "copy", "line": s.Line, "src": src, "dst": dst, "length": n,
})
default:
return info, fmt.Errorf("line %d: unknown op %q", s.Line, s.Op)
}
}
return info, nil
}
// Apply runs the script against the document. Callers may want to
// first snapshot a checkpoint so apply can be reverted.
func Apply(d *document.Document, steps []Step) error {
if _, err := Plan(d, steps); err != nil {
return err
}
for _, s := range steps {
switch s.Op {
case "set":
off, _ := parseOffset(s.Args[0])
b, _ := parseHexRest(s.Args[1:])
for i, bb := range b {
if _, err := d.Set(off+int64(i), bb); err != nil {
return err
}
}
case "fill":
off, _ := parseOffset(s.Args[0])
n, _ := parseOffset(s.Args[1])
by, _ := parseHexByte(s.Args[2])
if err := d.Fill(off, n, by); err != nil {
return err
}
case "xor":
off, _ := parseOffset(s.Args[0])
n, _ := parseOffset(s.Args[1])
key, _ := parseKey(s.Args[2])
if err := d.XOR(off, n, key); err != nil {
return err
}
case "copy":
src, _ := parseOffset(s.Args[0])
n, _ := parseOffset(s.Args[1])
dst, _ := parseOffset(s.Args[2])
if err := d.Copy(src, n, dst); err != nil {
return err
}
}
}
return nil
}
func validRange(size, start, length int64) bool {
return start >= 0 && start <= size && length >= 0 && length <= size-start
}
func parseOffset(s string) (int64, error) {
s = strings.TrimSpace(s)
if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") {
v, err := strconv.ParseInt(s[2:], 16, 64)
return v, err
}
return strconv.ParseInt(s, 10, 64)
}
func parseHexRest(args []string) ([]byte, error) {
joined := strings.Join(args, "")
joined = strings.ReplaceAll(joined, " ", "")
joined = strings.ReplaceAll(joined, ",", "")
return hex.DecodeString(joined)
}
func parseHexByte(s string) (byte, error) {
if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") {
v, err := strconv.ParseUint(s[2:], 16, 8)
return byte(v), err
}
b, err := hex.DecodeString(s)
if err != nil {
v, err := strconv.ParseUint(s, 10, 8)
if err != nil {
return 0, errors.New("not a byte")
}
return byte(v), nil
}
if len(b) != 1 {
return 0, errors.New("expected one byte")
}
return b[0], nil
}
func parseKey(s string) ([]byte, error) {
if strings.HasPrefix(s, "hex:") {
return hex.DecodeString(s[4:])
}
if s == "" {
return nil, errors.New("empty key")
}
return []byte(s), nil
}
internal/leviathan/pdf/filters.go
+411
View File
@@ -0,0 +1,411 @@
package pdf
import (
"bytes"
"errors"
"fmt"
"github.com/andydixon/nexus/internal/leviathan/transforms"
)
const MaxDecodedStreamOutput = 64 << 20
// ApplyFilter decodes one PDF stream filter. When a predictor is in use
// it must be applied separately after this function returns raw filter
// output; see ApplyPredictor.
func ApplyFilter(name string, in []byte) ([]byte, error) {
switch name {
case "FlateDecode", "Fl":
if out, err := transforms.InflateZlib(in); err == nil {
return out, nil
}
return transforms.InflateRaw(in)
case "ASCIIHexDecode", "AHx":
return decodeASCIIHex(in)
case "ASCII85Decode", "A85":
return decodeASCII85(in)
case "RunLengthDecode", "RL":
return decodeRunLength(in)
case "LZWDecode", "LZW":
return decodeLZW(in)
case "CCITTFaxDecode", "CCF":
return nil, errors.New("CCITTFaxDecode not supported (returns raw bytes)")
case "JBIG2Decode":
return nil, errors.New("JBIG2Decode not supported")
case "DCTDecode", "DCT":
// JPEG payload — return as-is, caller can save as .jpg.
return in, nil
case "JPXDecode":
return in, nil
case "Crypt":
return nil, errors.New("Crypt filter not supported")
}
return nil, fmt.Errorf("unknown filter %q", name)
}
// ApplyPredictor undoes the PNG-family predictor used with Flate/LZW. If
// predictor <= 1 the data is returned unchanged.
func ApplyPredictor(in []byte, predictor, columns, colors, bits int) ([]byte, error) {
if predictor <= 1 {
return in, nil
}
if len(in) > MaxDecodedStreamOutput {
return nil, fmt.Errorf("predictor input exceeds %d byte limit", MaxDecodedStreamOutput)
}
bytesPerPixel, rowLen, err := predictorGeometry(columns, colors, bits)
if err != nil {
return nil, err
}
switch {
case predictor == 2:
// TIFF predictor — apply left delta per sample.
out := make([]byte, len(in))
copy(out, in)
step := bytesPerPixel
for i := step; i < len(out); i++ {
if i%rowLen < step {
continue
}
out[i] = out[i] + out[i-step]
}
return out, nil
case predictor >= 10 && predictor <= 15:
// PNG predictors — each row is prefixed with a filter-type byte.
stride := rowLen + 1
out := make([]byte, 0, len(in))
var prev []byte
for i := 0; i < len(in); i += stride {
if i+stride > len(in) {
break
}
filterType := in[i]
row := make([]byte, rowLen)
copy(row, in[i+1:i+stride])
switch filterType {
case 0: // None
case 1: // Sub
for j := bytesPerPixel; j < rowLen; j++ {
row[j] += row[j-bytesPerPixel]
}
case 2: // Up
if prev != nil {
for j := 0; j < rowLen; j++ {
row[j] += prev[j]
}
}
case 3: // Average
for j := 0; j < rowLen; j++ {
left := byte(0)
if j >= bytesPerPixel {
left = row[j-bytesPerPixel]
}
up := byte(0)
if prev != nil {
up = prev[j]
}
row[j] += byte((int(left) + int(up)) / 2)
}
case 4: // Paeth
for j := 0; j < rowLen; j++ {
a := 0
if j >= bytesPerPixel {
a = int(row[j-bytesPerPixel])
}
b := 0
if prev != nil {
b = int(prev[j])
}
c := 0
if prev != nil && j >= bytesPerPixel {
c = int(prev[j-bytesPerPixel])
}
p := a + b - c
pa := abs(p - a)
pb := abs(p - b)
pc := abs(p - c)
pred := 0
switch {
case pa <= pb && pa <= pc:
pred = a
case pb <= pc:
pred = b
default:
pred = c
}
row[j] += byte(pred)
}
default:
return nil, fmt.Errorf("PNG predictor: invalid filter type %d", filterType)
}
out = append(out, row...)
prev = row
}
return out, nil
}
return nil, fmt.Errorf("predictor %d not supported", predictor)
}
func predictorGeometry(columns, colors, bits int) (int, int, error) {
if columns <= 0 || colors <= 0 {
return 0, 0, errors.New("predictor columns and colors must be positive")
}
switch bits {
case 1, 2, 4, 8, 16:
default:
return 0, 0, fmt.Errorf("predictor bits per component %d is invalid", bits)
}
pixelBits := int64(colors) * int64(bits)
rowBits := int64(columns) * pixelBits
if pixelBits <= 0 || rowBits <= 0 || rowBits > int64(MaxDecodedStreamOutput)*8 {
return 0, 0, errors.New("predictor dimensions exceed resource limits")
}
return int((pixelBits + 7) / 8), int((rowBits + 7) / 8), nil
}
func ensureDecodedSize(current, additional int) error {
if additional < 0 || current > MaxDecodedStreamOutput-additional {
return fmt.Errorf("decoded stream exceeds %d byte limit", MaxDecodedStreamOutput)
}
return nil
}
func abs(x int) int {
if x < 0 {
return -x
}
return x
}
// ---------------------------------------------------------------------------
// ASCIIHexDecode
// ---------------------------------------------------------------------------
func decodeASCIIHex(in []byte) ([]byte, error) {
out := make([]byte, 0, len(in)/2)
hi := -1
for _, b := range in {
switch {
case b >= '0' && b <= '9':
b = b - '0'
case b >= 'a' && b <= 'f':
b = b - 'a' + 10
case b >= 'A' && b <= 'F':
b = b - 'A' + 10
case b == '>':
if hi >= 0 {
out = append(out, byte(hi)<<4)
}
return out, nil
case isPDFWhitespace(b):
continue
default:
continue
}
if hi < 0 {
hi = int(b)
} else {
out = append(out, byte(hi)<<4|b)
hi = -1
}
}
if hi >= 0 {
out = append(out, byte(hi)<<4)
}
return out, nil
}
func isPDFWhitespace(b byte) bool {
return b == 0 || b == '\t' || b == '\n' || b == '\f' || b == '\r' || b == ' '
}
// ---------------------------------------------------------------------------
// ASCII85Decode
// ---------------------------------------------------------------------------
func decodeASCII85(in []byte) ([]byte, error) {
// Strip optional <~ ... ~> framing and whitespace.
start := 0
if bytes.HasPrefix(in, []byte("<~")) {
start = 2
}
out := make([]byte, 0, len(in))
var group [5]byte
n := 0
for i := start; i < len(in); i++ {
c := in[i]
if isPDFWhitespace(c) {
continue
}
if c == '~' {
break
}
if c == 'z' && n == 0 {
if err := ensureDecodedSize(len(out), 4); err != nil {
return nil, err
}
out = append(out, 0, 0, 0, 0)
continue
}
if c < '!' || c > 'u' {
return nil, fmt.Errorf("ASCII85: invalid byte 0x%02x", c)
}
group[n] = c - '!'
n++
if n == 5 {
v := uint32(group[0])*85*85*85*85 +
uint32(group[1])*85*85*85 +
uint32(group[2])*85*85 +
uint32(group[3])*85 +
uint32(group[4])
if err := ensureDecodedSize(len(out), 4); err != nil {
return nil, err
}
out = append(out, byte(v>>24), byte(v>>16), byte(v>>8), byte(v))
n = 0
}
}
if n > 0 {
for i := n; i < 5; i++ {
group[i] = 84
}
v := uint32(group[0])*85*85*85*85 +
uint32(group[1])*85*85*85 +
uint32(group[2])*85*85 +
uint32(group[3])*85 +
uint32(group[4])
full := []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)}
if err := ensureDecodedSize(len(out), n-1); err != nil {
return nil, err
}
out = append(out, full[:n-1]...)
}
return out, nil
}
// ---------------------------------------------------------------------------
// RunLengthDecode
// ---------------------------------------------------------------------------
func decodeRunLength(in []byte) ([]byte, error) {
out := make([]byte, 0, len(in))
i := 0
for i < len(in) {
b := in[i]
i++
switch {
case b == 128:
return out, nil
case b < 128:
n := int(b) + 1
if i+n > len(in) {
return nil, errors.New("RunLength: truncated literal")
}
if err := ensureDecodedSize(len(out), n); err != nil {
return nil, err
}
out = append(out, in[i:i+n]...)
i += n
default:
n := 257 - int(b)
if i >= len(in) {
return nil, errors.New("RunLength: truncated run")
}
if err := ensureDecodedSize(len(out), n); err != nil {
return nil, err
}
for k := 0; k < n; k++ {
out = append(out, in[i])
}
i++
}
}
return out, nil
}
// ---------------------------------------------------------------------------
// LZWDecode (variable-width, early-change = 1)
// ---------------------------------------------------------------------------
func decodeLZW(in []byte) ([]byte, error) {
const (
clearCode = 256
eodCode = 257
)
reader := &bitReader{data: in}
table := make([][]byte, 258)
for i := 0; i < 256; i++ {
table[i] = []byte{byte(i)}
}
codeWidth := 9
out := []byte{}
prev := []byte(nil)
for {
c, err := reader.read(codeWidth)
if err != nil {
return out, nil
}
if c == eodCode {
return out, nil
}
if c == clearCode {
table = table[:258]
codeWidth = 9
prev = nil
continue
}
var entry []byte
switch {
case c < len(table):
entry = table[c]
case c == len(table) && prev != nil:
entry = append([]byte{}, prev...)
entry = append(entry, prev[0])
default:
return out, errors.New("LZW: invalid code")
}
if err := ensureDecodedSize(len(out), len(entry)); err != nil {
return nil, err
}
out = append(out, entry...)
if prev != nil {
ne := append([]byte{}, prev...)
ne = append(ne, entry[0])
table = append(table, ne)
// PDF LZW uses "early change": widen one code before full.
switch len(table) {
case 511, 1023, 2047:
codeWidth++
case 4096:
// Implicit CLEAR expected; some encoders omit it.
table = table[:258]
codeWidth = 9
prev = nil
continue
}
}
prev = entry
}
}
type bitReader struct {
data []byte
bi int // bit offset into data
}
func (r *bitReader) read(n int) (int, error) {
if n <= 0 || n > 32 {
return 0, errors.New("bitReader: bad width")
}
v := 0
for i := 0; i < n; i++ {
byteIdx := r.bi / 8
if byteIdx >= len(r.data) {
return 0, errors.New("bitReader: EOF")
}
bit := (r.data[byteIdx] >> uint(7-(r.bi%8))) & 1
v = (v << 1) | int(bit)
r.bi++
}
return v, nil
}
internal/leviathan/pdf/pdf.go
+352
View File
@@ -0,0 +1,352 @@
// Package pdf provides PDF surgery: object enumeration, full stream
// filter chain, cross-reference parsing (classical tables and xref
// streams), compressed object-stream decoding, and surfacing of hostile
// or otherwise-interesting constructs (JavaScript, embedded files, URI
// and launch actions, XFA, OpenAction triggers).
//
// It is not a rendering engine — it will not produce pixels. It is a
// structural tool for humans and detection pipelines.
package pdf
import (
"bytes"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
const maxPDFParseBytes = 128 << 20
// Object is one PDF indirect object.
type Object struct {
Num int64
Gen int64
Offset int64
Dict string
HasStream bool
StreamOff int64
StreamLen int64
Filters []string
Type string
Subtype string
Source string // "direct" or "objstm"
Container int64 // when Source == "objstm", the ObjStm number
}
// Parse returns a summary of the PDF structure, including cross-
// reference entries, an object list (direct + recovered from ObjStms),
// surfaced suspicious constructs, and top-level flags.
func Parse(d *document.Document) (map[string]any, error) {
size := d.Size()
head, _ := d.Range(0, 16)
if len(head) < 5 || string(head[:5]) != "%PDF-" {
return nil, errors.New("not a PDF header")
}
if size > maxPDFParseBytes {
return nil, fmt.Errorf("pdf parse skipped: file > %d MiB", maxPDFParseBytes>>20)
}
version := string(head[5:8])
whole, err := d.Range(0, size)
if err != nil {
return nil, err
}
objs := scanIndirectObjects(whole)
// Try xref first; fall back gracefully if it isn't parseable.
xref, xrefMode, xrefErr := ParseXRef(d, whole)
xrefNote := ""
if xrefErr != nil {
xrefNote = xrefErr.Error()
}
// Decode any ObjStm streams we find and append recovered children.
recovered := recoverObjectStreams(whole, d, objs)
objs = append(objs, recovered...)
// Surface flags.
surfaced := Surface(objs)
encrypted := bytes.Contains(whole, []byte("/Encrypt"))
linearised := HasLinearisation(whole)
// Locate trailer text (for quick eyeball).
trailerText := ""
if tIdx := bytes.LastIndex(whole, []byte("trailer")); tIdx >= 0 {
end := bytes.Index(whole[tIdx:], []byte("startxref"))
if end >= 0 {
trailerText = string(whole[tIdx : tIdx+end])
}
}
return map[string]any{
"version": version,
"size": size,
"encrypted": encrypted,
"linearised": linearised,
"trailer": trailerText,
"objects": objectsToMaps(objs),
"xref": xrefEntriesToMaps(xref),
"xrefMode": xrefMode,
"xrefNote": xrefNote,
"suspicious": surfaceToMaps(surfaced),
}, nil
}
func scanIndirectObjects(whole []byte) []Object {
objRE := regexp.MustCompile(`(?m)(\d+)\s+(\d+)\s+obj\b`)
locs := objRE.FindAllSubmatchIndex(whole, -1)
objs := make([]Object, 0, len(locs))
for _, loc := range locs {
start := int64(loc[0])
num, _ := strconv.ParseInt(string(whole[loc[2]:loc[3]]), 10, 64)
gen, _ := strconv.ParseInt(string(whole[loc[4]:loc[5]]), 10, 64)
o := Object{Num: num, Gen: gen, Offset: start, Source: "direct"}
endIdx := bytes.Index(whole[loc[1]:], []byte("endobj"))
if endIdx < 0 {
continue
}
body := whole[loc[1] : loc[1]+endIdx]
streamIdx := bytes.Index(body, []byte("stream"))
var dict []byte
if streamIdx >= 0 {
dict = body[:streamIdx]
s := streamIdx + len("stream")
if s < len(body) && body[s] == '\r' {
s++
}
if s < len(body) && body[s] == '\n' {
s++
}
o.HasStream = true
o.StreamOff = int64(loc[1]) + int64(s)
endStreamIdx := bytes.LastIndex(body, []byte("endstream"))
if endStreamIdx < 0 {
continue
}
endStreamAbs := int64(loc[1]) + int64(endStreamIdx)
// trim trailing newline/cr
for k := 0; k < 2; k++ {
if endStreamAbs > 0 && endStreamAbs-1 < int64(len(whole)) {
if whole[endStreamAbs-1] == '\n' || whole[endStreamAbs-1] == '\r' {
endStreamAbs--
}
}
}
o.StreamLen = endStreamAbs - o.StreamOff
} else {
dict = body
}
o.Dict = strings.TrimSpace(string(dict))
o.Filters = detectFilters(o.Dict)
o.Type = detectType(o.Dict)
o.Subtype = detectSubtype(o.Dict)
objs = append(objs, o)
}
return objs
}
func recoverObjectStreams(whole []byte, d *document.Document, objs []Object) []Object {
out := []Object{}
for _, o := range objs {
if !o.HasStream {
continue
}
if !strings.Contains(o.Dict, "/Type /ObjStm") && !strings.Contains(o.Dict, "/Type/ObjStm") {
continue
}
raw, err := d.Range(o.StreamOff, o.StreamLen)
if err != nil {
continue
}
// Apply filters in order before handing to ObjStm decoder.
children, err := DecodeObjStm(raw, o.Dict)
if err != nil {
continue
}
for num, body := range children {
out = append(out, Object{
Num: num,
Gen: 0,
Offset: o.StreamOff,
Dict: body,
HasStream: false,
Type: detectType(body),
Subtype: detectSubtype(body),
Source: "objstm",
Container: o.Num,
})
}
}
return out
}
func objectsToMaps(objs []Object) []map[string]any {
out := make([]map[string]any, len(objs))
for i, o := range objs {
filters := make([]any, len(o.Filters))
for j, f := range o.Filters {
filters[j] = f
}
out[i] = map[string]any{
"num": o.Num,
"gen": o.Gen,
"offset": o.Offset,
"dict": o.Dict,
"hasStream": o.HasStream,
"streamOff": o.StreamOff,
"streamLen": o.StreamLen,
"filters": filters,
"type": o.Type,
"subtype": o.Subtype,
"source": o.Source,
"container": o.Container,
}
}
return out
}
func xrefEntriesToMaps(entries []XRefEntry) []map[string]any {
out := make([]map[string]any, len(entries))
for i, e := range entries {
out[i] = map[string]any{
"type": e.Type,
"num": e.ObjNum,
"gen": e.Gen,
"offset": e.Offset,
"objStm": e.ObjStmNum,
"index": e.Index,
}
}
return out
}
func surfaceToMaps(hits []SuspiciousHit) []map[string]any {
out := make([]map[string]any, len(hits))
for i, h := range hits {
out[i] = map[string]any{
"kind": h.Kind,
"num": h.ObjNum,
"gen": h.Gen,
"offset": h.Offset,
"detail": h.Detail,
}
}
return out
}
var filtersRE = regexp.MustCompile(`/Filter\s*(?:\[([^\]]*)\]|/([A-Za-z0-9]+))`)
func detectFilters(dict string) []string {
m := filtersRE.FindStringSubmatch(dict)
if m == nil {
return nil
}
if m[2] != "" {
return []string{m[2]}
}
parts := strings.Fields(m[1])
out := make([]string, 0, len(parts))
for _, p := range parts {
p = strings.TrimPrefix(p, "/")
if p != "" {
out = append(out, p)
}
}
return out
}
var (
typeRE = regexp.MustCompile(`/Type\s*/([A-Za-z0-9]+)`)
subtypeRE = regexp.MustCompile(`/Subtype\s*/([A-Za-z0-9]+)`)
)
func detectType(dict string) string {
m := typeRE.FindStringSubmatch(dict)
if len(m) > 1 {
return m[1]
}
return ""
}
func detectSubtype(dict string) string {
m := subtypeRE.FindStringSubmatch(dict)
if len(m) > 1 {
return m[1]
}
return ""
}
// DecodeStream reads the stream for object objNum and runs it through
// the declared filter chain (with predictor). Works for both direct
// objects and those recovered from ObjStm containers.
func DecodeStream(d *document.Document, objNum int64) (map[string]any, error) {
summary, err := Parse(d)
if err != nil {
return nil, err
}
for _, o := range summary["objects"].([]map[string]any) {
n, _ := o["num"].(int64)
if n != objNum {
continue
}
hasStream, _ := o["hasStream"].(bool)
if !hasStream {
return nil, fmt.Errorf("object %d has no stream", objNum)
}
off, _ := o["streamOff"].(int64)
ln, _ := o["streamLen"].(int64)
raw, err := d.Range(off, ln)
if err != nil {
return nil, err
}
dict, _ := o["dict"].(string)
filters, _ := o["filters"].([]any)
predictor, columns, colors, bits := parseDecodeParms(dict)
decoded := append([]byte(nil), raw...)
applied := []string{}
var runErr error
for _, f := range filters {
name, _ := f.(string)
decoded, runErr = ApplyFilter(name, decoded)
if runErr != nil {
return map[string]any{
"objNum": objNum,
"raw": raw,
"decoded": nil,
"error": runErr.Error(),
"applied": applied,
"filters": filters,
}, nil
}
applied = append(applied, name)
}
if predictor > 1 {
decoded, runErr = ApplyPredictor(decoded, predictor, columns, colors, bits)
if runErr != nil {
return map[string]any{
"objNum": objNum,
"raw": raw,
"decoded": nil,
"error": runErr.Error(),
"applied": applied,
"filters": filters,
}, nil
}
applied = append(applied, fmt.Sprintf("predictor=%d", predictor))
}
return map[string]any{
"objNum": objNum,
"raw": raw,
"decoded": decoded,
"applied": applied,
"filters": filters,
}, nil
}
return nil, fmt.Errorf("object %d not found", objNum)
}
internal/leviathan/pdf/pdf_test.go
+37
View File
@@ -0,0 +1,37 @@
package pdf
import (
"os"
"strings"
"testing"
"github.com/andydixon/nexus/internal/leviathan/document"
)
func TestParseRejectsOversizedPDFBeforeWholeRange(t *testing.T) {
file, err := os.CreateTemp(t.TempDir(), "huge-*.pdf")
if err != nil {
t.Fatalf("CreateTemp() failed: %v", err)
}
if _, err := file.WriteString("%PDF-1.7\n"); err != nil {
t.Fatalf("WriteString() failed: %v", err)
}
if err := file.Truncate(maxPDFParseBytes + 1); err != nil {
t.Fatalf("Truncate() failed: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("Close() failed: %v", err)
}
doc, err := document.Open(file.Name())
if err != nil {
t.Fatalf("document.Open() failed: %v", err)
}
_, err = Parse(doc)
if err == nil {
t.Fatal("expected oversized PDF to be rejected")
}
if !strings.Contains(err.Error(), "pdf parse skipped") {
t.Fatalf("unexpected error: %v", err)
}
}
internal/leviathan/pdf/surface.go
+128
View File
@@ -0,0 +1,128 @@
package pdf
import (
"bytes"
"regexp"
"strings"
)
// SuspiciousHit marks a dictionary fragment worth a closer look. It is
// surfaced to the UI with its containing object number so a user can
// jump straight to it.
type SuspiciousHit struct {
Kind string `json:"kind"`
ObjNum int64 `json:"objNum"`
Gen int64 `json:"gen"`
Offset int64 `json:"offset"`
Detail string `json:"detail,omitempty"`
}
// Surface walks the enumerated object list and flags anything routinely
// abused by hostile PDFs: embedded files, JavaScript, launch actions,
// URI actions, form-submission targets, OpenAction / AA triggers.
func Surface(objects []Object) []SuspiciousHit {
hits := []SuspiciousHit{}
for _, o := range objects {
dict := o.Dict
push := func(kind, detail string) {
hits = append(hits, SuspiciousHit{
Kind: kind, ObjNum: o.Num, Gen: o.Gen,
Offset: o.Offset, Detail: detail,
})
}
// Key markers. Order matters only for readability of the UI.
if containsName(dict, "JavaScript") || containsName(dict, "JS") {
push("javascript", firstLine(dict, 240))
}
if containsName(dict, "EmbeddedFile") || containsName(dict, "Filespec") {
push("embedded-file", extractName(dict, "/F"))
}
if containsName(dict, "Launch") {
push("launch-action", extractLaunchTarget(dict))
}
if containsName(dict, "URI") {
if uri := extractURI(dict); uri != "" {
push("uri-action", uri)
}
}
if containsName(dict, "SubmitForm") {
push("submit-form", "")
}
if containsName(dict, "OpenAction") {
push("open-action", "")
}
if strings.Contains(dict, "/AA") {
push("additional-action", "")
}
if strings.Contains(dict, "/Type /XFA") || strings.Contains(dict, "/XFA ") {
push("xfa-form", "")
}
if strings.Contains(dict, "/RichMedia") {
push("rich-media", "")
}
if strings.Contains(dict, "/Type /Metadata") && strings.Contains(dict, "/Subtype /XML") {
push("xmp-metadata", "")
}
}
return hits
}
func containsName(dict, name string) bool {
// Match `/Name` as a whole token — not inside /Names/ or /FooJavaScript.
pat := regexp.MustCompile(`/` + regexp.QuoteMeta(name) + `\b`)
return pat.MatchString(dict)
}
// Go's regexp caps bounded repetitions at 1000 — do not raise this.
var uriRE = regexp.MustCompile(`/URI\s*\(([^)]{1,1000})\)`)
func extractURI(dict string) string {
m := uriRE.FindStringSubmatch(dict)
if m == nil {
return ""
}
return m[1]
}
var fnameRE = regexp.MustCompile(`/F\s*\(([^)]{1,512})\)`)
func extractName(dict, key string) string {
_ = key
m := fnameRE.FindStringSubmatch(dict)
if m == nil {
return ""
}
return m[1]
}
var launchRE = regexp.MustCompile(`/F\s*\(([^)]{1,512})\)`)
func extractLaunchTarget(dict string) string {
m := launchRE.FindStringSubmatch(dict)
if m == nil {
return ""
}
return m[1]
}
func firstLine(s string, max int) string {
nl := strings.IndexByte(s, '\n')
if nl > 0 {
s = s[:nl]
}
s = strings.TrimSpace(s)
if len(s) > max {
s = s[:max] + "…"
}
return s
}
// HasLinearisation reports whether a PDF is linearised (Fast Web View).
// A linearised PDF has an obj dict near the start containing /Linearized.
func HasLinearisation(whole []byte) bool {
n := 4096
if n > len(whole) {
n = len(whole)
}
return bytes.Contains(whole[:n], []byte("/Linearized"))
}
internal/leviathan/pdf/xref.go
+422
View File
@@ -0,0 +1,422 @@
package pdf
import (
"bytes"
"errors"
"fmt"
"regexp"
"strconv"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// XRefEntry describes one cross-reference entry after parsing either a
// classical xref table or a modern xref stream.
//
// Type 0 — free object
// Type 1 — in-use object; Offset is the byte offset in the file
// Type 2 — object inside an object stream; ObjStmNum identifies the
// container ObjStm and Index is the 0-based position within it
type XRefEntry struct {
Type int `json:"type"`
ObjNum int64 `json:"objNum"`
Gen int64 `json:"gen"`
Offset int64 `json:"offset"`
ObjStmNum int64 `json:"objStmNum,omitempty"`
Index int64 `json:"index,omitempty"`
}
const maxObjectStreamEntries = 1 << 20
// ParseXRef attempts to reconstruct the cross-reference map. It handles
// both classical "xref\n0 N\n..." tables and xref streams (/Type /XRef).
// Returns a flat slice of entries covering every object it could resolve.
func ParseXRef(d *document.Document, whole []byte) ([]XRefEntry, string, error) {
mode := ""
entries := map[int64]XRefEntry{}
// Find "startxref" near the end.
idx := bytes.LastIndex(whole, []byte("startxref"))
if idx < 0 {
return nil, "", errors.New("no startxref")
}
after := whole[idx+len("startxref"):]
after = bytes.TrimLeft(after, " \r\n\t")
var xrefPos int64
fmt.Sscanf(string(after), "%d", &xrefPos)
if xrefPos <= 0 || xrefPos >= int64(len(whole)) {
return nil, "", fmt.Errorf("bad startxref offset %d", xrefPos)
}
// Walk the xref chain via /Prev pointers.
visited := map[int64]bool{}
pos := xrefPos
for pos > 0 && !visited[pos] {
visited[pos] = true
// Classical table if the first non-whitespace token is "xref".
tail := whole[pos:]
trimmed := bytes.TrimLeft(tail, " \r\n\t")
if bytes.HasPrefix(trimmed, []byte("xref")) {
if mode == "" {
mode = "table"
} else if mode != "table" {
mode = "hybrid"
}
nextPrev, err := parseClassicalXRef(trimmed, entries)
if err != nil {
return mapToSlice(entries), mode, err
}
pos = nextPrev
continue
}
// Otherwise an xref stream: an indirect object at `pos`.
if mode == "" {
mode = "stream"
} else if mode != "stream" {
mode = "hybrid"
}
nextPrev, err := parseXRefStream(d, whole, pos, entries)
if err != nil {
return mapToSlice(entries), mode, err
}
pos = nextPrev
}
return mapToSlice(entries), mode, nil
}
func mapToSlice(m map[int64]XRefEntry) []XRefEntry {
out := make([]XRefEntry, 0, len(m))
for _, e := range m {
out = append(out, e)
}
return out
}
var xrefRangeRE = regexp.MustCompile(`(?m)^\s*(\d+)\s+(\d+)\s*$`)
var xrefRowRE = regexp.MustCompile(`^(\d{10})\s+(\d{5})\s+([nf])`)
var prevRE = regexp.MustCompile(`/Prev\s+(\d+)`)
func parseClassicalXRef(block []byte, entries map[int64]XRefEntry) (int64, error) {
// Skip "xref" line.
nl := bytes.IndexByte(block, '\n')
if nl < 0 {
return 0, errors.New("xref: no newline after keyword")
}
cursor := block[nl+1:]
for {
m := xrefRangeRE.FindSubmatchIndex(cursor)
if m == nil {
break
}
if m[0] > 0 {
// Subsection must begin at the start of the cursor.
break
}
start, _ := strconv.ParseInt(string(cursor[m[2]:m[3]]), 10, 64)
count, _ := strconv.ParseInt(string(cursor[m[4]:m[5]]), 10, 64)
// Advance past the subsection line.
nl2 := bytes.IndexByte(cursor, '\n')
if nl2 < 0 {
break
}
cursor = cursor[nl2+1:]
for i := int64(0); i < count; i++ {
if len(cursor) < 20 {
return 0, errors.New("xref: truncated subsection")
}
row := cursor[:20]
rm := xrefRowRE.FindSubmatch(row)
if rm != nil {
off, _ := strconv.ParseInt(string(rm[1]), 10, 64)
gen, _ := strconv.ParseInt(string(rm[2]), 10, 64)
typ := 1
if rm[3][0] == 'f' {
typ = 0
}
num := start + i
if _, seen := entries[num]; !seen {
entries[num] = XRefEntry{
Type: typ, ObjNum: num, Gen: gen, Offset: off,
}
}
}
cursor = cursor[20:]
}
}
// Locate trailer and /Prev.
tIdx := bytes.Index(block, []byte("trailer"))
if tIdx < 0 {
return 0, nil
}
trailer := block[tIdx:]
if e := bytes.Index(trailer, []byte("startxref")); e > 0 {
trailer = trailer[:e]
}
if pm := prevRE.FindSubmatch(trailer); pm != nil {
prev, _ := strconv.ParseInt(string(pm[1]), 10, 64)
return prev, nil
}
return 0, nil
}
func parseXRefStream(d *document.Document, whole []byte, pos int64, entries map[int64]XRefEntry) (int64, error) {
// Locate "obj" after pos, then "stream"/"endstream".
if pos < 0 || int(pos) >= len(whole) {
return 0, errors.New("xref stream: bad offset")
}
body := whole[pos:]
objIdx := bytes.Index(body, []byte("obj"))
if objIdx < 0 {
return 0, errors.New("xref stream: no obj")
}
dictEnd := bytes.Index(body[objIdx:], []byte("stream"))
if dictEnd < 0 {
return 0, errors.New("xref stream: no stream")
}
dict := string(body[objIdx : objIdx+dictEnd])
streamStart := objIdx + dictEnd + len("stream")
if streamStart < len(body) && body[streamStart] == '\r' {
streamStart++
}
if streamStart < len(body) && body[streamStart] == '\n' {
streamStart++
}
endIdx := bytes.Index(body[streamStart:], []byte("endstream"))
if endIdx < 0 {
return 0, errors.New("xref stream: no endstream")
}
raw := body[streamStart : streamStart+endIdx]
// trim trailing newline.
if len(raw) > 0 && raw[len(raw)-1] == '\n' {
raw = raw[:len(raw)-1]
}
if len(raw) > 0 && raw[len(raw)-1] == '\r' {
raw = raw[:len(raw)-1]
}
// Decode the xref-stream dictionary fields.
filters := parseDictNameList(dict, "/Filter")
w := parseDictIntArray(dict, "/W")
index := parseDictIntArray(dict, "/Index")
sizeN := parseDictInt(dict, "/Size")
predictor, columns, colors, bits := parseDecodeParms(dict)
// Apply filters.
data := append([]byte(nil), raw...)
var err error
for _, f := range filters {
data, err = ApplyFilter(f, data)
if err != nil {
return 0, fmt.Errorf("xref stream: filter %s: %w", f, err)
}
}
if predictor > 1 {
data, err = ApplyPredictor(data, predictor, columns, colors, bits)
if err != nil {
return 0, err
}
}
if len(w) != 3 {
return 0, fmt.Errorf("xref stream: bad /W array %v", w)
}
entryLen := 0
for _, width := range w {
if width < 0 || width > 8 {
return 0, fmt.Errorf("xref stream: invalid field width %d", width)
}
entryLen += width
}
if entryLen <= 0 {
return 0, errors.New("xref stream: zero-length entries")
}
if len(index) == 0 {
if sizeN < 0 {
return 0, errors.New("xref stream: negative /Size")
}
index = []int{0, sizeN}
}
if len(index)%2 != 0 {
return 0, errors.New("xref stream: /Index must contain start/count pairs")
}
// Walk subsections.
pi := 0
for s := 0; s+1 < len(index); s += 2 {
first := int64(index[s])
count := int64(index[s+1])
if first < 0 || count < 0 {
return 0, errors.New("xref stream: negative /Index value")
}
for i := int64(0); i < count; i++ {
if pi+entryLen > len(data) {
break
}
row := data[pi : pi+entryLen]
pi += entryLen
t := readBE(row[:w[0]])
f2 := readBE(row[w[0] : w[0]+w[1]])
f3 := readBE(row[w[0]+w[1] : w[0]+w[1]+w[2]])
if w[0] == 0 {
t = 1 // default Type 1 per spec
}
num := first + i
if _, seen := entries[num]; seen {
continue
}
switch t {
case 0:
entries[num] = XRefEntry{Type: 0, ObjNum: num, Gen: f3}
case 1:
entries[num] = XRefEntry{Type: 1, ObjNum: num, Gen: f3, Offset: f2}
case 2:
entries[num] = XRefEntry{
Type: 2, ObjNum: num, ObjStmNum: f2, Index: f3,
}
}
}
}
if prev := parseDictInt(dict, "/Prev"); prev > 0 {
return int64(prev), nil
}
return 0, nil
}
func readBE(b []byte) int64 {
v := int64(0)
for _, c := range b {
v = (v << 8) | int64(c)
}
return v
}
// ---------------------------------------------------------------------------
// ObjStm — compressed object streams
// ---------------------------------------------------------------------------
// DecodeObjStm decodes an /ObjStm stream and returns a map of
// object-number → raw object body (text, never with "stream").
func DecodeObjStm(raw []byte, dict string) (map[int64]string, error) {
filters := parseDictNameList(dict, "/Filter")
predictor, columns, colors, bits := parseDecodeParms(dict)
data := append([]byte(nil), raw...)
var err error
for _, f := range filters {
data, err = ApplyFilter(f, data)
if err != nil {
return nil, err
}
}
if predictor > 1 {
data, err = ApplyPredictor(data, predictor, columns, colors, bits)
if err != nil {
return nil, err
}
}
n := parseDictInt(dict, "/N")
first := parseDictInt(dict, "/First")
if n <= 0 || first <= 0 || first >= len(data) {
return nil, fmt.Errorf("ObjStm: bad /N=%d /First=%d", n, first)
}
header := strings.Fields(string(data[:first]))
if n > maxObjectStreamEntries || n > len(header)/2 {
return nil, fmt.Errorf("ObjStm: header has %d tokens, need %d", len(header), n*2)
}
out := map[int64]string{}
offsets := make([]int, n)
nums := make([]int64, n)
for i := 0; i < n; i++ {
num, _ := strconv.ParseInt(header[i*2], 10, 64)
off, _ := strconv.Atoi(header[i*2+1])
nums[i] = num
offsets[i] = off
}
for i := 0; i < n; i++ {
start := first + offsets[i]
end := len(data)
if i+1 < n {
end = first + offsets[i+1]
}
if start < 0 || end > len(data) || start > end {
continue
}
out[nums[i]] = strings.TrimSpace(string(data[start:end]))
}
return out, nil
}
// ---------------------------------------------------------------------------
// Small dictionary helpers
// ---------------------------------------------------------------------------
var decodeParmRE = regexp.MustCompile(`/DecodeParms\s*<<([^>]*)>>`)
func parseDictInt(dict, key string) int {
pat := regexp.MustCompile(regexp.QuoteMeta(key) + `\s+(-?\d+)\b`)
m := pat.FindStringSubmatch(dict)
if m == nil {
return 0
}
n, _ := strconv.Atoi(m[1])
return n
}
func parseDictIntArray(dict, key string) []int {
pat := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*\[([^\]]*)\]`)
m := pat.FindStringSubmatch(dict)
if m == nil {
return nil
}
parts := strings.Fields(m[1])
out := make([]int, 0, len(parts))
for _, p := range parts {
n, err := strconv.Atoi(p)
if err != nil {
continue
}
out = append(out, n)
}
return out
}
func parseDictNameList(dict, key string) []string {
// Handles both `/Filter /FlateDecode` and `/Filter [/A85 /Flate]`.
pat1 := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*/([A-Za-z0-9]+)`)
if m := pat1.FindStringSubmatch(dict); m != nil {
return []string{m[1]}
}
pat2 := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*\[([^\]]*)\]`)
if m := pat2.FindStringSubmatch(dict); m != nil {
parts := strings.Fields(m[1])
out := make([]string, 0, len(parts))
for _, p := range parts {
out = append(out, strings.TrimPrefix(p, "/"))
}
return out
}
return nil
}
func parseDecodeParms(dict string) (predictor, columns, colors, bits int) {
m := decodeParmRE.FindStringSubmatch(dict)
if m == nil {
return 1, 1, 1, 8
}
inner := m[1]
predictor = parseDictInt(inner, "/Predictor")
if predictor == 0 {
predictor = 1
}
columns = parseDictInt(inner, "/Columns")
if columns == 0 {
columns = 1
}
colors = parseDictInt(inner, "/Colors")
if colors == 0 {
colors = 1
}
bits = parseDictInt(inner, "/BitsPerComponent")
if bits == 0 {
bits = 8
}
return
}
internal/leviathan/spectrogram/spectrogram.go
+798
View File
@@ -0,0 +1,798 @@
package spectrogram
import (
"bytes"
"encoding/binary"
"errors"
"fmt"
"image"
"image/color"
"image/png"
"io"
"math"
"path/filepath"
"strings"
_ "image/gif"
_ "image/jpeg"
_ "image/png"
"github.com/hajimehoshi/go-mp3"
)
const (
defaultSampleRate = 44100
defaultDuration = 5.0
// MaxImageBytes caps compressed image inputs before decode. Pixel count is
// checked separately because small compressed images can still expand huge.
MaxImageBytes = 64 << 20
// MaxAudioBytes caps encoded WAV/MP3 inputs before full in-memory decode.
MaxAudioBytes = 128 << 20
maxImagePixels = 24_000_000
maxDecodedBytes = 64 << 20
twoPi = 2 * math.Pi
)
// EncodeOptions controls image-to-audio spectrogram synthesis.
type EncodeOptions struct {
DurationSec float64
SampleRate int
MinHz float64
MaxHz float64
Bins int
Columns int
Invert bool
PreserveAR bool
Gain float64
}
// InsertOptions controls embedding synthesized spectrogram audio into a carrier.
type InsertOptions struct {
EncodeOptions
StartSec float64
MixLevel float64
}
// ViewOptions controls spectrogram image rendering from audio.
type ViewOptions struct {
FFTSize int
Height int
MaxWidth int
Aspect float64
MinHz float64
MaxHz float64
Brightness float64
Contrast float64
}
type audioBuffer struct {
SampleRate int
Channels int
Samples []float64
}
// ImageToWAV converts an image into a WAV whose spectrogram reveals that image.
func ImageToWAV(imageData []byte, opts EncodeOptions) ([]byte, map[string]any, error) {
grid, meta, opts, err := imageGrid(imageData, opts)
if err != nil {
return nil, nil, err
}
mono := synthesize(grid, opts)
audio := audioBuffer{SampleRate: opts.SampleRate, Channels: 1, Samples: mono}
wav, err := encodeWAV(audio)
if err != nil {
return nil, nil, err
}
summary := map[string]any{
"mode": "image-to-wav",
"imageFormat": meta["format"],
"imageWidth": meta["width"],
"imageHeight": meta["height"],
"imageAspect": meta["aspect"],
"durationSec": opts.DurationSec,
"sampleRate": opts.SampleRate,
"minHz": opts.MinHz,
"maxHz": opts.MaxHz,
"bins": opts.Bins,
"columns": opts.Columns,
"preserveAR": opts.PreserveAR,
"wavBytes": len(wav),
}
return wav, summary, nil
}
// InsertImageIntoAudio mixes a synthesized image spectrogram into WAV or MP3 carrier audio.
func InsertImageIntoAudio(imageData, audioData []byte, audioName string, opts InsertOptions) ([]byte, map[string]any, error) {
carrier, err := DecodeAudio(audioData, audioName)
if err != nil {
return nil, nil, err
}
opts.EncodeOptions.SampleRate = carrier.SampleRate
grid, meta, encodeOpts, err := imageGrid(imageData, opts.EncodeOptions)
if err != nil {
return nil, nil, err
}
tone := synthesize(grid, encodeOpts)
if opts.MixLevel <= 0 {
opts.MixLevel = 0.35
}
if opts.MixLevel > 1 {
opts.MixLevel = 1
}
if opts.StartSec < 0 {
opts.StartSec = 0
}
startFrame := int(math.Round(opts.StartSec * float64(carrier.SampleRate)))
requiredFrames := startFrame + len(tone)
currentFrames := len(carrier.Samples) / carrier.Channels
if requiredFrames > currentFrames {
extended := make([]float64, requiredFrames*carrier.Channels)
copy(extended, carrier.Samples)
carrier.Samples = extended
}
for i, sample := range tone {
frame := startFrame + i
for ch := 0; ch < carrier.Channels; ch++ {
idx := frame*carrier.Channels + ch
carrier.Samples[idx] += sample * opts.MixLevel
}
}
limitPeak(carrier.Samples, 0.98)
wav, err := encodeWAV(carrier)
if err != nil {
return nil, nil, err
}
summary := map[string]any{
"mode": "insert-into-audio",
"imageFormat": meta["format"],
"imageWidth": meta["width"],
"imageHeight": meta["height"],
"imageAspect": meta["aspect"],
"carrierSampleRate": carrier.SampleRate,
"carrierChannels": carrier.Channels,
"startSec": opts.StartSec,
"mixLevel": opts.MixLevel,
"durationSec": encodeOpts.DurationSec,
"minHz": encodeOpts.MinHz,
"maxHz": encodeOpts.MaxHz,
"bins": encodeOpts.Bins,
"columns": encodeOpts.Columns,
"preserveAR": encodeOpts.PreserveAR,
"wavBytes": len(wav),
}
return wav, summary, nil
}
// AudioSpectrogramPNG renders WAV or MP3 audio as a PNG spectrogram.
func AudioSpectrogramPNG(audioData []byte, audioName string, opts ViewOptions) ([]byte, map[string]any, error) {
audio, err := DecodeAudio(audioData, audioName)
if err != nil {
return nil, nil, err
}
mono := toMono(audio)
if len(mono) == 0 {
return nil, nil, errors.New("audio contains no samples")
}
opts = normalizeViewOptions(opts, audio.SampleRate)
img := renderSpectrogram(mono, audio.SampleRate, opts)
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
return nil, nil, err
}
duration := float64(len(mono)) / float64(audio.SampleRate)
summary := map[string]any{
"mode": "audio-to-spectrogram",
"sampleRate": audio.SampleRate,
"channels": audio.Channels,
"durationSec": duration,
"fftSize": opts.FFTSize,
"width": img.Bounds().Dx(),
"height": img.Bounds().Dy(),
"aspectRatio": opts.Aspect,
"dynamicRange": 80,
"minHz": opts.MinHz,
"maxHz": opts.MaxHz,
"pngBytes": buf.Len(),
}
return buf.Bytes(), summary, nil
}
// DecodeAudio decodes WAV directly and MP3 through go-mp3.
func DecodeAudio(data []byte, name string) (audioBuffer, error) {
if len(data) == 0 {
return audioBuffer{}, errors.New("audio file is empty")
}
if len(data) > MaxAudioBytes {
return audioBuffer{}, fmt.Errorf("audio input is too large: %d bytes exceeds %d bytes", len(data), MaxAudioBytes)
}
if isWAV(data) {
return decodeWAV(data)
}
ext := strings.ToLower(filepath.Ext(name))
if ext == ".mp3" || isMP3(data) {
return decodeMP3(data)
}
return audioBuffer{}, errors.New("unsupported audio format: load a WAV or MP3 file")
}
func imageGrid(data []byte, opts EncodeOptions) ([][]float64, map[string]any, EncodeOptions, error) {
if len(data) == 0 {
return nil, nil, opts, errors.New("image file is empty")
}
if len(data) > MaxImageBytes {
return nil, nil, opts, fmt.Errorf("image input is too large: %d bytes exceeds %d bytes", len(data), MaxImageBytes)
}
cfg, format, err := image.DecodeConfig(bytes.NewReader(data))
if err != nil {
return nil, nil, opts, fmt.Errorf("decode image config: %w", err)
}
if cfg.Width <= 0 || cfg.Height <= 0 {
return nil, nil, opts, errors.New("image has no pixels")
}
if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels {
return nil, nil, opts, fmt.Errorf("image dimensions are too large: %dx%d exceeds %d pixels", cfg.Width, cfg.Height, maxImagePixels)
}
img, format, err := image.Decode(bytes.NewReader(data))
if err != nil {
return nil, nil, opts, fmt.Errorf("decode image: %w", err)
}
bounds := img.Bounds()
if bounds.Dx() <= 0 || bounds.Dy() <= 0 {
return nil, nil, opts, errors.New("image has no pixels")
}
opts = normalizeEncodeOptions(opts, bounds.Dx(), bounds.Dy())
grid := make([][]float64, opts.Bins)
for bin := 0; bin < opts.Bins; bin++ {
grid[bin] = make([]float64, opts.Columns)
sourceY := bounds.Min.Y + int((float64(opts.Bins-1-bin)+0.5)*float64(bounds.Dy())/float64(opts.Bins))
if sourceY >= bounds.Max.Y {
sourceY = bounds.Max.Y - 1
}
for col := 0; col < opts.Columns; col++ {
sourceX := bounds.Min.X + int((float64(col)+0.5)*float64(bounds.Dx())/float64(opts.Columns))
if sourceX >= bounds.Max.X {
sourceX = bounds.Max.X - 1
}
grid[bin][col] = pixelAmplitude(img.At(sourceX, sourceY), opts.Invert)
}
}
meta := map[string]any{"format": format, "width": bounds.Dx(), "height": bounds.Dy(), "aspect": float64(bounds.Dy()) / float64(bounds.Dx())}
return grid, meta, opts, nil
}
func normalizeEncodeOptions(opts EncodeOptions, imageWidth, imageHeight int) EncodeOptions {
if opts.DurationSec <= 0 {
opts.DurationSec = defaultDuration
}
opts.DurationSec = clampFloat(opts.DurationSec, 0.25, 60)
if opts.SampleRate <= 0 {
opts.SampleRate = defaultSampleRate
}
opts.SampleRate = clampInt(opts.SampleRate, 8000, 96000)
if opts.MinHz <= 0 {
opts.MinHz = 200
}
if opts.MaxHz <= opts.MinHz {
opts.MaxHz = math.Min(float64(opts.SampleRate)/2-200, 20000)
}
opts.MinHz = clampFloat(opts.MinHz, 20, float64(opts.SampleRate)/2-100)
opts.MaxHz = clampFloat(opts.MaxHz, opts.MinHz+100, float64(opts.SampleRate)/2-50)
if opts.PreserveAR && imageWidth > 0 && imageHeight > 0 {
aspect := float64(imageHeight) / float64(imageWidth)
baseRange := opts.MaxHz - opts.MinHz
desiredRange := baseRange * aspect
opts.MaxHz = clampFloat(opts.MinHz+desiredRange, opts.MinHz+100, float64(opts.SampleRate)/2-50)
}
if opts.Bins <= 0 {
opts.Bins = clampInt(imageHeight, 48, 192)
}
opts.Bins = clampInt(opts.Bins, 16, 384)
if opts.Columns <= 0 {
opts.Columns = clampInt(imageWidth, 64, 1024)
}
opts.Columns = clampInt(opts.Columns, 16, 2048)
if opts.Gain <= 0 {
opts.Gain = 0.85
}
opts.Gain = clampFloat(opts.Gain, 0.05, 1)
return opts
}
func normalizeViewOptions(opts ViewOptions, sampleRate int) ViewOptions {
if opts.FFTSize <= 0 {
opts.FFTSize = 2048
}
opts.FFTSize = nearestPowerOfTwo(clampInt(opts.FFTSize, 256, 8192))
if opts.Height <= 0 {
opts.Height = 512
}
opts.Height = clampInt(opts.Height, 64, 1024)
if opts.MaxWidth <= 0 {
opts.MaxWidth = 1400
}
opts.MaxWidth = clampInt(opts.MaxWidth, 64, 2400)
if opts.Aspect <= 0 {
opts.Aspect = 1
}
opts.Aspect = clampFloat(opts.Aspect, 0.25, 4)
if opts.MinHz < 0 {
opts.MinHz = 0
}
if opts.MaxHz <= opts.MinHz {
opts.MaxHz = float64(sampleRate) / 2
}
opts.MinHz = clampFloat(opts.MinHz, 0, float64(sampleRate)/2-10)
opts.MaxHz = clampFloat(opts.MaxHz, opts.MinHz+10, float64(sampleRate)/2)
if opts.Brightness <= 0 {
opts.Brightness = 1
}
opts.Brightness = clampFloat(opts.Brightness, 0.1, 4)
if opts.Contrast <= 0 {
opts.Contrast = 1
}
opts.Contrast = clampFloat(opts.Contrast, 0.1, 4)
return opts
}
func pixelAmplitude(c color.Color, invert bool) float64 {
r, g, b, a := c.RGBA()
alpha := float64(a) / 65535
if alpha <= 0.001 {
return 0
}
luma := (0.2126*float64(r) + 0.7152*float64(g) + 0.0722*float64(b)) / 65535
amp := luma
if invert {
amp = 1 - luma
}
amp *= alpha
return math.Pow(clampFloat(amp, 0, 1), 1.15)
}
func synthesize(grid [][]float64, opts EncodeOptions) []float64 {
bins := len(grid)
cols := len(grid[0])
frames := int(math.Round(opts.DurationSec * float64(opts.SampleRate)))
if frames < 1 {
frames = 1
}
out := make([]float64, frames)
phase := make([]float64, bins)
inc := make([]float64, bins)
for i := range inc {
var pos float64
if bins > 1 {
pos = float64(i) / float64(bins-1)
}
freq := opts.MinHz + (opts.MaxHz-opts.MinHz)*pos
inc[i] = twoPi * freq / float64(opts.SampleRate)
}
for i := 0; i < frames; i++ {
var x float64
if frames > 1 {
x = float64(i) * float64(cols-1) / float64(frames-1)
}
c0 := int(math.Floor(x))
c1 := c0 + 1
if c1 >= cols {
c1 = cols - 1
}
t := x - float64(c0)
var sample, weight float64
for bin := 0; bin < bins; bin++ {
amp := grid[bin][c0]*(1-t) + grid[bin][c1]*t
if amp < 0.002 {
continue
}
phase[bin] += inc[bin]
if phase[bin] > twoPi {
phase[bin] -= twoPi
}
sample += math.Sin(phase[bin]) * amp
weight += amp
}
if weight > 1 {
sample /= math.Sqrt(weight)
}
out[i] = sample
}
normalizePeak(out, opts.Gain)
fade(out, opts.SampleRate/100)
return out
}
func renderSpectrogram(samples []float64, sampleRate int, opts ViewOptions) image.Image {
fftSize := opts.FFTSize
hop := fftSize / 4
if hop < 1 {
hop = 1
}
frameCount := 1
if len(samples) > fftSize {
frameCount = 1 + (len(samples)-fftSize)/hop
}
width := frameCount
if width > opts.MaxWidth {
width = opts.MaxWidth
}
if width < 1 {
width = 1
}
height := clampInt(int(math.Round(float64(opts.Height)*opts.Aspect)), 32, 4096)
img := image.NewRGBA(image.Rect(0, 0, width, height))
values := make([]float32, width*height)
maxDB := math.Inf(-1)
window := hamming(fftSize)
for x := 0; x < width; x++ {
frameIndex := 0
if width > 1 && frameCount > 1 {
frameIndex = int(math.Round(float64(x) * float64(frameCount-1) / float64(width-1)))
}
start := frameIndex * hop
buf := make([]complex128, fftSize)
for i := 0; i < fftSize; i++ {
if start+i < len(samples) {
buf[i] = complex(samples[start+i]*window[i], 0)
}
}
fft(buf)
for y := 0; y < height; y++ {
pos := 0.0
if height > 1 {
pos = float64(height-1-y) / float64(height-1)
}
freq := opts.MinHz + (opts.MaxHz-opts.MinHz)*pos
bin := int(math.Round(freq * float64(fftSize) / float64(sampleRate)))
if bin < 0 {
bin = 0
}
if bin >= fftSize/2 {
bin = fftSize/2 - 1
}
mag := cmplxAbs(buf[bin]) / float64(fftSize)
db := 20 * math.Log10(mag+1e-10)
values[y*width+x] = float32(db)
if db > maxDB {
maxDB = db
}
}
}
if math.IsInf(maxDB, -1) {
maxDB = -80
}
minDB := maxDB - 80
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
db := float64(values[y*width+x])
norm := clampFloat((db-minDB)/(maxDB-minDB), 0, 1)
norm = clampFloat((norm-0.5)*opts.Contrast+0.5, 0, 1)
norm = clampFloat(norm*opts.Brightness, 0, 1)
img.Set(x, y, palette(norm))
}
}
return img
}
func decodeWAV(data []byte) (audioBuffer, error) {
if !isWAV(data) {
return audioBuffer{}, errors.New("not a WAV file")
}
var fmtChunk, dataChunk []byte
pos := 12
for pos+8 <= len(data) {
id := string(data[pos : pos+4])
size := int(binary.LittleEndian.Uint32(data[pos+4 : pos+8]))
pos += 8
if size < 0 || pos+size > len(data) {
return audioBuffer{}, errors.New("invalid WAV chunk size")
}
chunk := data[pos : pos+size]
switch id {
case "fmt ":
fmtChunk = chunk
case "data":
dataChunk = chunk
}
pos += size
if size%2 == 1 {
pos++
}
}
if len(fmtChunk) < 16 || len(dataChunk) == 0 {
return audioBuffer{}, errors.New("WAV is missing fmt or data chunk")
}
audioFormat := binary.LittleEndian.Uint16(fmtChunk[0:2])
channels := int(binary.LittleEndian.Uint16(fmtChunk[2:4]))
sampleRate := int(binary.LittleEndian.Uint32(fmtChunk[4:8]))
blockAlign := int(binary.LittleEndian.Uint16(fmtChunk[12:14]))
bits := int(binary.LittleEndian.Uint16(fmtChunk[14:16]))
if channels < 1 || channels > 8 || sampleRate < 1 || blockAlign < channels {
return audioBuffer{}, errors.New("unsupported WAV stream parameters")
}
if len(dataChunk) > maxDecodedBytes {
return audioBuffer{}, errors.New("audio data is too large to decode safely")
}
bytesPerSample := bits / 8
if bytesPerSample < 1 || blockAlign < channels*bytesPerSample {
return audioBuffer{}, errors.New("unsupported WAV bit depth")
}
frames := len(dataChunk) / blockAlign
samples := make([]float64, frames*channels)
for frame := 0; frame < frames; frame++ {
base := frame * blockAlign
for ch := 0; ch < channels; ch++ {
off := base + ch*bytesPerSample
samples[frame*channels+ch] = decodeWAVSample(dataChunk[off:], audioFormat, bits)
}
}
return audioBuffer{SampleRate: sampleRate, Channels: channels, Samples: samples}, nil
}
func decodeWAVSample(b []byte, audioFormat uint16, bits int) float64 {
switch audioFormat {
case 1:
switch bits {
case 8:
return (float64(b[0]) - 128) / 128
case 16:
return float64(int16(binary.LittleEndian.Uint16(b[:2]))) / 32768
case 24:
v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16
if v&0x800000 != 0 {
v |= ^0xffffff
}
return float64(v) / 8388608
case 32:
return float64(int32(binary.LittleEndian.Uint32(b[:4]))) / 2147483648
}
case 3:
switch bits {
case 32:
return float64(math.Float32frombits(binary.LittleEndian.Uint32(b[:4])))
case 64:
return math.Float64frombits(binary.LittleEndian.Uint64(b[:8]))
}
}
return 0
}
func decodeMP3(data []byte) (audioBuffer, error) {
decoder, err := mp3.NewDecoder(bytes.NewReader(data))
if err != nil {
return audioBuffer{}, err
}
limited := io.LimitReader(decoder, maxDecodedBytes+1)
pcm, err := io.ReadAll(limited)
if err != nil {
return audioBuffer{}, err
}
if len(pcm) > maxDecodedBytes {
return audioBuffer{}, errors.New("decoded MP3 is too large to process safely")
}
if len(pcm) < 2 {
return audioBuffer{}, errors.New("MP3 decoded no samples")
}
sampleCount := len(pcm) / 2
channels := 2
if sampleCount%2 != 0 {
channels = 1
}
samples := make([]float64, sampleCount)
for i := 0; i < sampleCount; i++ {
samples[i] = float64(int16(binary.LittleEndian.Uint16(pcm[i*2:i*2+2]))) / 32768
}
return audioBuffer{SampleRate: decoder.SampleRate(), Channels: channels, Samples: samples}, nil
}
func encodeWAV(audio audioBuffer) ([]byte, error) {
if audio.SampleRate <= 0 || audio.Channels <= 0 {
return nil, errors.New("invalid audio parameters")
}
var buf bytes.Buffer
dataSize := int64(len(audio.Samples)) * 2
if dataSize > int64(^uint32(0))-44 {
return nil, errors.New("audio is too large for a WAV file")
}
buf.WriteString("RIFF")
_ = binary.Write(&buf, binary.LittleEndian, uint32(36+dataSize))
buf.WriteString("WAVE")
buf.WriteString("fmt ")
_ = binary.Write(&buf, binary.LittleEndian, uint32(16))
_ = binary.Write(&buf, binary.LittleEndian, uint16(1))
_ = binary.Write(&buf, binary.LittleEndian, uint16(audio.Channels))
_ = binary.Write(&buf, binary.LittleEndian, uint32(audio.SampleRate))
byteRate := audio.SampleRate * audio.Channels * 2
_ = binary.Write(&buf, binary.LittleEndian, uint32(byteRate))
_ = binary.Write(&buf, binary.LittleEndian, uint16(audio.Channels*2))
_ = binary.Write(&buf, binary.LittleEndian, uint16(16))
buf.WriteString("data")
_ = binary.Write(&buf, binary.LittleEndian, uint32(dataSize))
for _, sample := range audio.Samples {
sample = clampFloat(sample, -1, 1)
_ = binary.Write(&buf, binary.LittleEndian, int16(math.Round(sample*32767)))
}
return buf.Bytes(), nil
}
func toMono(audio audioBuffer) []float64 {
if audio.Channels == 1 {
out := make([]float64, len(audio.Samples))
copy(out, audio.Samples)
return out
}
frames := len(audio.Samples) / audio.Channels
out := make([]float64, frames)
for frame := 0; frame < frames; frame++ {
var sum float64
for ch := 0; ch < audio.Channels; ch++ {
sum += audio.Samples[frame*audio.Channels+ch]
}
out[frame] = sum / float64(audio.Channels)
}
return out
}
func isWAV(data []byte) bool {
return len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WAVE"
}
func isMP3(data []byte) bool {
return len(data) >= 3 && (string(data[:3]) == "ID3" || (data[0] == 0xff && data[1]&0xe0 == 0xe0))
}
func normalizePeak(samples []float64, target float64) {
var peak float64
for _, s := range samples {
if a := math.Abs(s); a > peak {
peak = a
}
}
if peak <= 0 {
return
}
scale := target / peak
for i := range samples {
samples[i] *= scale
}
}
func limitPeak(samples []float64, target float64) {
var peak float64
for _, s := range samples {
if a := math.Abs(s); a > peak {
peak = a
}
}
if peak <= target || peak <= 0 {
return
}
scale := target / peak
for i := range samples {
samples[i] *= scale
}
}
func fade(samples []float64, fadeLen int) {
if fadeLen <= 0 || len(samples) == 0 {
return
}
if fadeLen*2 > len(samples) {
fadeLen = len(samples) / 2
}
for i := 0; i < fadeLen; i++ {
gain := float64(i) / float64(fadeLen)
samples[i] *= gain
samples[len(samples)-1-i] *= gain
}
}
func hamming(n int) []float64 {
w := make([]float64, n)
if n <= 1 {
if n == 1 {
w[0] = 1
}
return w
}
for i := range w {
w[i] = 0.54 - 0.46*math.Cos(twoPi*float64(i)/float64(n-1))
}
return w
}
func fft(a []complex128) {
n := len(a)
j := 0
for i := 1; i < n; i++ {
bit := n >> 1
for ; j&bit != 0; bit >>= 1 {
j &^= bit
}
j |= bit
if i < j {
a[i], a[j] = a[j], a[i]
}
}
for length := 2; length <= n; length <<= 1 {
angle := -twoPi / float64(length)
wlen := complex(math.Cos(angle), math.Sin(angle))
for i := 0; i < n; i += length {
w := complex(1, 0)
for k := 0; k < length/2; k++ {
u := a[i+k]
v := a[i+k+length/2] * w
a[i+k] = u + v
a[i+k+length/2] = u - v
w *= wlen
}
}
}
}
func cmplxAbs(c complex128) float64 {
return math.Hypot(real(c), imag(c))
}
func palette(v float64) color.RGBA {
v = clampFloat(v, 0, 1)
colors := []color.RGBA{
{R: 68, G: 1, B: 84, A: 255},
{R: 72, G: 40, B: 120, A: 255},
{R: 62, G: 73, B: 137, A: 255},
{R: 49, G: 104, B: 142, A: 255},
{R: 38, G: 130, B: 142, A: 255},
{R: 31, G: 158, B: 137, A: 255},
{R: 53, G: 183, B: 121, A: 255},
{R: 110, G: 206, B: 88, A: 255},
{R: 181, G: 222, B: 43, A: 255},
{R: 253, G: 231, B: 37, A: 255},
}
scaled := v * float64(len(colors)-1)
idx := int(math.Floor(scaled))
if idx >= len(colors)-1 {
return colors[len(colors)-1]
}
t := scaled - float64(idx)
a, b := colors[idx], colors[idx+1]
return color.RGBA{
R: uint8(float64(a.R) + (float64(b.R)-float64(a.R))*t),
G: uint8(float64(a.G) + (float64(b.G)-float64(a.G))*t),
B: uint8(float64(a.B) + (float64(b.B)-float64(a.B))*t),
A: 255,
}
}
func nearestPowerOfTwo(n int) int {
if n <= 1 {
return 1
}
p := 1
for p < n {
p <<= 1
}
prev := p >> 1
if p-n < n-prev {
return p
}
return prev
}
func clampInt(v, min, max int) int {
if v < min {
return min
}
if v > max {
return max
}
return v
}
func clampFloat(v, min, max float64) float64 {
if v < min {
return min
}
if v > max {
return max
}
return v
}
internal/leviathan/spectrogram/spectrogram_test.go
+177
View File
@@ -0,0 +1,177 @@
package spectrogram
import (
"bytes"
"encoding/binary"
"hash/crc32"
"image"
"image/color"
"image/png"
"testing"
)
func TestImageToWAVAndRenderSpectrogram(t *testing.T) {
img := testPNG(t)
wav, summary, err := ImageToWAV(img, EncodeOptions{
DurationSec: 0.25,
SampleRate: 8000,
MinHz: 300,
MaxHz: 3000,
Bins: 32,
Columns: 32,
Invert: true,
})
if err != nil {
t.Fatalf("ImageToWAV failed: %v", err)
}
if !isWAV(wav) {
t.Fatalf("expected WAV header")
}
if summary["wavBytes"].(int) != len(wav) {
t.Fatalf("summary wavBytes mismatch")
}
rendered, viewSummary, err := AudioSpectrogramPNG(wav, "test.wav", ViewOptions{
FFTSize: 512,
Height: 96,
Aspect: 2,
MaxWidth: 128,
})
if err != nil {
t.Fatalf("AudioSpectrogramPNG failed: %v", err)
}
if _, err := png.Decode(bytes.NewReader(rendered)); err != nil {
t.Fatalf("rendered spectrogram is not a PNG: %v", err)
}
if viewSummary["sampleRate"].(int) != 8000 {
t.Fatalf("unexpected sample rate: %v", viewSummary["sampleRate"])
}
if viewSummary["height"].(int) != 192 {
t.Fatalf("expected aspect-adjusted height 192, got %v", viewSummary["height"])
}
}
func TestPreserveAspectRatioAdjustsFrequencyRange(t *testing.T) {
img := testPNGSize(t, 32, 16)
_, summary, err := ImageToWAV(img, EncodeOptions{
DurationSec: 0.1,
SampleRate: 8000,
MinHz: 200,
MaxHz: 3000,
Bins: 16,
Columns: 16,
PreserveAR: true,
Invert: true,
})
if err != nil {
t.Fatalf("ImageToWAV failed: %v", err)
}
if summary["preserveAR"] != true {
t.Fatalf("expected preserveAR summary")
}
if got := summary["maxHz"].(float64); got != 1600 {
t.Fatalf("expected landscape image to halve frequency range to 1600Hz, got %v", got)
}
}
func TestImageToWAVRejectsOversizedImageDimensionsBeforeDecode(t *testing.T) {
_, _, err := ImageToWAV(hugePNGHeader(t, 100000, 100000), EncodeOptions{})
if err == nil {
t.Fatal("expected oversized image dimensions to be rejected")
}
if !bytes.Contains([]byte(err.Error()), []byte("image dimensions are too large")) {
t.Fatalf("unexpected error: %v", err)
}
}
func TestInsertImageIntoAudioExtendsCarrier(t *testing.T) {
img := testPNG(t)
carrier, _, err := ImageToWAV(img, EncodeOptions{
DurationSec: 0.1,
SampleRate: 8000,
MinHz: 300,
MaxHz: 2500,
Bins: 16,
Columns: 16,
Invert: true,
})
if err != nil {
t.Fatalf("carrier generation failed: %v", err)
}
out, summary, err := InsertImageIntoAudio(img, carrier, "carrier.wav", InsertOptions{
EncodeOptions: EncodeOptions{
DurationSec: 0.2,
MinHz: 300,
MaxHz: 2500,
Bins: 16,
Columns: 16,
Invert: true,
},
StartSec: 0.05,
MixLevel: 0.25,
})
if err != nil {
t.Fatalf("InsertImageIntoAudio failed: %v", err)
}
if !isWAV(out) {
t.Fatalf("expected WAV output")
}
if summary["mode"] != "insert-into-audio" {
t.Fatalf("unexpected mode: %v", summary["mode"])
}
if len(out) <= len(carrier) {
t.Fatalf("expected carrier to be extended")
}
}
func testPNG(t *testing.T) []byte {
t.Helper()
return testPNGSize(t, 32, 32)
}
func testPNGSize(t *testing.T, width, height int) []byte {
t.Helper()
img := image.NewRGBA(image.Rect(0, 0, width, height))
for y := 0; y < height; y++ {
for x := 0; x < width; x++ {
img.Set(x, y, color.RGBA{R: 255, G: 255, B: 255, A: 255})
}
}
limit := width
if height < limit {
limit = height
}
for i := 0; i < limit; i++ {
img.Set(i, i, color.RGBA{A: 255})
img.Set(width-1-i, i, color.RGBA{A: 255})
}
var buf bytes.Buffer
if err := png.Encode(&buf, img); err != nil {
t.Fatalf("encode png: %v", err)
}
return buf.Bytes()
}
func hugePNGHeader(t *testing.T, width, height uint32) []byte {
t.Helper()
var buf bytes.Buffer
buf.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'})
ihdr := make([]byte, 13)
binary.BigEndian.PutUint32(ihdr[0:4], width)
binary.BigEndian.PutUint32(ihdr[4:8], height)
ihdr[8] = 8
ihdr[9] = 2
writePNGChunk(&buf, "IHDR", ihdr)
writePNGChunk(&buf, "IEND", nil)
return buf.Bytes()
}
func writePNGChunk(buf *bytes.Buffer, kind string, data []byte) {
_ = binary.Write(buf, binary.BigEndian, uint32(len(data)))
buf.WriteString(kind)
buf.Write(data)
crc := crc32.NewIEEE()
crc.Write([]byte(kind))
crc.Write(data)
_ = binary.Write(buf, binary.BigEndian, crc.Sum32())
}
internal/leviathan/squashfs/squashfs.go
+782
View File
@@ -0,0 +1,782 @@
// Package squashfs implements a read-only SquashFS 4.0 extractor.
//
// Scope is deliberately pragmatic: gzip, xz and zstd-compressed images,
// basic and extended file and directory inodes, symlink inodes. Other
// compressors (lzma/lzo/lz4) return a structured error so the UI can show
// what's missing.
//
// The module never trusts its input: every table offset and length is
// bounded against the image size and a hard cap on uncompressed metadata
// block size. A malformed or truncated image returns an error rather
// than causing a wild read.
package squashfs
import (
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"fmt"
"io"
"path"
"sort"
"github.com/andydixon/nexus/internal/leviathan/document"
"github.com/klauspost/compress/zstd"
"github.com/ulikunitz/xz"
)
const (
magicLE uint32 = 0x73717368 // "hsqs"
metadataBlockSize = 8192
dataBlockMax = 1 << 20 // 1 MiB guard for a single data block
maxEntries = 1 << 20 // total files + dirs we're willing to walk
)
// Compression IDs from the spec.
const (
compGZIP = 1
compLZMA = 2
compLZO = 3
compXZ = 4
compLZ4 = 5
compZSTD = 6
)
// Inode type codes.
const (
itBasicDir = 1
itBasicFile = 2
itBasicSymlink = 3
itBasicBlockDev = 4
itBasicCharDev = 5
itBasicFifo = 6
itBasicSocket = 7
itExtDir = 8
itExtFile = 9
itExtSymlink = 10
itExtBlockDev = 11
itExtCharDev = 12
itExtFifo = 13
itExtSocket = 14
)
// Superblock is the parsed 96-byte header.
type Superblock struct {
Magic uint32
InodeCount uint32
ModTime uint32
BlockSize uint32
FragCount uint32
CompressionID uint16
BlockLog uint16
Flags uint16
IDCount uint16
VersionMajor uint16
VersionMinor uint16
RootInodeRef uint64
BytesUsed uint64
IDTableStart uint64
XattrTableStart uint64
InodeTable uint64
DirTable uint64
FragTable uint64
ExportTable uint64
}
// Entry describes a filesystem entry discovered by the walker.
type Entry struct {
Path string
Type string // "file", "dir", "symlink", "special"
Size int64
Mode uint16
UID uint16 // id_idx (not resolved)
GID uint16
Mtime uint32
InodeNumber uint32
InodeRef uint64 // 64-bit metadata reference (for targeted extraction)
LinkTarget string // for symlinks
// For file extraction; zero for non-files.
blocksStart uint64
fileSize int64
fragIndex uint32
fragOffset uint32
blockSizes []uint32 // raw (top bit=uncompressed)
}
// FS is the opened image handle.
type FS struct {
d *document.Document
sb Superblock
}
// Open validates the superblock and returns an FS for further queries.
func Open(d *document.Document) (*FS, error) {
head, err := d.Range(0, 96)
if err != nil {
return nil, fmt.Errorf("read superblock: %w", err)
}
sb, err := parseSuperblock(head)
if err != nil {
return nil, err
}
if sb.VersionMajor != 4 {
return nil, fmt.Errorf("unsupported squashfs version %d.%d (only 4.x)", sb.VersionMajor, sb.VersionMinor)
}
if sb.BytesUsed > uint64(d.Size()) {
return nil, fmt.Errorf("superblock bytes_used=%d exceeds document size=%d", sb.BytesUsed, d.Size())
}
return &FS{d: d, sb: sb}, nil
}
// Superblock returns the parsed header (copy).
func (f *FS) Superblock() Superblock { return f.sb }
// CompressionName returns the human name of the image compressor.
func (f *FS) CompressionName() string {
return compressionName(f.sb.CompressionID)
}
func compressionName(id uint16) string {
switch id {
case compGZIP:
return "gzip"
case compLZMA:
return "lzma"
case compLZO:
return "lzo"
case compXZ:
return "xz"
case compLZ4:
return "lz4"
case compZSTD:
return "zstd"
}
return fmt.Sprintf("unknown(%d)", id)
}
func parseSuperblock(b []byte) (Superblock, error) {
var s Superblock
if len(b) < 96 {
return s, errors.New("superblock: short read")
}
s.Magic = binary.LittleEndian.Uint32(b[0:4])
if s.Magic != magicLE {
return s, fmt.Errorf("bad magic 0x%08x (want 0x%08x)", s.Magic, magicLE)
}
s.InodeCount = binary.LittleEndian.Uint32(b[4:8])
s.ModTime = binary.LittleEndian.Uint32(b[8:12])
s.BlockSize = binary.LittleEndian.Uint32(b[12:16])
s.FragCount = binary.LittleEndian.Uint32(b[16:20])
s.CompressionID = binary.LittleEndian.Uint16(b[20:22])
s.BlockLog = binary.LittleEndian.Uint16(b[22:24])
s.Flags = binary.LittleEndian.Uint16(b[24:26])
s.IDCount = binary.LittleEndian.Uint16(b[26:28])
s.VersionMajor = binary.LittleEndian.Uint16(b[28:30])
s.VersionMinor = binary.LittleEndian.Uint16(b[30:32])
s.RootInodeRef = binary.LittleEndian.Uint64(b[32:40])
s.BytesUsed = binary.LittleEndian.Uint64(b[40:48])
s.IDTableStart = binary.LittleEndian.Uint64(b[48:56])
s.XattrTableStart = binary.LittleEndian.Uint64(b[56:64])
s.InodeTable = binary.LittleEndian.Uint64(b[64:72])
s.DirTable = binary.LittleEndian.Uint64(b[72:80])
s.FragTable = binary.LittleEndian.Uint64(b[80:88])
s.ExportTable = binary.LittleEndian.Uint64(b[88:96])
return s, nil
}
// ---------- metadata block reader ----------
func (f *FS) decompress(buf []byte) ([]byte, error) {
return f.decompressWithLimit(buf, metadataBlockSize+16)
}
func (f *FS) decompressWithLimit(buf []byte, limit int64) ([]byte, error) {
var r io.ReadCloser
switch f.sb.CompressionID {
case compGZIP:
zr, err := zlib.NewReader(bytes.NewReader(buf))
if err != nil {
return nil, fmt.Errorf("zlib init: %w", err)
}
r = zr
case compXZ:
xr, err := xz.NewReader(bytes.NewReader(buf))
if err != nil {
return nil, fmt.Errorf("xz init: %w", err)
}
r = io.NopCloser(xr)
case compZSTD:
zr, err := zstd.NewReader(bytes.NewReader(buf))
if err != nil {
return nil, fmt.Errorf("zstd init: %w", err)
}
r = zr.IOReadCloser()
default:
return nil, fmt.Errorf("compressor %q not implemented in this build", compressionName(f.sb.CompressionID))
}
defer r.Close()
return readLimited(r, limit)
}
// readMetadataBlock reads one metadata block starting at absolute offset
// `off` and returns (data, bytesConsumed, error). bytesConsumed includes
// the 2-byte header.
func (f *FS) readMetadataBlock(off int64) ([]byte, int64, error) {
if off < 0 || off+2 > f.d.Size() {
return nil, 0, fmt.Errorf("metadata block offset %d out of range", off)
}
hdrBuf, err := f.d.Range(off, 2)
if err != nil {
return nil, 0, err
}
hdr := binary.LittleEndian.Uint16(hdrBuf)
compressed := hdr&0x8000 == 0
size := int64(hdr & 0x7fff)
if size == 0 || size > metadataBlockSize {
return nil, 0, fmt.Errorf("metadata block size %d invalid", size)
}
body, err := f.d.Range(off+2, size)
if err != nil {
return nil, 0, err
}
if !compressed {
return append([]byte(nil), body...), size + 2, nil
}
out, err := f.decompress(body)
if err != nil {
return nil, 0, err
}
return out, size + 2, nil
}
// metaCursor streams uncompressed data out of a chain of metadata blocks.
// table[i] is the absolute file offset of the i-th metadata block, plus
// a running byte cursor within the uncompressed stream.
type metaCursor struct {
f *FS
base int64 // absolute file offset of first metadata block
blkAbs int64 // absolute file offset of *current* metadata block
inBlk int // index within current uncompressed block
buffer []byte // current decompressed block
}
func (f *FS) metaCursorAt(base int64, blockOffset uint32, inBlk uint16) (*metaCursor, error) {
mc := &metaCursor{f: f, base: base, blkAbs: base + int64(blockOffset)}
if err := mc.load(); err != nil {
return nil, err
}
if int(inBlk) > len(mc.buffer) {
return nil, fmt.Errorf("inode intra-block offset %d > block length %d", inBlk, len(mc.buffer))
}
mc.inBlk = int(inBlk)
return mc, nil
}
func (mc *metaCursor) load() error {
data, _, err := mc.f.readMetadataBlock(mc.blkAbs)
if err != nil {
return err
}
mc.buffer = data
mc.inBlk = 0
return nil
}
// advance one metadata block. The on-disk size is the consumed length of
// the current compressed block, which we must read from the header again.
func (mc *metaCursor) nextBlock() error {
// Re-read just the 2-byte header of current block to find its length.
hdrBuf, err := mc.f.d.Range(mc.blkAbs, 2)
if err != nil {
return err
}
hdr := binary.LittleEndian.Uint16(hdrBuf)
size := int64(hdr & 0x7fff)
mc.blkAbs += 2 + size
return mc.load()
}
// readN returns n uncompressed bytes, crossing metadata-block boundaries
// as required.
func (mc *metaCursor) readN(n int) ([]byte, error) {
if n <= 0 {
return nil, nil
}
out := make([]byte, 0, n)
for n > 0 {
if mc.inBlk >= len(mc.buffer) {
if err := mc.nextBlock(); err != nil {
return nil, err
}
}
take := len(mc.buffer) - mc.inBlk
if take > n {
take = n
}
out = append(out, mc.buffer[mc.inBlk:mc.inBlk+take]...)
mc.inBlk += take
n -= take
}
return out, nil
}
// ---------- inode ----------
type inodeHeader struct {
Type uint16
Mode uint16
UID uint16
GID uint16
Mtime uint32
Num uint32
}
func readInodeHeader(b []byte) (inodeHeader, error) {
var h inodeHeader
if len(b) < 16 {
return h, errors.New("inode header short")
}
h.Type = binary.LittleEndian.Uint16(b[0:2])
h.Mode = binary.LittleEndian.Uint16(b[2:4])
h.UID = binary.LittleEndian.Uint16(b[4:6])
h.GID = binary.LittleEndian.Uint16(b[6:8])
h.Mtime = binary.LittleEndian.Uint32(b[8:12])
h.Num = binary.LittleEndian.Uint32(b[12:16])
return h, nil
}
// readInodeAt reads the inode described by a 64-bit ref and returns the
// filled Entry plus the metadata location of the directory contents (if
// it's a directory).
type inodeContent struct {
entry Entry
dirStart uint32 // offset into DirTable (bytes from sb.DirTable)
dirBlockOff uint16 // intra-metadata-block offset for first dir header
dirSize uint32 // directory listing size, real value (file_size - 3)
parentInode uint32
}
func (f *FS) readInode(ref uint64) (*inodeContent, error) {
blkOff := uint32(ref >> 16)
inBlk := uint16(ref & 0xffff)
mc, err := f.metaCursorAt(int64(f.sb.InodeTable), blkOff, inBlk)
if err != nil {
return nil, err
}
hdrBytes, err := mc.readN(16)
if err != nil {
return nil, err
}
h, err := readInodeHeader(hdrBytes)
if err != nil {
return nil, err
}
ic := &inodeContent{
entry: Entry{
Mode: h.Mode,
UID: h.UID,
GID: h.GID,
Mtime: h.Mtime,
InodeNumber: h.Num,
InodeRef: ref,
},
}
switch h.Type {
case itBasicDir:
body, err := mc.readN(16)
if err != nil {
return nil, err
}
ic.entry.Type = "dir"
ic.dirStart = binary.LittleEndian.Uint32(body[0:4])
// nlink := binary.LittleEndian.Uint32(body[4:8])
fileSize := binary.LittleEndian.Uint16(body[8:10])
ic.dirBlockOff = binary.LittleEndian.Uint16(body[10:12])
ic.parentInode = binary.LittleEndian.Uint32(body[12:16])
// file_size in dir inodes is real dir bytes + 3. Zero-size (3) → empty dir.
if fileSize >= 3 {
ic.dirSize = uint32(fileSize) - 3
}
case itExtDir:
body, err := mc.readN(24)
if err != nil {
return nil, err
}
ic.entry.Type = "dir"
// nlink := binary.LittleEndian.Uint32(body[0:4])
fileSize := binary.LittleEndian.Uint32(body[4:8])
ic.dirStart = binary.LittleEndian.Uint32(body[8:12])
ic.parentInode = binary.LittleEndian.Uint32(body[12:16])
indexCount := binary.LittleEndian.Uint16(body[16:18])
ic.dirBlockOff = binary.LittleEndian.Uint16(body[18:20])
// xattr := binary.LittleEndian.Uint32(body[20:24])
if fileSize >= 3 {
ic.dirSize = fileSize - 3
}
// Skip index entries: indexCount × (4 + 4 + 4 + name[...variable]).
for i := uint16(0); i < indexCount; i++ {
hdr, err := mc.readN(12)
if err != nil {
return nil, err
}
nameSize := binary.LittleEndian.Uint32(hdr[8:12])
if nameSize > 256 {
return nil, fmt.Errorf("dir index name_size too large: %d", nameSize)
}
if _, err := mc.readN(int(nameSize) + 1); err != nil {
return nil, err
}
}
case itBasicFile:
body, err := mc.readN(16)
if err != nil {
return nil, err
}
ic.entry.Type = "file"
ic.entry.blocksStart = uint64(binary.LittleEndian.Uint32(body[0:4]))
ic.entry.fragIndex = binary.LittleEndian.Uint32(body[4:8])
ic.entry.fragOffset = binary.LittleEndian.Uint32(body[8:12])
fileSize := binary.LittleEndian.Uint32(body[12:16])
ic.entry.fileSize = int64(fileSize)
ic.entry.Size = int64(fileSize)
blocks, err := readBlockSizes(mc, fileSize, ic.entry.fragIndex, f.sb.BlockSize)
if err != nil {
return nil, err
}
ic.entry.blockSizes = blocks
case itExtFile:
body, err := mc.readN(40)
if err != nil {
return nil, err
}
ic.entry.Type = "file"
ic.entry.blocksStart = binary.LittleEndian.Uint64(body[0:8])
fileSize := binary.LittleEndian.Uint64(body[8:16])
// sparse := binary.LittleEndian.Uint64(body[16:24])
// nlink := binary.LittleEndian.Uint32(body[24:28])
ic.entry.fragIndex = binary.LittleEndian.Uint32(body[28:32])
ic.entry.fragOffset = binary.LittleEndian.Uint32(body[32:36])
// xattr := binary.LittleEndian.Uint32(body[36:40])
ic.entry.fileSize = int64(fileSize)
ic.entry.Size = int64(fileSize)
if fileSize > uint64(1)<<48 {
return nil, fmt.Errorf("unreasonable file size %d", fileSize)
}
blocks, err := readBlockSizes(mc, uint32(fileSize), ic.entry.fragIndex, f.sb.BlockSize)
if err != nil {
return nil, err
}
ic.entry.blockSizes = blocks
case itBasicSymlink, itExtSymlink:
body, err := mc.readN(8)
if err != nil {
return nil, err
}
// nlink := binary.LittleEndian.Uint32(body[0:4])
target := binary.LittleEndian.Uint32(body[4:8])
if target > 4096 {
return nil, fmt.Errorf("symlink target length %d too large", target)
}
link, err := mc.readN(int(target))
if err != nil {
return nil, err
}
ic.entry.Type = "symlink"
ic.entry.LinkTarget = string(link)
ic.entry.Size = int64(target)
if h.Type == itExtSymlink {
// trailing xattr u32
if _, err := mc.readN(4); err != nil {
return nil, err
}
}
default:
ic.entry.Type = "special"
}
return ic, nil
}
func readBlockSizes(mc *metaCursor, fileSize uint32, fragIdx uint32, blockSize uint32) ([]uint32, error) {
// If there is a fragment (fragIdx != 0xFFFFFFFF), the tail of the file
// (< blockSize) is stored there — do not count it as a full block.
full := fileSize / blockSize
if fragIdx == 0xFFFFFFFF && fileSize%blockSize != 0 {
full++ // final partial block stored on disk as a block
}
if full > (1 << 20) {
return nil, fmt.Errorf("unreasonable block count %d", full)
}
out := make([]uint32, full)
for i := range out {
b, err := mc.readN(4)
if err != nil {
return nil, err
}
out[i] = binary.LittleEndian.Uint32(b)
}
return out, nil
}
// ---------- directory walking ----------
// readDirEntries returns all (name, inode-ref) tuples in a directory.
func (f *FS) readDirEntries(parent *inodeContent) ([]dirEntry, error) {
if parent.dirSize == 0 {
return nil, nil
}
// dirStart is an offset into the dir table (relative to sb.DirTable).
// dirBlockOff is the intra-block start.
mc, err := f.metaCursorAt(int64(f.sb.DirTable), parent.dirStart, parent.dirBlockOff)
if err != nil {
return nil, err
}
remaining := int(parent.dirSize)
var out []dirEntry
for remaining > 0 {
hdr, err := mc.readN(12)
if err != nil {
return nil, err
}
remaining -= 12
count := binary.LittleEndian.Uint32(hdr[0:4]) + 1 // stored as count-1
startBlock := binary.LittleEndian.Uint32(hdr[4:8])
baseInode := binary.LittleEndian.Uint32(hdr[8:12])
for i := uint32(0); i < count; i++ {
entHdr, err := mc.readN(8)
if err != nil {
return nil, err
}
remaining -= 8
entOff := binary.LittleEndian.Uint16(entHdr[0:2])
iDelta := int16(binary.LittleEndian.Uint16(entHdr[2:4]))
entType := binary.LittleEndian.Uint16(entHdr[4:6])
nameSize := binary.LittleEndian.Uint16(entHdr[6:8])
if nameSize+1 > 256 {
return nil, fmt.Errorf("directory entry name_size %d too large", nameSize)
}
name, err := mc.readN(int(nameSize) + 1) // stored as name_size-1
if err != nil {
return nil, err
}
remaining -= int(nameSize) + 1
ref := (uint64(startBlock) << 16) | uint64(entOff)
out = append(out, dirEntry{
name: string(name),
ref: ref,
inodeNum: uint32(int64(baseInode) + int64(iDelta)),
entType: entType,
})
if len(out) > maxEntries {
return nil, fmt.Errorf("directory entry cap %d exceeded", maxEntries)
}
}
}
return out, nil
}
type dirEntry struct {
name string
ref uint64
inodeNum uint32
entType uint16
}
// Walk walks the entire filesystem from the root, emitting every Entry.
// The returned list is sorted by path. Walk is bounded by maxEntries
// and by recursion depth.
func (f *FS) Walk() ([]Entry, error) {
root, err := f.readInode(f.sb.RootInodeRef)
if err != nil {
return nil, fmt.Errorf("root inode: %w", err)
}
if root.entry.Type != "dir" {
return nil, fmt.Errorf("root is not a directory (type %q)", root.entry.Type)
}
out := []Entry{}
root.entry.Path = "/"
out = append(out, root.entry)
if err := f.walk(root, "/", &out, 0); err != nil {
return nil, err
}
sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path })
return out, nil
}
func (f *FS) walk(node *inodeContent, prefix string, out *[]Entry, depth int) error {
if depth > 64 {
return errors.New("directory depth exceeds 64 — possible cycle")
}
entries, err := f.readDirEntries(node)
if err != nil {
return err
}
for _, de := range entries {
child, err := f.readInode(de.ref)
if err != nil {
return fmt.Errorf("%s/%s: %w", prefix, de.name, err)
}
full := path.Join(prefix, de.name)
child.entry.Path = full
*out = append(*out, child.entry)
if len(*out) > maxEntries {
return fmt.Errorf("entry cap %d exceeded", maxEntries)
}
if child.entry.Type == "dir" {
if err := f.walk(child, full, out, depth+1); err != nil {
return err
}
}
}
return nil
}
// ---------- file extraction ----------
// fragmentEntry describes one entry in the fragment table.
type fragmentEntry struct {
Start uint64
Size uint32 // top bit = uncompressed
Compressed bool
}
// fragmentTable reads the entire fragment lookup table and returns the
// flat list of entries by index. Result is cached on first call.
func (f *FS) fragmentTable() ([]fragmentEntry, error) {
if f.sb.FragCount == 0 {
return nil, nil
}
// The fragment_table_start points at a u64[] of metadata-block offsets,
// one per 512 fragments (ceil).
indirCount := (int(f.sb.FragCount) + 511) / 512
indirBytes, err := f.d.Range(int64(f.sb.FragTable), int64(indirCount*8))
if err != nil {
return nil, fmt.Errorf("frag indirection: %w", err)
}
out := make([]fragmentEntry, 0, f.sb.FragCount)
remaining := int(f.sb.FragCount)
for i := 0; i < indirCount && remaining > 0; i++ {
absOff := int64(binary.LittleEndian.Uint64(indirBytes[i*8 : i*8+8]))
block, _, err := f.readMetadataBlock(absOff)
if err != nil {
return nil, err
}
// Each fragment entry is 16 bytes: u64 start, u32 size, u32 unused.
take := remaining
if take > 512 {
take = 512
}
if len(block) < take*16 {
return nil, fmt.Errorf("fragment block short: have %d need %d", len(block), take*16)
}
for j := 0; j < take; j++ {
off := j * 16
sz := binary.LittleEndian.Uint32(block[off+8 : off+12])
out = append(out, fragmentEntry{
Start: binary.LittleEndian.Uint64(block[off : off+8]),
Size: sz & 0x00ffffff, // documented size field occupies low 24 bits
Compressed: sz&0x01000000 == 0,
})
}
remaining -= take
}
return out, nil
}
// ExtractFile returns the full decompressed contents of the file at the
// given InodeRef. Returns an error if the inode is not a file.
func (f *FS) ExtractFile(ref uint64) ([]byte, error) {
ic, err := f.readInode(ref)
if err != nil {
return nil, err
}
if ic.entry.Type != "file" {
return nil, fmt.Errorf("inode %d is not a file (%s)", ic.entry.InodeNumber, ic.entry.Type)
}
out := make([]byte, 0, ic.entry.fileSize)
cursor := int64(ic.entry.blocksStart)
for i, bs := range ic.entry.blockSizes {
size := int64(bs & 0x00ffffff)
compressed := bs&0x01000000 == 0
if size == 0 {
// sparse hole: emit blockSize zeroes (or trailing partial)
remaining := ic.entry.fileSize - int64(len(out))
n := int64(f.sb.BlockSize)
if remaining < n {
n = remaining
}
out = append(out, make([]byte, n)...)
continue
}
if size > dataBlockMax {
return nil, fmt.Errorf("data block %d size %d exceeds cap", i, size)
}
body, err := f.d.Range(cursor, size)
if err != nil {
return nil, fmt.Errorf("data block %d: %w", i, err)
}
cursor += size
if !compressed {
out = append(out, body...)
continue
}
dec, err := f.decompressBlock(body)
if err != nil {
return nil, fmt.Errorf("data block %d: %w", i, err)
}
out = append(out, dec...)
}
if ic.entry.fragIndex != 0xFFFFFFFF {
frags, err := f.fragmentTable()
if err != nil {
return nil, err
}
if int(ic.entry.fragIndex) >= len(frags) {
return nil, fmt.Errorf("fragment index %d out of range (%d entries)", ic.entry.fragIndex, len(frags))
}
fe := frags[ic.entry.fragIndex]
fragBody, err := f.d.Range(int64(fe.Start), int64(fe.Size))
if err != nil {
return nil, fmt.Errorf("fragment data: %w", err)
}
var payload []byte
if fe.Compressed {
payload, err = f.decompressBlock(fragBody)
if err != nil {
return nil, err
}
} else {
payload = fragBody
}
tailLen := int(ic.entry.fileSize) - len(out)
if int(ic.entry.fragOffset)+tailLen > len(payload) {
return nil, fmt.Errorf("fragment slice out of range: off=%d len=%d payload=%d",
ic.entry.fragOffset, tailLen, len(payload))
}
out = append(out, payload[ic.entry.fragOffset:int(ic.entry.fragOffset)+tailLen]...)
}
if int64(len(out)) != ic.entry.fileSize {
return nil, fmt.Errorf("extraction length mismatch: got %d, want %d", len(out), ic.entry.fileSize)
}
return out, nil
}
// decompressBlock is like decompress but bounded by dataBlockMax; used for fragment payloads.
func (f *FS) decompressBlock(buf []byte) ([]byte, error) {
return f.decompressWithLimit(buf, dataBlockMax)
}
func readLimited(r io.Reader, limit int64) ([]byte, error) {
if limit <= 0 {
return nil, errors.New("invalid decompression limit")
}
lr := &io.LimitedReader{R: r, N: limit + 1}
out, err := io.ReadAll(lr)
if err != nil {
return nil, err
}
if int64(len(out)) > limit {
return nil, fmt.Errorf("decompressed block exceeds cap %d", limit)
}
return out, nil
}
internal/leviathan/stego/metadata.go
+694
View File
@@ -0,0 +1,694 @@
package stego
import (
"bytes"
"compress/zlib"
"encoding/binary"
"errors"
"fmt"
"io"
"strings"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// MetaEntry is one piece of textual/EXIF metadata recovered from an image.
type MetaEntry struct {
Offset int64 `json:"offset"`
Length int64 `json:"length"`
Container string `json:"container"`
Kind string `json:"kind"`
Keyword string `json:"keyword"`
Value string `json:"value"`
Note string `json:"note,omitempty"`
}
// ScanMetadata walks PNG chunks or JPEG segments (APPn / COM) and decodes
// human-readable metadata. Output is best-effort — malformed containers
// return whatever has been parsed up to the failure point.
func ScanMetadata(d *document.Document) ([]MetaEntry, error) {
size := d.Size()
if size < 8 {
return nil, errors.New("file too small")
}
head, err := d.Range(0, minInt64(16, size))
if err != nil {
return nil, err
}
switch {
case len(head) >= 8 && string(head[:8]) == "\x89PNG\r\n\x1a\n":
return scanPNGMeta(d)
case len(head) >= 3 && head[0] == 0xff && head[1] == 0xd8 && head[2] == 0xff:
return scanJPEGMeta(d)
case len(head) >= 6 && (string(head[:6]) == "GIF87a" || string(head[:6]) == "GIF89a"):
return scanGIFMeta(d)
}
return nil, errors.New("unsupported container (PNG, JPEG and GIF handled)")
}
func scanPNGMeta(d *document.Document) ([]MetaEntry, error) {
size := d.Size()
out := []MetaEntry{}
off := int64(8)
for off+12 <= size {
hdr, err := d.Range(off, 8)
if err != nil {
break
}
ln := int64(binary.BigEndian.Uint32(hdr[0:4]))
typ := string(hdr[4:8])
if ln < 0 || off+12+ln > size {
break
}
if typ == "tEXt" || typ == "zTXt" || typ == "iTXt" {
data, err := d.Range(off+8, ln)
if err == nil {
entry := decodePNGTextChunk(typ, data)
entry.Offset = off
entry.Length = 12 + ln
entry.Container = "png"
out = append(out, entry)
}
}
off += 12 + ln
if typ == "IEND" {
break
}
}
return out, nil
}
func decodePNGTextChunk(typ string, data []byte) MetaEntry {
e := MetaEntry{Kind: typ}
i := bytes.IndexByte(data, 0)
if i < 0 {
e.Note = "no keyword separator"
e.Value = safeText(data)
return e
}
e.Keyword = string(data[:i])
rest := data[i+1:]
switch typ {
case "tEXt":
e.Value = safeText(rest)
case "zTXt":
if len(rest) < 1 {
e.Note = "truncated"
return e
}
// rest[0] = compression method (0 = zlib)
method := rest[0]
raw, err := zlibDecompress(rest[1:])
if err != nil {
e.Note = fmt.Sprintf("zlib fail (method=%d): %v", method, err)
return e
}
e.Value = safeText(raw)
case "iTXt":
// compression flag, compression method, language tag\0, translated keyword\0, text
if len(rest) < 2 {
e.Note = "truncated"
return e
}
compressed := rest[0] == 1
rest = rest[2:]
j := bytes.IndexByte(rest, 0)
if j < 0 {
e.Value = safeText(rest)
return e
}
lang := string(rest[:j])
rest = rest[j+1:]
k := bytes.IndexByte(rest, 0)
if k < 0 {
e.Value = safeText(rest)
return e
}
trk := string(rest[:k])
rest = rest[k+1:]
if compressed {
if raw, err := zlibDecompress(rest); err == nil {
rest = raw
} else {
e.Note = "zlib fail: " + err.Error()
}
}
v := safeText(rest)
if lang != "" || trk != "" {
v = fmt.Sprintf("[%s/%s] %s", lang, trk, v)
}
e.Value = v
}
return e
}
func scanJPEGMeta(d *document.Document) ([]MetaEntry, error) {
size := d.Size()
out := []MetaEntry{}
off := int64(2) // skip SOI
for off+4 <= size {
hdr, err := d.Range(off, 2)
if err != nil || hdr[0] != 0xff {
break
}
marker := hdr[1]
// Standalone markers (no payload): SOI/EOI/RSTn
if marker == 0xd9 || marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) {
off += 2
continue
}
// Read segment length (big-endian, includes the length bytes themselves)
lnBytes, err := d.Range(off+2, 2)
if err != nil {
break
}
segLen := int64(binary.BigEndian.Uint16(lnBytes))
if segLen < 2 || off+2+segLen > size {
break
}
payload, err := d.Range(off+4, segLen-2)
if err != nil {
break
}
switch {
case marker == 0xfe: // COM
out = append(out, MetaEntry{
Offset: off, Length: 2 + segLen,
Container: "jpeg", Kind: "COM",
Value: safeText(payload),
})
case marker == 0xe1: // APP1 — EXIF / XMP
entries := decodeJPEGApp1(payload)
for _, e := range entries {
e.Offset = off
e.Length = 2 + segLen
e.Container = "jpeg"
out = append(out, e)
}
case marker == 0xe0: // APP0 — JFIF
out = append(out, MetaEntry{
Offset: off, Length: 2 + segLen,
Container: "jpeg", Kind: "APP0",
Keyword: sniffMarker(payload),
Value: fmt.Sprintf("%d bytes", len(payload)),
})
case marker >= 0xe2 && marker <= 0xef: // APP2..APP15
out = append(out, MetaEntry{
Offset: off, Length: 2 + segLen,
Container: "jpeg", Kind: fmt.Sprintf("APP%d", marker-0xe0),
Keyword: sniffMarker(payload),
Value: fmt.Sprintf("%d bytes", len(payload)),
})
}
off += 2 + segLen
// SOS (Start of Scan) — entropy-coded data follows, stop walking.
if marker == 0xda {
break
}
}
return out, nil
}
func sniffMarker(p []byte) string {
i := bytes.IndexByte(p, 0)
if i < 0 || i > 16 {
if len(p) > 16 {
return string(p[:16])
}
return ""
}
return string(p[:i])
}
func decodeJPEGApp1(p []byte) []MetaEntry {
out := []MetaEntry{}
if len(p) >= 6 && string(p[:6]) == "Exif\x00\x00" {
tiff := p[6:]
entries, err := decodeEXIF(tiff)
if err == nil {
out = append(out, entries...)
} else {
out = append(out, MetaEntry{Kind: "APP1/EXIF", Note: err.Error()})
}
return out
}
if len(p) >= 28 && string(p[:28]) == "http://ns.adobe.com/xap/1.0/" {
xmp := p[29:]
out = append(out, MetaEntry{
Kind: "APP1/XMP", Keyword: "xmp", Value: safeText(xmp),
})
return out
}
out = append(out, MetaEntry{
Kind: "APP1", Keyword: sniffMarker(p), Value: fmt.Sprintf("%d bytes", len(p)),
})
return out
}
// EXIF reader: TIFF header + IFD0 + IFD1 + Exif SubIFD + GPS SubIFD +
// Interoperability IFD. Handles ASCII/BYTE/SHORT/LONG/RATIONAL/SRATIONAL
// values, plus UNDEFINED as hex preview. Detects MakerNote presence.
func decodeEXIF(tiff []byte) ([]MetaEntry, error) {
if len(tiff) < 8 {
return nil, errors.New("short TIFF header")
}
var bo binary.ByteOrder
switch string(tiff[:2]) {
case "II":
bo = binary.LittleEndian
case "MM":
bo = binary.BigEndian
default:
return nil, errors.New("bad TIFF byte-order mark")
}
if bo.Uint16(tiff[2:4]) != 0x002a {
return nil, errors.New("bad TIFF magic")
}
ifd0 := int(bo.Uint32(tiff[4:8]))
out := []MetaEntry{}
ptrs := ifdPointers{}
nextIFD := 0
out = append(out, readIFD(tiff, ifd0, bo, "IFD0", &ptrs, &nextIFD)...)
// IFD1 (thumbnail) if present.
if nextIFD > 0 {
out = append(out, readIFD(tiff, nextIFD, bo, "IFD1", nil, nil)...)
}
if ptrs.exif > 0 {
out = append(out, readIFD(tiff, ptrs.exif, bo, "ExifIFD", &ptrs, nil)...)
}
if ptrs.gps > 0 {
out = append(out, readIFD(tiff, ptrs.gps, bo, "GPSIFD", nil, nil)...)
}
if ptrs.interop > 0 {
out = append(out, readIFD(tiff, ptrs.interop, bo, "InteropIFD", nil, nil)...)
}
return out, nil
}
type ifdPointers struct {
exif int
gps int
interop int
}
func readIFD(tiff []byte, off int, bo binary.ByteOrder, name string, ptrs *ifdPointers, nextPtr *int) []MetaEntry {
out := []MetaEntry{}
if off < 0 || off+2 > len(tiff) {
return out
}
n := int(bo.Uint16(tiff[off : off+2]))
base := off + 2
// Cap directory count to head off hostile input.
if n > 512 {
n = 512
}
for i := 0; i < n; i++ {
p := base + i*12
if p+12 > len(tiff) {
break
}
tag := bo.Uint16(tiff[p : p+2])
typ := bo.Uint16(tiff[p+2 : p+4])
count := int(bo.Uint32(tiff[p+4 : p+8]))
valRaw := tiff[p+8 : p+12]
// Sub-IFD pointers never appear as regular entries in the output.
if ptrs != nil {
switch tag {
case 0x8769: // Exif SubIFD pointer
ptrs.exif = int(bo.Uint32(valRaw))
continue
case 0x8825: // GPS IFD pointer
ptrs.gps = int(bo.Uint32(valRaw))
continue
case 0xa005: // Interoperability IFD pointer
ptrs.interop = int(bo.Uint32(valRaw))
continue
}
}
e := MetaEntry{Kind: name, Keyword: exifTagName(tag, name)}
if tag == 0x927c { // MakerNote
e.Value = fmt.Sprintf("%d bytes (vendor-specific, not decoded)", count)
out = append(out, e)
continue
}
e.Value = readEXIFValue(tiff, bo, typ, count, valRaw)
out = append(out, e)
}
// "Next IFD" pointer follows the directory entries (4 bytes).
if nextPtr != nil {
tail := base + n*12
if tail+4 <= len(tiff) {
*nextPtr = int(bo.Uint32(tiff[tail : tail+4]))
}
}
return out
}
func readEXIFValue(tiff []byte, bo binary.ByteOrder, typ uint16, count int, valRaw []byte) string {
typeSize := map[uint16]int{1: 1, 2: 1, 3: 2, 4: 4, 5: 8, 7: 1, 9: 4, 10: 8}
sz, ok := typeSize[typ]
if !ok {
return fmt.Sprintf("type=%d count=%d", typ, count)
}
total := sz * count
var data []byte
if total <= 4 {
data = valRaw[:total]
} else {
off := int(bo.Uint32(valRaw))
if off < 0 || off+total > len(tiff) {
return fmt.Sprintf("type=%d count=%d (oob)", typ, count)
}
data = tiff[off : off+total]
}
switch typ {
case 2: // ASCII (null-terminated)
return safeText(bytes.TrimRight(data, "\x00"))
case 1, 7: // BYTE, UNDEFINED — show as hex preview
if len(data) > 32 {
return fmt.Sprintf("%x… (%d bytes)", data[:32], len(data))
}
return fmt.Sprintf("%x", data)
case 3: // SHORT
parts := []string{}
for i := 0; i < count && i < 8; i++ {
parts = append(parts, fmt.Sprintf("%d", bo.Uint16(data[i*2:i*2+2])))
}
if count > 8 {
parts = append(parts, "…")
}
return strings.Join(parts, ", ")
case 4: // LONG
parts := []string{}
for i := 0; i < count && i < 6; i++ {
parts = append(parts, fmt.Sprintf("%d", bo.Uint32(data[i*4:i*4+4])))
}
if count > 6 {
parts = append(parts, "…")
}
return strings.Join(parts, ", ")
case 5, 10: // RATIONAL / SRATIONAL
parts := []string{}
for i := 0; i < count && i < 4; i++ {
n := bo.Uint32(data[i*8 : i*8+4])
dn := bo.Uint32(data[i*8+4 : i*8+8])
if dn == 0 {
parts = append(parts, fmt.Sprintf("%d/0", n))
} else {
parts = append(parts, fmt.Sprintf("%d/%d", n, dn))
}
}
return strings.Join(parts, ", ")
case 9: // SLONG
parts := []string{}
for i := 0; i < count && i < 6; i++ {
parts = append(parts, fmt.Sprintf("%d", int32(bo.Uint32(data[i*4:i*4+4]))))
}
return strings.Join(parts, ", ")
}
return fmt.Sprintf("type=%d count=%d", typ, count)
}
func exifTagName(tag uint16, ifd string) string {
if ifd == "GPSIFD" {
if n, ok := gpsTags[tag]; ok {
return n
}
return fmt.Sprintf("GPS tag 0x%04x", tag)
}
if n, ok := exifTags[tag]; ok {
return n
}
return fmt.Sprintf("tag 0x%04x", tag)
}
// exifTags covers the standard TIFF/Exif tags routinely seen in
// JPEG/TIFF images produced by consumer cameras and phones.
var exifTags = map[uint16]string{
// TIFF / IFD0
0x0100: "ImageWidth", 0x0101: "ImageLength",
0x0102: "BitsPerSample", 0x0103: "Compression",
0x0106: "PhotometricInterpretation", 0x010a: "FillOrder",
0x010d: "DocumentName", 0x010e: "ImageDescription",
0x010f: "Make", 0x0110: "Model", 0x0111: "StripOffsets",
0x0112: "Orientation", 0x0115: "SamplesPerPixel",
0x0116: "RowsPerStrip", 0x0117: "StripByteCounts",
0x011a: "XResolution", 0x011b: "YResolution",
0x011c: "PlanarConfiguration", 0x0128: "ResolutionUnit",
0x012d: "TransferFunction", 0x0131: "Software", 0x0132: "DateTime",
0x013b: "Artist", 0x013c: "HostComputer",
0x013e: "WhitePoint", 0x013f: "PrimaryChromaticities",
0x0211: "YCbCrCoefficients", 0x0212: "YCbCrSubSampling",
0x0213: "YCbCrPositioning", 0x0214: "ReferenceBlackWhite",
0x02bc: "XMP",
0x8298: "Copyright", 0x83bb: "IPTCNAA",
0x8769: "ExifIFDPointer", 0x8773: "InterColorProfile",
0x8825: "GPSIFDPointer",
// Exif SubIFD
0x829a: "ExposureTime", 0x829d: "FNumber",
0x8822: "ExposureProgram", 0x8824: "SpectralSensitivity",
0x8827: "ISOSpeedRatings", 0x8828: "OECF",
0x9000: "ExifVersion",
0x9003: "DateTimeOriginal", 0x9004: "DateTimeDigitized",
0x9010: "OffsetTime", 0x9011: "OffsetTimeOriginal", 0x9012: "OffsetTimeDigitized",
0x9101: "ComponentsConfiguration", 0x9102: "CompressedBitsPerPixel",
0x9201: "ShutterSpeedValue", 0x9202: "ApertureValue",
0x9203: "BrightnessValue", 0x9204: "ExposureBiasValue",
0x9205: "MaxApertureValue", 0x9206: "SubjectDistance",
0x9207: "MeteringMode", 0x9208: "LightSource", 0x9209: "Flash",
0x920a: "FocalLength",
0x9214: "SubjectArea", 0x927c: "MakerNote", 0x9286: "UserComment",
0x9290: "SubSecTime", 0x9291: "SubSecTimeOriginal", 0x9292: "SubSecTimeDigitized",
0xa000: "FlashpixVersion", 0xa001: "ColorSpace",
0xa002: "PixelXDimension", 0xa003: "PixelYDimension",
0xa004: "RelatedSoundFile", 0xa005: "InteropIFDPointer",
0xa20b: "FlashEnergy", 0xa20c: "SpatialFrequencyResponse",
0xa20e: "FocalPlaneXResolution", 0xa20f: "FocalPlaneYResolution",
0xa210: "FocalPlaneResolutionUnit",
0xa214: "SubjectLocation", 0xa215: "ExposureIndex",
0xa217: "SensingMethod", 0xa300: "FileSource", 0xa301: "SceneType",
0xa302: "CFAPattern", 0xa401: "CustomRendered",
0xa402: "ExposureMode", 0xa403: "WhiteBalance",
0xa404: "DigitalZoomRatio", 0xa405: "FocalLengthIn35mmFilm",
0xa406: "SceneCaptureType", 0xa407: "GainControl",
0xa408: "Contrast", 0xa409: "Saturation", 0xa40a: "Sharpness",
0xa40b: "DeviceSettingDescription", 0xa40c: "SubjectDistanceRange",
0xa420: "ImageUniqueID", 0xa430: "OwnerName", 0xa431: "SerialNumber",
0xa432: "LensSpecification", 0xa433: "LensMake", 0xa434: "LensModel",
0xa435: "LensSerialNumber",
// Thumbnail IFD1 re-uses IFD0 tags; additional ones:
0x0201: "JPEGInterchangeFormat", 0x0202: "JPEGInterchangeFormatLength",
}
var gpsTags = map[uint16]string{
0x0000: "GPSVersionID", 0x0001: "GPSLatitudeRef", 0x0002: "GPSLatitude",
0x0003: "GPSLongitudeRef", 0x0004: "GPSLongitude",
0x0005: "GPSAltitudeRef", 0x0006: "GPSAltitude",
0x0007: "GPSTimeStamp", 0x0008: "GPSSatellites", 0x0009: "GPSStatus",
0x000a: "GPSMeasureMode", 0x000b: "GPSDOP", 0x000c: "GPSSpeedRef",
0x000d: "GPSSpeed", 0x000e: "GPSTrackRef", 0x000f: "GPSTrack",
0x0010: "GPSImgDirectionRef", 0x0011: "GPSImgDirection",
0x0012: "GPSMapDatum", 0x0013: "GPSDestLatitudeRef", 0x0014: "GPSDestLatitude",
0x0015: "GPSDestLongitudeRef", 0x0016: "GPSDestLongitude",
0x0017: "GPSDestBearingRef", 0x0018: "GPSDestBearing",
0x0019: "GPSDestDistanceRef", 0x001a: "GPSDestDistance",
0x001b: "GPSProcessingMethod", 0x001c: "GPSAreaInformation",
0x001d: "GPSDateStamp", 0x001e: "GPSDifferential", 0x001f: "GPSHPositioningError",
}
func zlibDecompress(b []byte) ([]byte, error) {
r, err := zlib.NewReader(bytes.NewReader(b))
if err != nil {
return nil, err
}
defer r.Close()
// 2 MiB cap — textual metadata should never be near this.
lim := &io.LimitedReader{R: r, N: 2 * 1024 * 1024}
return io.ReadAll(lim)
}
func safeText(b []byte) string {
// Preserve text but trim extreme size and strip control chars other than tab/newline.
if len(b) > 4096 {
b = b[:4096]
}
sb := strings.Builder{}
sb.Grow(len(b))
for _, c := range b {
if c == '\t' || c == '\n' || c == '\r' || (c >= 0x20 && c <= 0x7e) {
sb.WriteByte(c)
continue
}
if c >= 0x80 {
// Pass through; browser will render as UTF-8 if valid.
sb.WriteByte(c)
continue
}
// Drop other controls.
}
return sb.String()
}
func minInt64(a, b int64) int64 {
if a < b {
return a
}
return b
}
// scanGIFMeta walks a GIF file and surfaces every extension block that
// can carry human-readable data: Comment (0x21 0xFE), Plain Text
// (0x21 0x01), and Application (0x21 0xFF — XMP, Netscape loop, etc.).
func scanGIFMeta(d *document.Document) ([]MetaEntry, error) {
size := d.Size()
out := []MetaEntry{}
// Skip header (6) + logical screen descriptor (7 fixed) and the
// optional global colour table.
hdr, err := d.Range(0, 13)
if err != nil {
return nil, err
}
off := int64(13)
packed := hdr[10]
if packed&0x80 != 0 {
gctSize := int64(3) * (int64(1) << uint((packed&0x07)+1))
off += gctSize
}
for off < size {
b, err := d.Range(off, 1)
if err != nil {
break
}
switch b[0] {
case 0x3b: // trailer
return out, nil
case 0x2c: // Image Descriptor: skip descriptor + optional LCT + LZW data
next, ok := skipGIFImage(d, off)
if !ok {
return out, nil
}
off = next
case 0x21: // Extension Introducer
if off+2 > size {
return out, nil
}
lbl, _ := d.Range(off+1, 1)
start := off
payload, next, ok := readGIFSubBlocks(d, off+2)
if !ok {
return out, nil
}
switch lbl[0] {
case 0xfe:
out = append(out, MetaEntry{
Offset: start, Length: next - start,
Container: "gif", Kind: "Comment",
Value: safeText(payload),
})
case 0x01:
// Plain Text: first 12 bytes are text-grid parameters.
text := payload
if len(text) > 12 {
text = text[12:]
} else {
text = nil
}
out = append(out, MetaEntry{
Offset: start, Length: next - start,
Container: "gif", Kind: "PlainText",
Value: safeText(text),
})
case 0xff:
// Application Extension: first sub-block is an 11-byte
// identifier like "NETSCAPE2.0" or "XMP DataXMP".
if len(payload) >= 11 {
appID := string(bytes.TrimRight(payload[:11], "\x00 "))
rest := payload[11:]
out = append(out, MetaEntry{
Offset: start, Length: next - start,
Container: "gif", Kind: "Application",
Keyword: appID,
Value: safeText(rest),
})
} else {
out = append(out, MetaEntry{
Offset: start, Length: next - start,
Container: "gif", Kind: "Application",
Value: safeText(payload),
})
}
default:
// Graphic control (0xf9) and others — skip.
}
off = next
default:
// Unknown byte; bail.
return out, nil
}
}
return out, nil
}
// readGIFSubBlocks reads a chain of GIF sub-blocks: each starts with a
// length byte (0 terminates). Returns (concatenated payload, offset of
// next byte after terminator, ok).
func readGIFSubBlocks(d *document.Document, off int64) ([]byte, int64, bool) {
size := d.Size()
out := []byte{}
for off < size {
ln, err := d.Range(off, 1)
if err != nil {
return nil, off, false
}
n := int64(ln[0])
off++
if n == 0 {
return out, off, true
}
if off+n > size {
return nil, off, false
}
block, err := d.Range(off, n)
if err != nil {
return nil, off, false
}
out = append(out, block...)
off += n
// Hard cap to prevent OOM on malformed input.
if len(out) > 16*1024*1024 {
return out, off, true
}
}
return nil, off, false
}
// skipGIFImage walks an Image Descriptor block starting at off (the 0x2c
// byte), skipping the descriptor, any local colour table, the LZW
// minimum-code-size byte and the image data sub-blocks. Returns the
// offset of the next block, or (0, false) on truncation.
func skipGIFImage(d *document.Document, off int64) (int64, bool) {
size := d.Size()
if off+10 > size {
return 0, false
}
desc, err := d.Range(off, 10)
if err != nil {
return 0, false
}
packed := desc[9]
off += 10
if packed&0x80 != 0 {
lctSize := int64(3) * (int64(1) << uint((packed&0x07)+1))
off += lctSize
}
// LZW minimum code size byte, then sub-blocks.
if off+1 > size {
return 0, false
}
off++
_, next, ok := readGIFSubBlocks(d, off)
if !ok {
return 0, false
}
return next, true
}
internal/leviathan/stego/stego.go
+501
View File
@@ -0,0 +1,501 @@
// Package stego provides practical steganography helpers:
//
// - trailer detection (bytes after a declared end) for PNG, JPEG,
// GIF, BMP, WAV and RIFF/WebP containers
// - multi-trailer surfacing: identifies any embedded container
// (PNG/JPEG/ZIP/GIF/PDF/RAR/7z) inside the trailer region
// - LSB extract/inject over an arbitrary byte buffer
// - PNG IDAT, BMP pixel-array and WAV sample-data resolvers so
// callers can point LSB at the correct carrier region
// - textual-metadata extraction for PNG, JPEG, GIF (see metadata.go)
package stego
import (
"encoding/binary"
"errors"
"github.com/andydixon/nexus/internal/leviathan/document"
)
// ---------------------------------------------------------------------------
// Trailer detection
// ---------------------------------------------------------------------------
// EmbeddedHit describes a recognised container signature found inside the
// trailer region (the bytes after a file's declared end).
type EmbeddedHit struct {
Offset int64 `json:"offset"` // absolute offset within the outer file
Kind string `json:"kind"`
Note string `json:"note,omitempty"`
}
// AnalyseAppended inspects the end of the file for plausible trailers: data
// after the declared end of a known container. The result also lists any
// nested container magic bytes discovered inside the trailer region.
func AnalyseAppended(d *document.Document) (map[string]any, error) {
size := d.Size()
head, _ := d.Range(0, 16)
result := map[string]any{
"fileSize": size,
"declaredEnd": int64(-1),
"trailerSize": int64(0),
"container": "",
"note": "no recognised container",
"embedded": []EmbeddedHit{},
}
end := int64(-1)
container := ""
switch {
case len(head) >= 8 && string(head[:8]) == "\x89PNG\r\n\x1a\n":
container = "png"
end = findPNGEnd(d)
case len(head) >= 3 && head[0] == 0xff && head[1] == 0xd8 && head[2] == 0xff:
container = "jpeg"
end = findJPEGEnd(d)
case len(head) >= 6 && (string(head[:6]) == "GIF87a" || string(head[:6]) == "GIF89a"):
container = "gif"
end = findGIFEnd(d)
case len(head) >= 2 && head[0] == 'B' && head[1] == 'M':
container = "bmp"
end = findBMPEnd(d)
case len(head) >= 12 && string(head[:4]) == "RIFF":
sub := string(head[8:12])
if sub == "WAVE" {
container = "wav"
} else if sub == "WEBP" {
container = "webp"
} else {
container = "riff"
}
end = findRIFFEnd(d)
}
if container != "" {
result["container"] = container
}
if end > 0 && end <= size {
result["declaredEnd"] = end
if end < size {
trailer := size - end
result["trailerSize"] = trailer
result["note"] = "bytes after " + container + " declared end"
if trailer > 0 {
// Cap scanning the trailer at 16 MiB — otherwise we might
// pull a 2-GiB tail into memory on hostile inputs.
scanLen := trailer
if scanLen > 16*1024*1024 {
scanLen = 16 * 1024 * 1024
}
tail, _ := d.Range(end, scanLen)
result["embedded"] = scanEmbedded(tail, end)
}
} else {
result["note"] = "no trailing bytes"
}
}
return result, nil
}
// ExtractTrailer returns the bytes after a container's declared end.
func ExtractTrailer(d *document.Document) ([]byte, error) {
a, err := AnalyseAppended(d)
if err != nil {
return nil, err
}
end, _ := a["declaredEnd"].(int64)
ts, _ := a["trailerSize"].(int64)
if end < 0 || ts <= 0 {
return nil, errors.New("no trailer found")
}
return d.Range(end, ts)
}
func findPNGEnd(d *document.Document) int64 {
size := d.Size()
off := int64(8)
for off+12 <= size {
hdr, err := d.Range(off, 8)
if err != nil {
break
}
ln := int64(binary.BigEndian.Uint32(hdr[0:4]))
if ln < 0 || off+12+ln > size {
break
}
typ := string(hdr[4:8])
off += 12 + ln
if typ == "IEND" {
return off
}
}
return -1
}
func findJPEGEnd(d *document.Document) int64 {
size := d.Size()
tailLen := int64(128 * 1024)
if tailLen > size {
tailLen = size
}
tail, _ := d.Range(size-tailLen, tailLen)
for i := len(tail) - 2; i >= 0; i-- {
if tail[i] == 0xff && tail[i+1] == 0xd9 {
return size - tailLen + int64(i+2)
}
}
return -1
}
func findGIFEnd(d *document.Document) int64 {
size := d.Size()
// The GIF terminator 0x3B is a single byte that follows the final
// data block of the final image. Walking the container correctly is
// overkill; scan from the tail and accept the last 0x3B that is
// preceded by a block terminator (0x00) or a plausible frame end.
tailLen := int64(32 * 1024)
if tailLen > size {
tailLen = size
}
tail, _ := d.Range(size-tailLen, tailLen)
for i := len(tail) - 1; i >= 0; i-- {
if tail[i] == 0x3b {
return size - tailLen + int64(i+1)
}
}
return -1
}
func findBMPEnd(d *document.Document) int64 {
size := d.Size()
hdr, err := d.Range(0, 14)
if err != nil {
return -1
}
bfSize := int64(binary.LittleEndian.Uint32(hdr[2:6]))
if bfSize > 0 && bfSize <= size {
return bfSize
}
return -1
}
func findRIFFEnd(d *document.Document) int64 {
size := d.Size()
hdr, err := d.Range(0, 8)
if err != nil {
return -1
}
// RIFF chunk size (little-endian) is the size of the rest of the
// file *after* the first 8 bytes, so declared end = 8 + riffSize.
riffSize := int64(binary.LittleEndian.Uint32(hdr[4:8]))
end := 8 + riffSize
if end > 0 && end <= size {
// RIFF pads to even alignment.
if end%2 == 1 && end+1 <= size {
end++
}
return end
}
return -1
}
// scanEmbedded walks a trailer buffer for known container magic bytes.
func scanEmbedded(buf []byte, base int64) []EmbeddedHit {
hits := []EmbeddedHit{}
type sig struct {
bytes []byte
kind string
}
sigs := []sig{
{[]byte("\x89PNG\r\n\x1a\n"), "png"},
{[]byte{0xff, 0xd8, 0xff}, "jpeg"},
{[]byte("GIF87a"), "gif"},
{[]byte("GIF89a"), "gif"},
{[]byte("PK\x03\x04"), "zip"},
{[]byte("PK\x05\x06"), "zip-eocd"},
{[]byte("Rar!\x1a\x07\x00"), "rar"},
{[]byte("Rar!\x1a\x07\x01\x00"), "rar5"},
{[]byte("7z\xbc\xaf\x27\x1c"), "7z"},
{[]byte("%PDF-"), "pdf"},
{[]byte("\x1f\x8b\x08"), "gzip"},
{[]byte{0xfd, '7', 'z', 'X', 'Z', 0x00}, "xz"},
{[]byte("BZh"), "bzip2"},
{[]byte{0x28, 0xb5, 0x2f, 0xfd}, "zstd"},
{[]byte{0x7f, 'E', 'L', 'F'}, "elf"},
{[]byte("MZ"), "mz/pe"},
{[]byte("ustar"), "tar"},
{[]byte("OggS"), "ogg"},
{[]byte("ID3"), "id3/mp3"},
{[]byte("fLaC"), "flac"},
{[]byte("\x1aEߣ"), "matroska/webm"},
{[]byte("RIFF"), "riff"},
}
for _, s := range sigs {
if len(s.bytes) == 0 {
continue
}
start := 0
for {
i := indexOf(buf[start:], s.bytes)
if i < 0 {
break
}
hits = append(hits, EmbeddedHit{
Offset: base + int64(start+i),
Kind: s.kind,
})
// A plain "MZ" or "RIFF" hit without more context is noisy;
// the caller can filter on kind. Keep walking past this hit.
start += i + len(s.bytes)
// Bound total hits per signature to keep UI manageable.
if len(hits) > 256 {
break
}
}
if len(hits) > 256 {
break
}
}
return hits
}
func indexOf(hay, needle []byte) int {
if len(needle) == 0 || len(hay) < len(needle) {
return -1
}
first := needle[0]
last := len(hay) - len(needle)
for i := 0; i <= last; i++ {
if hay[i] != first {
continue
}
match := true
for j := 1; j < len(needle); j++ {
if hay[i+j] != needle[j] {
match = false
break
}
}
if match {
return i
}
}
return -1
}
// ---------------------------------------------------------------------------
// Carrier resolvers (PNG IDAT, BMP pixels, WAV samples)
// ---------------------------------------------------------------------------
// PNGImageData concatenates all IDAT chunks' payload bytes. Callers can then
// feed the result into LSBExtract/LSBInject. The returned buffer is the raw
// (still-deflated) IDAT stream — it is the conventional LSB carrier for
// tools like zsteg, not the decoded pixel array.
func PNGImageData(d *document.Document) ([]byte, error) {
size := d.Size()
off := int64(8)
buf := []byte{}
for off+12 <= size {
hdr, err := d.Range(off, 8)
if err != nil {
return nil, err
}
ln := int64(binary.BigEndian.Uint32(hdr[0:4]))
typ := string(hdr[4:8])
if typ == "IDAT" {
chunk, err := d.Range(off+8, ln)
if err != nil {
return nil, err
}
buf = append(buf, chunk...)
}
off += 12 + ln
if typ == "IEND" {
break
}
}
if len(buf) == 0 {
return nil, errors.New("no IDAT data")
}
return buf, nil
}
// BMPCarrier describes a resolved BMP pixel array that can be used as an
// LSB carrier.
type BMPCarrier struct {
Data []byte `json:"data"`
Offset int64 `json:"offset"` // offset of pixel array in the file
Width int32 `json:"width"`
Height int32 `json:"height"`
BitsPerPixel uint16 `json:"bitsPerPixel"`
RowStride int `json:"rowStride"`
}
// BMPPixels locates and returns the BMP pixel array. Supports 8/24/32 bpp
// uncompressed BI_RGB images, which are by far the most common carriers.
func BMPPixels(d *document.Document) (*BMPCarrier, error) {
size := d.Size()
if size < 54 {
return nil, errors.New("not a BMP (too small)")
}
hdr, err := d.Range(0, 54)
if err != nil {
return nil, err
}
if hdr[0] != 'B' || hdr[1] != 'M' {
return nil, errors.New("not a BMP")
}
bfOffBits := int64(binary.LittleEndian.Uint32(hdr[10:14]))
dibSize := binary.LittleEndian.Uint32(hdr[14:18])
// BITMAPINFOHEADER (40), BITMAPV4HEADER (108), BITMAPV5HEADER (124),
// OS/2 BITMAPCOREHEADER (12). Only handle the modern family.
if dibSize < 40 {
return nil, errors.New("BMP: unsupported DIB header (BITMAPCOREHEADER)")
}
w := int32(binary.LittleEndian.Uint32(hdr[18:22]))
h := int32(binary.LittleEndian.Uint32(hdr[22:26]))
bpp := binary.LittleEndian.Uint16(hdr[28:30])
compression := binary.LittleEndian.Uint32(hdr[30:34])
if compression != 0 && compression != 3 { // BI_RGB / BI_BITFIELDS
return nil, errors.New("BMP: compressed images not supported")
}
if bpp != 8 && bpp != 24 && bpp != 32 {
return nil, errors.New("BMP: only 8/24/32 bpp supported")
}
absH := h
if absH < 0 {
absH = -absH
}
rowStride := ((int(w)*int(bpp) + 31) / 32) * 4
total := int64(rowStride) * int64(absH)
if bfOffBits < 0 || bfOffBits+total > size {
return nil, errors.New("BMP: pixel array out of range")
}
data, err := d.Range(bfOffBits, total)
if err != nil {
return nil, err
}
return &BMPCarrier{
Data: data, Offset: bfOffBits,
Width: w, Height: h, BitsPerPixel: bpp, RowStride: rowStride,
}, nil
}
// WAVCarrier describes a resolved WAV "data" chunk that can be used as an
// LSB carrier.
type WAVCarrier struct {
Data []byte `json:"data"`
Offset int64 `json:"offset"` // offset of sample data
SampleRate uint32 `json:"sampleRate"`
NumChannels uint16 `json:"numChannels"`
BitsPerSample uint16 `json:"bitsPerSample"`
AudioFormat uint16 `json:"audioFormat"`
}
// WAVSamples locates the "data" sub-chunk of a RIFF/WAVE file and returns
// its payload. Only PCM (format 1) is usable as a straightforward LSB
// carrier; other formats still return bytes but are flagged in the caller.
func WAVSamples(d *document.Document) (*WAVCarrier, error) {
size := d.Size()
if size < 44 {
return nil, errors.New("not a WAV (too small)")
}
hdr, err := d.Range(0, 12)
if err != nil {
return nil, err
}
if string(hdr[:4]) != "RIFF" || string(hdr[8:12]) != "WAVE" {
return nil, errors.New("not a RIFF/WAVE file")
}
out := &WAVCarrier{}
off := int64(12)
for off+8 <= size {
ch, err := d.Range(off, 8)
if err != nil {
return nil, err
}
id := string(ch[:4])
cs := int64(binary.LittleEndian.Uint32(ch[4:8]))
if cs < 0 || off+8+cs > size {
break
}
switch id {
case "fmt ":
body, err := d.Range(off+8, cs)
if err == nil && len(body) >= 16 {
out.AudioFormat = binary.LittleEndian.Uint16(body[0:2])
out.NumChannels = binary.LittleEndian.Uint16(body[2:4])
out.SampleRate = binary.LittleEndian.Uint32(body[4:8])
out.BitsPerSample = binary.LittleEndian.Uint16(body[14:16])
}
case "data":
body, err := d.Range(off+8, cs)
if err != nil {
return nil, err
}
out.Data = body
out.Offset = off + 8
return out, nil
}
// Pad to even.
step := 8 + cs
if step%2 == 1 {
step++
}
off += step
}
return nil, errors.New("WAV: data chunk not found")
}
// ---------------------------------------------------------------------------
// LSB codec
// ---------------------------------------------------------------------------
// LSBExtract returns a bitstring-as-bytes built from the low bit of each
// byte in the selected range. Useful for exploring LSB payloads over raw
// buffers or specific regions.
func LSBExtract(buf []byte, bitsPerByte int) []byte {
if bitsPerByte <= 0 || bitsPerByte > 8 {
bitsPerByte = 1
}
total := len(buf) * bitsPerByte
out := make([]byte, total/8)
bitIdx := 0
for _, b := range buf {
for i := 0; i < bitsPerByte; i++ {
bit := (b >> uint(i)) & 1
if bit == 1 {
out[bitIdx/8] |= 1 << uint(7-(bitIdx%8))
}
bitIdx++
}
}
return out
}
// LSBInject replaces low bits in buf with bits from payload. Returns the
// modified buffer. When capacity is exceeded, injection stops.
func LSBInject(buf, payload []byte, bitsPerByte int) []byte {
if bitsPerByte <= 0 || bitsPerByte > 8 {
bitsPerByte = 1
}
out := append([]byte(nil), buf...)
mask := byte(1<<uint(bitsPerByte) - 1)
bitIdx := 0
total := len(payload) * 8
for i := 0; i < len(out) && bitIdx < total; i++ {
chunk := byte(0)
for b := 0; b < bitsPerByte && bitIdx < total; b++ {
bit := (payload[bitIdx/8] >> uint(7-(bitIdx%8))) & 1
if bit != 0 {
chunk |= 1 << uint(b)
}
bitIdx++
}
out[i] = (out[i] &^ mask) | (chunk & mask)
}
return out
}
internal/leviathan/transforms/cipher.go
+402
View File
@@ -0,0 +1,402 @@
package transforms
import (
"math"
)
// Caesar shifts A-Z/a-z forward by n, leaving other bytes alone.
func Caesar(in []byte, n int) []byte {
n = ((n % 26) + 26) % 26
out := make([]byte, len(in))
for i, b := range in {
switch {
case b >= 'A' && b <= 'Z':
out[i] = byte('A' + (int(b-'A')+n)%26)
case b >= 'a' && b <= 'z':
out[i] = byte('a' + (int(b-'a')+n)%26)
default:
out[i] = b
}
}
return out
}
// Atbash maps A..Z -> Z..A (and lowercase equivalently).
func Atbash(in []byte) []byte {
out := make([]byte, len(in))
for i, b := range in {
switch {
case b >= 'A' && b <= 'Z':
out[i] = 'A' + 'Z' - b
case b >= 'a' && b <= 'z':
out[i] = 'a' + 'z' - b
default:
out[i] = b
}
}
return out
}
// Vigenere applies the Vigenère cipher with an ASCII key. encrypt=true
// shifts forward, false shifts backward. Non-letters are preserved and
// do not advance the key.
func Vigenere(in, key []byte, encrypt bool) []byte {
if len(key) == 0 {
return append([]byte(nil), in...)
}
normKey := make([]int, 0, len(key))
for _, k := range key {
switch {
case k >= 'A' && k <= 'Z':
normKey = append(normKey, int(k-'A'))
case k >= 'a' && k <= 'z':
normKey = append(normKey, int(k-'a'))
}
}
if len(normKey) == 0 {
return append([]byte(nil), in...)
}
out := make([]byte, len(in))
ki := 0
for i, b := range in {
var base byte
var letter bool
if b >= 'A' && b <= 'Z' {
base = 'A'
letter = true
} else if b >= 'a' && b <= 'z' {
base = 'a'
letter = true
}
if letter {
shift := normKey[ki%len(normKey)]
if !encrypt {
shift = -shift
}
out[i] = byte(int(base) + ((int(b-base)+shift)%26+26)%26)
ki++
} else {
out[i] = b
}
}
return out
}
// English letter frequency (A..Z) in percent.
var englishFreq = [26]float64{
8.167, 1.492, 2.782, 4.253, 12.702, 2.228, 2.015, 6.094, 6.966, 0.153,
0.772, 4.025, 2.406, 6.749, 7.507, 1.929, 0.095, 5.987, 6.327, 9.056,
2.758, 0.978, 2.360, 0.150, 1.974, 0.074,
}
// ScoreEnglish returns a heuristic fit to English letter frequencies.
// Higher score = closer to English. ASCII-printable bonus added.
func ScoreEnglish(in []byte) float64 {
if len(in) == 0 {
return 0
}
var counts [26]int
total := 0
printable := 0
for _, b := range in {
if b >= 'A' && b <= 'Z' {
counts[b-'A']++
total++
} else if b >= 'a' && b <= 'z' {
counts[b-'a']++
total++
}
if (b >= 0x20 && b <= 0x7e) || b == '\n' || b == '\r' || b == '\t' {
printable++
}
}
printRatio := float64(printable) / float64(len(in))
if total == 0 {
return printRatio * 10 // give credit for printability
}
// Chi-squared inverse-like score
chi := 0.0
for i := 0; i < 26; i++ {
obs := float64(counts[i]) / float64(total) * 100
exp := englishFreq[i]
chi += math.Abs(obs - exp) // L1 distance; simpler than chi^2 for small samples
}
// Lower chi -> better. Convert to a bounded score.
score := (200.0 - chi) + printRatio*50
if score < 0 {
score = 0
}
return score
}
// BruteCaesar returns all 26 shifts with scores, best-first.
func BruteCaesar(in []byte) []map[string]any {
out := make([]map[string]any, 26)
for n := 0; n < 26; n++ {
dec := Caesar(in, n)
out[n] = map[string]any{
"shift": n,
"score": ScoreEnglish(dec),
"sample": previewText(dec, 120),
"text": string(dec),
}
}
// sort by score desc (simple insertion)
for i := 1; i < len(out); i++ {
for j := i; j > 0 && out[j-1]["score"].(float64) < out[j]["score"].(float64); j-- {
out[j-1], out[j] = out[j], out[j-1]
}
}
return out
}
// BruteXORByte returns scored 1-byte XOR candidates, best-first.
func BruteXORByte(in []byte) []map[string]any {
out := make([]map[string]any, 256)
for k := 0; k < 256; k++ {
buf := make([]byte, len(in))
for i, b := range in {
buf[i] = b ^ byte(k)
}
out[k] = map[string]any{
"key": k,
"keyHex": byteHex(byte(k)),
"score": ScoreEnglish(buf),
"sample": previewText(buf, 120),
"text": string(buf),
}
}
for i := 1; i < len(out); i++ {
for j := i; j > 0 && out[j-1]["score"].(float64) < out[j]["score"].(float64); j-- {
out[j-1], out[j] = out[j], out[j-1]
}
}
return out
}
// RepeatingXORGuess attempts to recover short repeating XOR keys using
// normalized Hamming distance for key-size ranking and per-column
// single-byte XOR scoring.
func RepeatingXORGuess(in []byte, minKey, maxKey, top int) []map[string]any {
if minKey <= 0 {
minKey = 2
}
if maxKey < minKey {
maxKey = minKey
}
if maxKey > 64 {
maxKey = 64
}
if top <= 0 {
top = 5
}
type keySizeScore struct {
size int
score float64
}
sizes := []keySizeScore{}
for keySize := minKey; keySize <= maxKey; keySize++ {
if len(in) < keySize*4 {
continue
}
dist := 0.0
pairs := 0
for block := 0; block < 4; block++ {
a := block * keySize
b := a + keySize
if b+keySize > len(in) {
break
}
dist += float64(hamming(in[a:a+keySize], in[b:b+keySize])) / float64(keySize)
pairs++
}
if pairs > 0 {
sizes = append(sizes, keySizeScore{size: keySize, score: dist / float64(pairs)})
}
}
for i := 1; i < len(sizes); i++ {
for j := i; j > 0 && sizes[j-1].score > sizes[j].score; j-- {
sizes[j-1], sizes[j] = sizes[j], sizes[j-1]
}
}
limit := top
if limit > len(sizes) {
limit = len(sizes)
}
out := make([]map[string]any, 0, limit)
for _, ks := range sizes[:limit] {
key := recoverRepeatingXORKey(in, ks.size)
dec := XORRepeating(in, key)
out = append(out, map[string]any{
"keySize": ks.size,
"distance": ks.score,
"keyHex": bytesHex(key),
"keyText": previewText(key, len(key)),
"score": ScoreEnglish(dec),
"sample": previewText(dec, 120),
"text": string(dec),
"normalizedScore": ScoreEnglish(dec) - ks.score,
})
}
for i := 1; i < len(out); i++ {
for j := i; j > 0 && out[j-1]["normalizedScore"].(float64) < out[j]["normalizedScore"].(float64); j-- {
out[j-1], out[j] = out[j], out[j-1]
}
}
return out
}
func recoverRepeatingXORKey(in []byte, keySize int) []byte {
key := make([]byte, keySize)
for pos := 0; pos < keySize; pos++ {
col := make([]byte, 0, len(in)/keySize+1)
for i := pos; i < len(in); i += keySize {
col = append(col, in[i])
}
bestScore := -1.0
bestKey := byte(0)
for k := 0; k < 256; k++ {
buf := make([]byte, len(col))
for i, b := range col {
buf[i] = b ^ byte(k)
}
score := ScoreEnglish(buf)
if score > bestScore {
bestScore = score
bestKey = byte(k)
}
}
key[pos] = bestKey
}
return key
}
func XORRepeating(in, key []byte) []byte {
if len(key) == 0 {
return append([]byte(nil), in...)
}
out := make([]byte, len(in))
for i, b := range in {
out[i] = b ^ key[i%len(key)]
}
return out
}
func hamming(a, b []byte) int {
n := len(a)
if len(b) < n {
n = len(b)
}
out := 0
for i := 0; i < n; i++ {
x := a[i] ^ b[i]
for x != 0 {
out++
x &= x - 1
}
}
return out
}
// TryEverything tries a handful of simple decodes and scores them.
func TryEverything(in []byte) []map[string]any {
candidates := []map[string]any{
{"name": "input", "text": string(in), "score": ScoreEnglish(in)},
{"name": "rot13", "text": string(Caesar(in, 13)), "score": ScoreEnglish(Caesar(in, 13))},
{"name": "atbash", "text": string(Atbash(in)), "score": ScoreEnglish(Atbash(in))},
}
if b, err := Base64Decode(in); err == nil && len(b) > 0 {
candidates = append(candidates, map[string]any{"name": "base64.decode", "text": string(b), "score": ScoreEnglish(b)})
}
if b, err := HexDecode(in); err == nil && len(b) > 0 {
candidates = append(candidates, map[string]any{"name": "hex.decode", "text": string(b), "score": ScoreEnglish(b)})
}
if b, err := URLDecode(in); err == nil && len(b) > 0 {
candidates = append(candidates, map[string]any{"name": "url.decode", "text": string(b), "score": ScoreEnglish(b)})
}
// Top 3 XOR 1-byte candidates
xr := BruteXORByte(in)
for i := 0; i < 3 && i < len(xr); i++ {
candidates = append(candidates, map[string]any{
"name": "xor1:" + xr[i]["keyHex"].(string),
"text": xr[i]["text"].(string),
"score": xr[i]["score"].(float64),
})
}
// Top repeating-key XOR candidates.
rx := RepeatingXORGuess(in, 2, 40, 3)
for i := 0; i < 3 && i < len(rx); i++ {
candidates = append(candidates, map[string]any{
"name": "xor-repeating:" + rx[i]["keyHex"].(string),
"keyHex": rx[i]["keyHex"].(string),
"keyText": rx[i]["keyText"].(string),
"text": rx[i]["text"].(string),
"score": rx[i]["score"].(float64),
})
}
// Top 2 caesar
cz := BruteCaesar(in)
for i := 0; i < 2 && i < len(cz); i++ {
candidates = append(candidates, map[string]any{
"name": "caesar:" + byteDec(byte(cz[i]["shift"].(int))),
"text": cz[i]["text"].(string),
"score": cz[i]["score"].(float64),
})
}
// Sort
for i := 1; i < len(candidates); i++ {
for j := i; j > 0 && candidates[j-1]["score"].(float64) < candidates[j]["score"].(float64); j-- {
candidates[j-1], candidates[j] = candidates[j], candidates[j-1]
}
}
// Samples
for _, c := range candidates {
t, _ := c["text"].(string)
c["sample"] = previewText([]byte(t), 120)
}
return candidates
}
func previewText(buf []byte, n int) string {
if n > len(buf) {
n = len(buf)
}
out := make([]byte, n)
for i := 0; i < n; i++ {
if buf[i] >= 0x20 && buf[i] <= 0x7e {
out[i] = buf[i]
} else if buf[i] == '\n' {
out[i] = ' '
} else {
out[i] = '.'
}
}
return string(out)
}
func byteHex(b byte) string {
const h = "0123456789abcdef"
return string([]byte{h[b>>4], h[b&0x0f]})
}
func bytesHex(buf []byte) string {
out := make([]byte, 0, len(buf)*2)
for _, b := range buf {
out = append(out, byteHex(b)...)
}
return string(out)
}
func byteDec(b byte) string {
if b == 0 {
return "0"
}
out := [3]byte{}
i := 3
for b > 0 {
i--
out[i] = '0' + b%10
b /= 10
}
return string(out[i:])
}
internal/leviathan/transforms/transforms.go
+237
View File
@@ -0,0 +1,237 @@
// Package transforms exposes chainable transform primitives. Each
// function takes an input []byte and options, and returns an output
// or an error. They are pure and do not mutate input.
package transforms
import (
"bytes"
"compress/flate"
"compress/gzip"
"compress/zlib"
"encoding/base64"
"encoding/hex"
"errors"
"io"
"net/url"
)
const MaxInflateOutput = 64 << 20
// HexEncode returns ASCII hex of src.
func HexEncode(src []byte) ([]byte, error) {
return []byte(hex.EncodeToString(src)), nil
}
// HexDecode parses ASCII hex into bytes. Whitespace is ignored.
func HexDecode(src []byte) ([]byte, error) {
stripped := stripHex(src)
out := make([]byte, len(stripped)/2)
_, err := hex.Decode(out, stripped)
return out, err
}
func stripHex(s []byte) []byte {
out := make([]byte, 0, len(s))
for _, b := range s {
if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == ',' || b == ':' {
continue
}
if b == '0' && len(out) == 0 {
continue
}
out = append(out, b)
}
return out
}
// Base64Encode returns the standard Base64 of src.
func Base64Encode(src []byte) ([]byte, error) {
return []byte(base64.StdEncoding.EncodeToString(src)), nil
}
// Base64Decode accepts standard or URL-safe Base64, with or without padding.
func Base64Decode(src []byte) ([]byte, error) {
s := string(bytes.TrimSpace(src))
attempts := []*base64.Encoding{
base64.StdEncoding,
base64.URLEncoding,
base64.RawStdEncoding,
base64.RawURLEncoding,
}
for _, enc := range attempts {
if b, err := enc.DecodeString(s); err == nil {
return b, nil
}
}
return nil, errors.New("not valid base64")
}
// URLEncode applies percent-encoding.
func URLEncode(src []byte) ([]byte, error) {
return []byte(url.QueryEscape(string(src))), nil
}
// URLDecode reverses percent-encoding.
func URLDecode(src []byte) ([]byte, error) {
v, err := url.QueryUnescape(string(src))
if err != nil {
return nil, err
}
return []byte(v), nil
}
// XOR returns src^key, cycling key.
func XOR(src, key []byte) ([]byte, error) {
if len(key) == 0 {
return nil, errors.New("empty key")
}
out := make([]byte, len(src))
for i := range src {
out[i] = src[i] ^ key[i%len(key)]
}
return out, nil
}
// InflateGzip, InflateZlib, InflateRaw attempt to decompress.
func InflateGzip(src []byte) ([]byte, error) {
r, err := gzip.NewReader(bytes.NewReader(src))
if err != nil {
return nil, err
}
defer r.Close()
return readLimitedInflate(r)
}
func InflateZlib(src []byte) ([]byte, error) {
r, err := zlib.NewReader(bytes.NewReader(src))
if err != nil {
return nil, err
}
defer r.Close()
return readLimitedInflate(r)
}
func InflateRaw(src []byte) ([]byte, error) {
r := flate.NewReader(bytes.NewReader(src))
defer r.Close()
return readLimitedInflate(r)
}
func readLimitedInflate(r io.Reader) ([]byte, error) {
lr := io.LimitReader(r, MaxInflateOutput+1)
out, err := io.ReadAll(lr)
if err != nil {
return nil, err
}
if len(out) > MaxInflateOutput {
return nil, errors.New("decompressed output exceeds 64 MiB limit")
}
return out, nil
}
// TryInflate attempts gzip, zlib, and raw deflate in order.
func TryInflate(src []byte) (out []byte, kind string, err error) {
if b, e := InflateGzip(src); e == nil {
return b, "gzip", nil
}
if b, e := InflateZlib(src); e == nil {
return b, "zlib", nil
}
if b, e := InflateRaw(src); e == nil {
return b, "raw", nil
}
return nil, "", errors.New("no decompression succeeded")
}
// Step is one entry in a transform pipeline.
type Step struct {
Op string // hex.encode, hex.decode, base64.encode, base64.decode,
// xor, inflate, url.encode, url.decode, caesar, rot13, atbash,
// vigenere.enc, vigenere.dec
Arg string // ASCII parameter (e.g. key for XOR/Vigenere, shift for Caesar)
}
// ApplyPipeline runs steps left-to-right. Intermediate results are
// returned for inspection.
func ApplyPipeline(input []byte, steps []Step) ([][]byte, error) {
results := make([][]byte, 0, len(steps)+1)
cur := append([]byte(nil), input...)
results = append(results, cur)
for _, s := range steps {
out, err := Apply(cur, s)
if err != nil {
return results, err
}
cur = out
results = append(results, cur)
}
return results, nil
}
// Apply one step.
func Apply(in []byte, s Step) ([]byte, error) {
switch s.Op {
case "hex.encode":
return HexEncode(in)
case "hex.decode":
return HexDecode(in)
case "base64.encode":
return Base64Encode(in)
case "base64.decode":
return Base64Decode(in)
case "url.encode":
return URLEncode(in)
case "url.decode":
return URLDecode(in)
case "xor":
key, err := parseKey(s.Arg)
if err != nil {
return nil, err
}
return XOR(in, key)
case "inflate":
switch s.Arg {
case "gzip":
return InflateGzip(in)
case "zlib":
return InflateZlib(in)
case "raw":
return InflateRaw(in)
default:
out, _, err := TryInflate(in)
return out, err
}
case "caesar":
n := 13
if s.Arg != "" {
var v int
for _, c := range s.Arg {
if c >= '0' && c <= '9' {
v = v*10 + int(c-'0')
}
}
n = v
}
return Caesar(in, n), nil
case "rot13":
return Caesar(in, 13), nil
case "atbash":
return Atbash(in), nil
case "vigenere.enc":
return Vigenere(in, []byte(s.Arg), true), nil
case "vigenere.dec":
return Vigenere(in, []byte(s.Arg), false), nil
}
return nil, errors.New("unknown op: " + s.Op)
}
// parseKey accepts either literal string (ASCII) or hex-prefixed "hex:..".
func parseKey(arg string) ([]byte, error) {
if len(arg) > 4 && arg[:4] == "hex:" {
return hex.DecodeString(arg[4:])
}
if arg == "" {
return nil, errors.New("empty key")
}
return []byte(arg), nil
}
internal/recording/recorder.go
+22 -11
View File
@@ -9,6 +9,7 @@
package recording
import (
"bufio"
"bytes"
"encoding/json"
"fmt"
@@ -33,6 +34,9 @@ const (
MaxRecordingBytes = int64(250 * 1024 * 1024)
// MaxPlaybackBytes bounds the file size loaded for playback.
MaxPlaybackBytes = int64(100 * 1024 * 1024)
// MaxPlaybackLineBytes bounds one asciicast line during streaming playback
// parsing. Terminal events should stay well below this in normal use.
MaxPlaybackLineBytes = 8 * 1024 * 1024
)
// Header is the asciicast v2 file header (first line).
@@ -298,26 +302,30 @@ func (s *Store) getWithLimit(id string, maxBytes int64) (*Header, []Event, error
if maxBytes > 0 && info.Size() > maxBytes {
return nil, nil, fmt.Errorf("recording: file is too large for playback (max %d bytes)", maxBytes)
}
data, err := os.ReadFile(path)
file, err := os.Open(path)
if err != nil {
return nil, nil, fmt.Errorf("recording: failed to read file: %w", err)
}
lines := strings.Split(strings.TrimSpace(string(data)), "\n")
if len(lines) < 1 {
return nil, nil, fmt.Errorf("recording: empty file")
return nil, nil, fmt.Errorf("recording: failed to open file: %w", err)
}
defer file.Close()
// Parse header (first line).
scanner := bufio.NewScanner(file)
scanner.Buffer(make([]byte, 0, 64*1024), MaxPlaybackLineBytes)
if !scanner.Scan() {
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("recording: failed to scan header: %w", err)
}
return nil, nil, fmt.Errorf("recording: empty file")
}
var hdr Header
if err := json.Unmarshal([]byte(lines[0]), &hdr); err != nil {
if err := json.Unmarshal([]byte(strings.TrimSpace(scanner.Text())), &hdr); err != nil {
return nil, nil, fmt.Errorf("recording: invalid header: %w", err)
}
// Parse events (remaining lines).
events := make([]Event, 0, len(lines)-1)
for _, line := range lines[1:] {
line = strings.TrimSpace(line)
events := make([]Event, 0)
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
continue
}
@@ -347,6 +355,9 @@ func (s *Store) getWithLimit(id string, maxBytes int64) (*Header, []Event, error
Data: evtData,
})
}
if err := scanner.Err(); err != nil {
return nil, nil, fmt.Errorf("recording: failed to scan events: %w", err)
}
return &hdr, events, nil
}
internal/secrets/secrets.go
+12
View File
@@ -28,6 +28,8 @@ const (
argonKeyLen = 32
saltLen = 32
maxVaultFileBytes = 10 * 1024 * 1024
)
// SecretStore manages encrypted credential storage.
@@ -331,6 +333,16 @@ func (s *SecretStore) save() error {
// load reads and unmarshals the vault from disc.
func (s *SecretStore) load() (*secretVault, error) {
info, err := os.Stat(s.vaultPath())
if err != nil {
return nil, fmt.Errorf("failed to stat vault file: %w", err)
}
if info.IsDir() {
return nil, errors.New("vault path is a directory")
}
if info.Size() > maxVaultFileBytes {
return nil, fmt.Errorf("vault file is too large: %d bytes exceeds %d bytes", info.Size(), maxVaultFileBytes)
}
data, err := os.ReadFile(s.vaultPath())
if err != nil {
return nil, fmt.Errorf("failed to read vault file: %w", err)
internal/secrets/secrets_test.go
+26
View File
@@ -1,7 +1,9 @@
package secrets
import (
"os"
"path/filepath"
"strings"
"testing"
)
@@ -267,6 +269,30 @@ func TestVaultPath(t *testing.T) {
}
}
func TestSecretStoreLoadRejectsOversizedVault(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, vaultFileName)
file, err := os.Create(path)
if err != nil {
t.Fatalf("Create() failed: %v", err)
}
if err := file.Truncate(maxVaultFileBytes + 1); err != nil {
t.Fatalf("Truncate() failed: %v", err)
}
if err := file.Close(); err != nil {
t.Fatalf("Close() failed: %v", err)
}
store := NewSecretStore(dir)
_, err = store.load()
if err == nil {
t.Fatal("expected oversized vault to be rejected")
}
if !strings.Contains(err.Error(), "vault file is too large") {
t.Fatalf("unexpected error: %v", err)
}
}
// ---------------------------------------------------------------------------
// Redaction helpers
// ---------------------------------------------------------------------------
internal/sessions/manager.go
+12
View File
@@ -32,6 +32,8 @@ import (
"github.com/andydixon/nexus/internal/vncclient"
)
const maxVNCTunnelPrivateKeyBytes = 1 << 20
// Session wraps a protocol client with metadata and state.
type Session struct {
Info models.SessionInfo
@@ -1031,6 +1033,16 @@ func (m *Manager) connectVNC(sess *Session, getSecret func(prompt string) (strin
cfg.SSHPassword = password
case "publickey":
if vncConf.SSHPrivateKey != "" {
info, err := os.Stat(vncConf.SSHPrivateKey)
if err != nil {
return fmt.Errorf("reading SSH key metadata: %w", err)
}
if info.IsDir() {
return fmt.Errorf("SSH key path is a directory")
}
if info.Size() > maxVNCTunnelPrivateKeyBytes {
return fmt.Errorf("SSH key is too large: %d bytes exceeds %d bytes", info.Size(), maxVNCTunnelPrivateKeyBytes)
}
keyData, err := os.ReadFile(vncConf.SSHPrivateKey)
if err != nil {
return fmt.Errorf("reading SSH key: %w", err)
internal/sshclient/sshclient.go
+12
View File
@@ -20,6 +20,8 @@ import (
"golang.org/x/net/proxy"
)
const maxSSHPrivateKeyBytes = 1 << 20
// Client manages a single SSH session lifecycle.
// It handles connection establishment, authentication, PTY allocation,
// terminal resize propagation, port forwarding, and clean shutdown.
@@ -664,6 +666,16 @@ func jumpHostClientConfig(jump models.JumpHost, auth []ssh.AuthMethod, finalConf
// parsePrivateKey loads and parses an SSH private key file. If the key is
// encrypted and getPassphrase is provided, it will attempt decryption.
func (c *Client) parsePrivateKey(path string, getPassphrase func() (string, error)) (ssh.Signer, error) {
info, err := os.Stat(path)
if err != nil {
return nil, fmt.Errorf("unable to stat key file: %w", err)
}
if info.IsDir() {
return nil, fmt.Errorf("key path is a directory")
}
if info.Size() > maxSSHPrivateKeyBytes {
return nil, fmt.Errorf("key file is too large: %d bytes exceeds %d bytes", info.Size(), maxSSHPrivateKeyBytes)
}
keyData, err := os.ReadFile(path)
if err != nil {
return nil, fmt.Errorf("unable to read key file: %w", err)
internal/updater/updater.go
+22 -2
View File
@@ -25,6 +25,10 @@ const (
// requestTimeout limits the HTTP request duration.
requestTimeout = 10 * time.Second
// maxUpdateResponseBytes bounds release and manifest responses before
// parsing them into memory.
maxUpdateResponseBytes = 1 << 20
)
// Release describes a GitHub release with the fields we care about.
@@ -135,8 +139,13 @@ func (c *Checker) Check(ctx context.Context) (*UpdateInfo, error) {
return nil, fmt.Errorf("updater: GitHub API returned %d", resp.StatusCode)
}
body, err := readLimitedUpdateBody(resp.Body)
if err != nil {
return nil, fmt.Errorf("updater: failed to read response: %w", err)
}
var release Release
if err := json.NewDecoder(resp.Body).Decode(&release); err != nil {
if err := json.Unmarshal(body, &release); err != nil {
return nil, fmt.Errorf("updater: failed to decode response: %w", err)
}
@@ -186,7 +195,7 @@ func (c *URLChecker) Check(ctx context.Context) (*UpdateInfo, error) {
return nil, fmt.Errorf("updater: update server returned %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
body, err := readLimitedUpdateBody(resp.Body)
if err != nil {
return nil, fmt.Errorf("updater: failed to read response: %w", err)
}
@@ -215,6 +224,17 @@ func normaliseVersion(v string) string {
return v
}
func readLimitedUpdateBody(r io.Reader) ([]byte, error) {
body, err := io.ReadAll(io.LimitReader(r, maxUpdateResponseBytes+1))
if err != nil {
return nil, err
}
if len(body) > maxUpdateResponseBytes {
return nil, fmt.Errorf("response exceeds %d bytes", maxUpdateResponseBytes)
}
return body, nil
}
func parseManifest(body []byte, current string) (*UpdateInfo, bool) {
var manifest Manifest
if err := json.Unmarshal(body, &manifest); err != nil {
internal/updater/updater_test.go
+20 -3
View File
@@ -37,9 +37,9 @@ func TestIsNewer(t *testing.T) {
{"1.0.1", "1.0.0", true},
{"1.0.0", "1.0.0", false},
{"0.9.0", "1.0.0", false},
{"1.0.0", "dev", false}, // dev builds ignored
{"1.0.0", "", false}, // empty current ignored
{"1.2", "1.1.0", true}, // fewer parts
{"1.0.0", "dev", false}, // dev builds ignored
{"1.0.0", "", false}, // empty current ignored
{"1.2", "1.1.0", true}, // fewer parts
{"1.0.0", "1.0.0.1", false},
}
for _, tt := range tests {
@@ -104,3 +104,20 @@ func TestURLCheckerPlainText(t *testing.T) {
t.Fatalf("expected update to be available")
}
}
func TestURLCheckerRejectsOversizedManifest(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte(strings.Repeat("x", maxUpdateResponseBytes+1)))
}))
defer server.Close()
checker := NewURLChecker(server.URL, "1.9.0")
_, err := checker.Check(context.Background())
if err == nil {
t.Fatal("expected oversized manifest to be rejected")
}
if !strings.Contains(err.Error(), "response exceeds") {
t.Fatalf("unexpected error: %v", err)
}
}