From 493bfed599d498aa1a05206459ce6e68e1c04b21 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Tue, 16 Jun 2026 11:51:07 +0100 Subject: [PATCH] v3 candidate --- .../20260616-1111-functionalityNextSteps.md | 81 + audits/20260616-1111-securityAudit.md | 135 + frontend/index.html | 19 +- frontend/package.json.md5 | 2 +- frontend/scripts/test-regressions.js | 95 + frontend/src/css/leviathan.css | 269 ++ frontend/src/css/sidebar.css | 47 + frontend/src/js/app.js | 32 + frontend/src/js/leviathan.js | 681 +++++ frontend/src/js/sidebar.js | 4 + frontend/src/js/tabs.js | 1 + frontend/src/js/terminal.js | 91 +- frontend/wailsjs/go/api/Service.d.ts | 34 + frontend/wailsjs/go/api/Service.js | 68 + frontend/wailsjs/go/models.ts | 42 + go.mod | 3 + go.sum | 10 +- internal/api/api.go | 22 +- internal/api/api_test.go | 59 + internal/api/leviathan.go | 907 +++++++ internal/githubclient/githubclient.go | 94 +- internal/githubclient/githubclient_test.go | 26 + internal/leviathan/analysis/carve.go | 876 +++++++ internal/leviathan/analysis/diff.go | 244 ++ internal/leviathan/analysis/entropy.go | 125 + internal/leviathan/analysis/fuzzy.go | 224 ++ internal/leviathan/analysis/hash.go | 63 + internal/leviathan/analysis/inspector.go | 162 ++ internal/leviathan/analysis/ioc.go | 89 + internal/leviathan/analysis/keys.go | 367 +++ internal/leviathan/analysis/multisearch.go | 113 + internal/leviathan/analysis/overview.go | 88 + internal/leviathan/analysis/rules.go | 648 +++++ internal/leviathan/analysis/search.go | 189 ++ internal/leviathan/analysis/strings.go | 198 ++ internal/leviathan/analysis/timestamps.go | 106 + internal/leviathan/binsnoop/README.md | 277 +++ internal/leviathan/binsnoop/binsnoop.go | 177 ++ internal/leviathan/binsnoop/common.go | 127 + internal/leviathan/binsnoop/common_test.go | 22 + internal/leviathan/binsnoop/extract.go | 124 + internal/leviathan/binsnoop/magic.go | 268 ++ internal/leviathan/binsnoop/match.go | 134 + internal/leviathan/binsnoop/parse.go | 107 + internal/leviathan/binsnoop/scanner.go | 352 +++ internal/leviathan/binsnoop/sig_archives.go | 1143 +++++++++ .../leviathan/binsnoop/sig_compression.go | 1081 ++++++++ internal/leviathan/binsnoop/sig_crypto.go | 795 ++++++ .../leviathan/binsnoop/sig_executables.go | 485 ++++ .../leviathan/binsnoop/sig_filesystems.go | 1289 ++++++++++ internal/leviathan/binsnoop/sig_firmware.go | 1333 ++++++++++ internal/leviathan/binsnoop/sig_images.go | 677 +++++ internal/leviathan/binsnoop/sig_misc.go | 1399 +++++++++++ internal/leviathan/binsnoop/types.go | 117 + internal/leviathan/disasm/aarch64.go | 288 +++ internal/leviathan/disasm/arm32.go | 209 ++ internal/leviathan/disasm/cfg.go | 541 ++++ internal/leviathan/disasm/image.go | 74 + internal/leviathan/disasm/mips.go | 144 ++ internal/leviathan/disasm/thumb.go | 117 + internal/leviathan/disasm/x86.go | 91 + internal/leviathan/disasm/x86_decode.go | 1172 +++++++++ internal/leviathan/document/document.go | 439 ++++ internal/leviathan/document/session.go | 62 + internal/leviathan/parsers/asn1.go | 321 +++ internal/leviathan/parsers/cert.go | 196 ++ internal/leviathan/parsers/executable.go | 412 ++++ internal/leviathan/parsers/formats.go | 2182 +++++++++++++++++ internal/leviathan/parsers/formats_more.go | 662 +++++ internal/leviathan/parsers/macho.go | 408 +++ internal/leviathan/parsers/peimports.go | 150 ++ internal/leviathan/patch/script.go | 232 ++ internal/leviathan/pdf/filters.go | 411 ++++ internal/leviathan/pdf/pdf.go | 352 +++ internal/leviathan/pdf/pdf_test.go | 37 + internal/leviathan/pdf/surface.go | 128 + internal/leviathan/pdf/xref.go | 422 ++++ internal/leviathan/spectrogram/spectrogram.go | 798 ++++++ .../leviathan/spectrogram/spectrogram_test.go | 177 ++ internal/leviathan/squashfs/squashfs.go | 782 ++++++ internal/leviathan/stego/metadata.go | 694 ++++++ internal/leviathan/stego/stego.go | 501 ++++ internal/leviathan/transforms/cipher.go | 402 +++ internal/leviathan/transforms/transforms.go | 237 ++ internal/recording/recorder.go | 33 +- internal/secrets/secrets.go | 12 + internal/secrets/secrets_test.go | 26 + internal/sessions/manager.go | 12 + internal/sshclient/sshclient.go | 12 + internal/updater/updater.go | 24 +- internal/updater/updater_test.go | 23 +- 91 files changed, 28824 insertions(+), 80 deletions(-) create mode 100644 audits/20260616-1111-functionalityNextSteps.md create mode 100644 audits/20260616-1111-securityAudit.md create mode 100644 frontend/src/css/leviathan.css create mode 100644 frontend/src/js/leviathan.js create mode 100644 internal/api/leviathan.go create mode 100644 internal/githubclient/githubclient_test.go create mode 100644 internal/leviathan/analysis/carve.go create mode 100644 internal/leviathan/analysis/diff.go create mode 100644 internal/leviathan/analysis/entropy.go create mode 100644 internal/leviathan/analysis/fuzzy.go create mode 100644 internal/leviathan/analysis/hash.go create mode 100644 internal/leviathan/analysis/inspector.go create mode 100644 internal/leviathan/analysis/ioc.go create mode 100644 internal/leviathan/analysis/keys.go create mode 100644 internal/leviathan/analysis/multisearch.go create mode 100644 internal/leviathan/analysis/overview.go create mode 100644 internal/leviathan/analysis/rules.go create mode 100644 internal/leviathan/analysis/search.go create mode 100644 internal/leviathan/analysis/strings.go create mode 100644 internal/leviathan/analysis/timestamps.go create mode 100644 internal/leviathan/binsnoop/README.md create mode 100644 internal/leviathan/binsnoop/binsnoop.go create mode 100644 internal/leviathan/binsnoop/common.go create mode 100644 internal/leviathan/binsnoop/common_test.go create mode 100644 internal/leviathan/binsnoop/extract.go create mode 100644 internal/leviathan/binsnoop/magic.go create mode 100644 internal/leviathan/binsnoop/match.go create mode 100644 internal/leviathan/binsnoop/parse.go create mode 100644 internal/leviathan/binsnoop/scanner.go create mode 100644 internal/leviathan/binsnoop/sig_archives.go create mode 100644 internal/leviathan/binsnoop/sig_compression.go create mode 100644 internal/leviathan/binsnoop/sig_crypto.go create mode 100644 internal/leviathan/binsnoop/sig_executables.go create mode 100644 internal/leviathan/binsnoop/sig_filesystems.go create mode 100644 internal/leviathan/binsnoop/sig_firmware.go create mode 100644 internal/leviathan/binsnoop/sig_images.go create mode 100644 internal/leviathan/binsnoop/sig_misc.go create mode 100644 internal/leviathan/binsnoop/types.go create mode 100644 internal/leviathan/disasm/aarch64.go create mode 100644 internal/leviathan/disasm/arm32.go create mode 100644 internal/leviathan/disasm/cfg.go create mode 100644 internal/leviathan/disasm/image.go create mode 100644 internal/leviathan/disasm/mips.go create mode 100644 internal/leviathan/disasm/thumb.go create mode 100644 internal/leviathan/disasm/x86.go create mode 100644 internal/leviathan/disasm/x86_decode.go create mode 100644 internal/leviathan/document/document.go create mode 100644 internal/leviathan/document/session.go create mode 100644 internal/leviathan/parsers/asn1.go create mode 100644 internal/leviathan/parsers/cert.go create mode 100644 internal/leviathan/parsers/executable.go create mode 100644 internal/leviathan/parsers/formats.go create mode 100644 internal/leviathan/parsers/formats_more.go create mode 100644 internal/leviathan/parsers/macho.go create mode 100644 internal/leviathan/parsers/peimports.go create mode 100644 internal/leviathan/patch/script.go create mode 100644 internal/leviathan/pdf/filters.go create mode 100644 internal/leviathan/pdf/pdf.go create mode 100644 internal/leviathan/pdf/pdf_test.go create mode 100644 internal/leviathan/pdf/surface.go create mode 100644 internal/leviathan/pdf/xref.go create mode 100644 internal/leviathan/spectrogram/spectrogram.go create mode 100644 internal/leviathan/spectrogram/spectrogram_test.go create mode 100644 internal/leviathan/squashfs/squashfs.go create mode 100644 internal/leviathan/stego/metadata.go create mode 100644 internal/leviathan/stego/stego.go create mode 100644 internal/leviathan/transforms/cipher.go create mode 100644 internal/leviathan/transforms/transforms.go diff --git a/audits/20260616-1111-functionalityNextSteps.md b/audits/20260616-1111-functionalityNextSteps.md new file mode 100644 index 0000000..9967154 --- /dev/null +++ b/audits/20260616-1111-functionalityNextSteps.md @@ -0,0 +1,81 @@ +# Functionality and Next Steps Report + +## 1. Executive Summary + +This fix pass moves Nexus closer to predictable large-file behavior: file-backed Leviathan documents stay file-backed for save/export, recordings parse incrementally, and unavoidable in-memory parsers now advertise and enforce clear limits. + +The most useful next work is to surface these limits in the Leviathan UI and add user-facing guidance when an operation needs a smaller selection or a streaming implementation. + +## 2. Current Functional Risks + +Leviathan spectrogram conversion still uses bounded in-memory audio/image buffers because the current encoder/decoder APIs operate on byte slices. Very large images, MP3s, WAVs, or decoded MP3 output will now be rejected with an error rather than processed. + +PDF, certificate, and key scanners still operate on bounded whole-document buffers for structural parsing. The cap avoids memory denial of service, but users working with very large firmware or document corpora may need range-based scans in future. + +Recording playback is now line-streamed, but the returned API shape still collects parsed events before sending them to the frontend. Very long recordings are still protected by the playback cap; true progressive playback would require an API contract change. + +## 3. Code Quality Improvements Completed + +Chunk-first behavior: + +- Leviathan working-copy save now writes with `Document.ForEachChunk`. +- Leviathan case export now streams `working.bin` into the zip file on disk. +- Recording playback parses line-by-line instead of reading and splitting the whole file. + +Bounded read behavior: + +- Spectrogram image/audio inputs reject oversize data before decode. +- Image decode checks dimensions before allocating decoded pixels. +- HTTP response helpers cap JSON and error bodies. +- SSH key, VNC key, vault, public-key, PDF, and BinSnoop file reads have explicit limits. + +Responsiveness: + +- Heavy Leviathan spectrogram/export operations no longer hold the engine lock while processing or showing native save dialogs. + +## 4. Frontend Interaction and Error Handling + +Recommended UI follow-up: + +- Show limit-aware messages for spectrogram tools, such as image max size, audio max size, and decoded image dimension limits. +- For PDF/cert/key tools, show “scan a smaller file or selected range” guidance when caps are hit. +- Add a small status line in Leviathan for whether an operation is streaming, bounded-buffer, or selection-based. +- For recording playback, consider progressive playback loading so long recordings do not need all parsed events returned at once. + +## 5. Testing Roadmap + +Completed in this pass: + +- Oversized spectrogram image dimensions are rejected before full decode. +- Leviathan bounded document reads reject oversize documents. +- Patched working-file writes preserve changed bytes. +- Oversized GitHub/update responses are rejected. +- Oversized SSH public keys, vault files, PDFs, and BinSnoop files are rejected. + +Recommended next tests: + +- Wails UI tests for Leviathan oversized image/audio/PDF error messages. +- A case-export test that opens a sparse file-backed document and verifies the zip is produced without materializing `working.bin`. +- A recording playback test with malformed lines and a long valid event stream. +- Browser interaction tests for the Leviathan tab while a long analysis is running. + +## 6. Recommended New Functionality + +Add range-based versions of whole-document Leviathan tools: + +- `keys.scanRange` +- `certs.scanRange` +- `pdf.quickScan` for headers/xref/trailer without full structural recovery +- `spectrogram.renderRange` for selected audio windows + +Add streaming media support later: + +- Decode audio from file/range readers instead of complete byte slices. +- Stream WAV output directly to disk for spectrogram generation. +- Stream image insertion output to disk when carrier audio is large. + +## 7. Caveats + +The new caps trade permissiveness for stability. Files above the limits now fail fast instead of attempting best-effort processing. + +The frontend build and regression checks passed, but I did not run a packaged Wails app manually in this pass. diff --git a/audits/20260616-1111-securityAudit.md b/audits/20260616-1111-securityAudit.md new file mode 100644 index 0000000..b2e3f09 --- /dev/null +++ b/audits/20260616-1111-securityAudit.md @@ -0,0 +1,135 @@ +# Security Audit Report + +## 1. Executive Summary + +This fix-mode audit focused on file-reading and response-reading paths that could allocate complete files or unbounded remote bodies in memory. + +The main risk fixed was local denial of service from large user-selected files, imported Leviathan artefacts, recordings, SSH key files, vault files, and remote update/GitHub API responses. Where the codebase already had a chunked document model, saves and exports now use it. Where a parser still requires a complete byte slice, the code now rejects oversized inputs before allocation. + +Verification passed with targeted package tests, `go test ./...`, frontend regression tests, frontend build, and `git diff --check`. + +## 2. Scope Reviewed + +Reviewed and changed: + +- Leviathan file-backed document save, case export, spectrogram image/audio conversion, PDF parsing, and BinSnoop file helpers. +- Wails API helpers that read SSH public keys and Leviathan image/audio inputs. +- SSH and VNC tunnel private-key loading. +- GitHub client and update-checker HTTP response handling. +- Recording playback parsing. +- Encrypted secrets vault loading. + +Not live-tested: + +- A packaged Wails desktop binary. +- Real remote SSH/VNC/GitHub endpoints. +- Very large real-world media corpora beyond automated guard tests. + +## 3. Methodology + +I searched for `ReadFile`, `ReadAll`, and whole-document `Range(0, size)` patterns, then classified each as chunked, bounded, embedded/test-only, or unsafe. Fixes were kept to low-risk local changes: + +- Prefer `Document.ForEachChunk` for writing/exporting working file bytes. +- Stream line-oriented recordings with `bufio.Scanner`. +- Limit HTTP response reads with `io.LimitReader`. +- Add `os.Stat` size checks before unavoidable bounded `os.ReadFile` calls. +- Add parser-specific caps before whole-document reads. + +## 4. Findings Summary + +| ID | Severity | Confidence | Title | Component | Status | +|---|---:|---:|---|---|---| +| SEC-20260616-001 | Medium | High | Leviathan save/export and spectrogram tools could materialize large files | Leviathan API, spectrogram | Fixed | +| SEC-20260616-002 | Medium | High | Remote API/update responses were read without size limits | GitHub client, updater | Fixed | +| SEC-20260616-003 | Medium | High | Local app data/key reads lacked size caps | SSH keys, VNC keys, vault, BinSnoop | Fixed | +| SEC-20260616-004 | Low | High | Recording playback loaded the whole capped recording file | Recording playback | Fixed | +| SEC-20260616-005 | Low | Medium | Some parsers still require complete bounded buffers | PDF, cert/key scanners, media decoders | Accepted with caps | + +## 5. Detailed Findings + +### SEC-20260616-001: Leviathan full-file allocation paths + +Leviathan save/export and spectrogram operations could duplicate entire working files in memory. Case export also built the zip archive in memory before writing it. + +Fixed in: + +- `internal/api/leviathan.go` +- `internal/leviathan/spectrogram/spectrogram.go` +- `internal/leviathan/pdf/pdf.go` + +Changes: + +- Working-copy save writes via `Document.ForEachChunk`. +- Case export streams `working.bin` directly into `zip.Writer` backed by the destination file. +- Spectrogram encode/view/insert rejects oversized image/audio documents before bounded decode. +- Image decoding checks compressed byte size and decoded pixel dimensions before full image decode. +- PDF parsing rejects files above 128 MiB before whole-document structural parsing. +- Long-running spectrogram/export tools no longer hold the Leviathan engine lock while doing heavy work or native save dialogs. + +### SEC-20260616-002: Unbounded remote response reads + +GitHub and update-check response bodies could be read without an explicit maximum. + +Fixed in: + +- `internal/githubclient/githubclient.go` +- `internal/updater/updater.go` + +Changes: + +- GitHub JSON responses are capped at 4 MiB. +- GitHub error bodies are capped at 64 KiB. +- Update release/manifest responses are capped at 1 MiB. +- Repository archive downloads continue to stream to disk instead of memory. + +### SEC-20260616-003: Local key/vault/helper reads lacked caps + +Several local helper paths read files directly even though the expected files are small. + +Fixed in: + +- `internal/api/api.go` +- `internal/sshclient/sshclient.go` +- `internal/sessions/manager.go` +- `internal/secrets/secrets.go` +- `internal/leviathan/binsnoop/common.go` +- `internal/leviathan/binsnoop/binsnoop.go` + +Changes: + +- SSH public keys are capped at 64 KiB. +- SSH private keys for SSH and VNC tunnel auth are capped at 1 MiB. +- Secret vault files are capped at 10 MiB. +- BinSnoop direct file helpers are capped at 64 MiB; the UI path already uses a bounded buffer. + +### SEC-20260616-004: Recording playback whole-file read + +Recording playback had a 100 MiB file cap, but still read and split the full file in memory. + +Fixed in: + +- `internal/recording/recorder.go` + +Changes: + +- Playback now opens the recording and scans line-by-line. +- Individual playback lines are capped at 8 MiB. +- Existing malformed-event tolerance is preserved. + +## 6. Residual Risk + +Some operations still need bounded complete buffers because downstream parsers or decoders require byte slices: image/audio spectrogram conversion, PDF structural parsing, certificate/key scanning, and small SSH/vault/public-key parsing. These now fail before allocation when inputs exceed explicit caps. + +Truly streaming versions of every binary parser would be a larger Leviathan architecture project. The current fixes remove the unbounded allocation behavior while preserving tool behavior for normal files. + +## 7. Verification + +Passed: + +- `go test ./internal/api ./internal/leviathan/spectrogram ./internal/leviathan/pdf ./internal/leviathan/binsnoop ./internal/updater ./internal/githubclient ./internal/secrets ./internal/recording ./internal/sshclient ./internal/sessions -count=1` +- `go test ./... -count=1` +- `npm --prefix frontend test` +- `npm --prefix frontend run build` +- `git diff --check` + +New tests cover oversized GitHub/update bodies, oversized spectrogram images, bounded Leviathan document reads, chunked patched-file writes, oversized public keys, oversized vault files, oversized PDFs, and BinSnoop file caps. diff --git a/frontend/index.html b/frontend/index.html index e6baa1e..b8ec7fb 100644 --- a/frontend/index.html +++ b/frontend/index.html @@ -23,6 +23,7 @@ +
@@ -47,7 +48,22 @@ - Nexus Terminal + + + +
@@ -324,6 +340,7 @@ + diff --git a/frontend/package.json.md5 b/frontend/package.json.md5 index b7744fd..fb65cc9 100755 --- a/frontend/package.json.md5 +++ b/frontend/package.json.md5 @@ -1 +1 @@ -98d10e21267ecd49ba1108b344949fbf +0726a81c633cf310d2b07f5c7e100225 \ No newline at end of file diff --git a/frontend/scripts/test-regressions.js b/frontend/scripts/test-regressions.js index 86f9668..9e7994a 100644 --- a/frontend/scripts/test-regressions.js +++ b/frontend/scripts/test-regressions.js @@ -38,6 +38,26 @@ assertContains( "SFTPUploadChunk", "drag-drop upload must send bounded chunks", ); +assertContains( + "frontend/src/js/terminal.js", + "scheduleFit", + "terminal resize handling must be throttled through scheduled fits", +); +assertContains( + "frontend/src/js/terminal.js", + "dimensionsChanged", + "terminal resize handling must skip unchanged container dimensions", +); +assertContains( + "frontend/src/js/terminal.js", + "cancelScheduledFit", + "terminal teardown must cancel pending fit work", +); +assertNotContains( + "frontend/src/js/terminal.js", + "new ResizeObserver(() => {\n try {\n fitAddon.fit();", + "ResizeObserver must not call fitAddon.fit() directly", +); assertNotContains( "frontend/src/js/terminal.js", "Use SFTP Upload for files over 25 MB", @@ -68,5 +88,80 @@ assertContains( "TestCreateProfileRejectsUnsafeGitHubBaseURL", "GitHub HTTP/HTTPS URL validation must stay covered", ); +assertContains( + "frontend/index.html", + "mode-leviathan", + "Leviathan mode toggle must be present beside Nexus Terminal", +); +assertContains( + "frontend/src/css/sidebar.css", + "user-select: none", + "sidebar title and mode switcher text must not be selectable", +); +assertContains( + "frontend/src/css/sidebar.css", + "#mode-leviathan", + "Leviathan mode label must have its own right-alignment rule", +); +assertContains( + "frontend/src/css/sidebar.css", + "margin-left: auto;", + "Leviathan mode label must sit on the right side of the sidebar header", +); +assertContains( + "frontend/src/js/leviathan.js", + "Hex Editor", + "Leviathan mode must put Hex Editor in the tool list", +); +assertContains( + "frontend/src/js/leviathan.js", + "LeviathanSetByte", + "Leviathan hex view must support byte editing", +); +assertContains( + "frontend/src/js/leviathan.js", + "spectrogram-encode", + "Leviathan must expose an image-to-audio spectrogram encoder", +); +assertContains( + "frontend/src/js/leviathan.js", + "spectrogram-view", + "Leviathan must expose an audio spectrogram visualiser", +); +assertContains( + "frontend/src/js/leviathan.js", + "preserveAspectRatio", + "Leviathan spectrogram encoding must expose aspect-ratio preservation", +); +assertContains( + "frontend/src/js/leviathan.js", + "aspectRatio", + "Leviathan spectrogram visualiser must expose display aspect-ratio control", +); +assertContains( + "frontend/src/js/leviathan.js", + "spectrogram-insert", + "Leviathan spectrogram encoder must support inserting images into carrier audio", +); +assertContains( + "frontend/src/js/app.js", + "setMode(\"leviathan\")", + "App mode switch must route to Leviathan", +); +assertContains( + "internal/api/leviathan.go", + "LeviathanRunTool", + "Nexus API must expose the imported Leviathan engine", +); +assertNotContains( + "internal/api/leviathan.go", + "Pattern: \"*\"", + "Leviathan open dialogs must not use wildcard filters that grey out files on macOS", +); +assertContains( + "internal/leviathan/spectrogram/spectrogram.go", + "github.com/hajimehoshi/go-mp3", + "Leviathan spectrogram visualiser must support MP3 decoding", +); console.log("frontend regression checks passed"); diff --git a/frontend/src/css/leviathan.css b/frontend/src/css/leviathan.css new file mode 100644 index 0000000..e6ac328 --- /dev/null +++ b/frontend/src/css/leviathan.css @@ -0,0 +1,269 @@ +/* Leviathan mode inside the Nexus shell */ + +.leviathan-actions { + display: grid; + grid-template-columns: 1fr 1fr; + gap: var(--space-2); + padding: var(--space-2) var(--space-3); +} + +.leviathan-group { + margin-top: var(--space-2); + color: var(--text-tertiary); + cursor: default; +} + +.leviathan-tool { + display: grid; + grid-template-columns: 28px minmax(0, 1fr); + gap: var(--space-2); +} + +.leviathan-tool-ico { + color: var(--warning); + font-size: var(--text-xs); + font-weight: var(--weight-semibold); + text-align: center; +} + +.badge-leviathan { + color: var(--warning); + border-color: rgba(255, 167, 38, 0.35); +} + +.leviathan-panel { + flex-direction: column; + min-width: 0; + overflow: hidden; + background: var(--bg-base); +} + +.leviathan-panel-header { + display: flex; + align-items: flex-start; + justify-content: space-between; + gap: var(--space-4); + padding: var(--space-4); + border-bottom: 1px solid var(--border-subtle); +} + +.leviathan-title-wrap { + min-width: 0; +} + +.leviathan-title-wrap h2 { + margin: 0; + font-size: var(--text-xl); + color: var(--text-primary); +} + +.leviathan-title-wrap p { + margin: var(--space-1) 0 0; + color: var(--text-secondary); + font-size: var(--text-sm); +} + +.leviathan-eyebrow, +.leviathan-status { + color: var(--text-tertiary); + font-size: var(--text-xs); +} + +.leviathan-header-actions, +.leviathan-tool-toolbar { + display: flex; + align-items: center; + gap: var(--space-2); + flex-wrap: wrap; +} + +.leviathan-panel-body { + min-height: 0; + overflow: auto; + padding: var(--space-4); +} + +.leviathan-params { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(220px, 1fr)); + gap: var(--space-3); + margin-bottom: var(--space-3); +} + +.leviathan-param, +.leviathan-param-inline { + display: grid; + gap: var(--space-1); + color: var(--text-tertiary); + font-size: var(--text-xs); +} + +.leviathan-param textarea { + min-height: 120px; +} + +.leviathan-checkbox { + width: 18px; + height: 18px; + min-height: 18px; + padding: 0; +} + +.leviathan-file-picker { + display: flex; + align-items: center; + gap: var(--space-2); + margin-bottom: var(--space-3); + min-width: 0; +} + +.leviathan-file-path { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--text-tertiary); + font-family: var(--font-mono); + font-size: var(--text-xs); +} + +.leviathan-summary { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(160px, 1fr)); + gap: var(--space-2); + margin: var(--space-3) 0; +} + +.leviathan-pill { + border: 1px solid var(--border-subtle); + background: var(--bg-surface); + border-radius: var(--radius-md); + padding: var(--space-2); + min-width: 0; +} + +.leviathan-pill span { + display: block; + color: var(--text-tertiary); + font-size: var(--text-xs); +} + +.leviathan-pill strong { + display: block; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.leviathan-results { + min-width: 0; +} + +.leviathan-spectrogram-frame { + border: 1px solid var(--border-subtle); + background: #05070c; + border-radius: var(--radius-md); + overflow: auto; + padding: var(--space-2); +} + +.leviathan-spectrogram-image { + display: block; + max-width: 100%; + height: auto; + image-rendering: pixelated; +} + +.leviathan-empty, +.leviathan-json { + border: 1px solid var(--border-subtle); + background: var(--bg-surface); + border-radius: var(--radius-md); + color: var(--text-secondary); + padding: var(--space-4); +} + +.leviathan-json { + white-space: pre-wrap; + font-family: var(--font-mono); +} + +.leviathan-table { + width: 100%; + border-collapse: collapse; + font-size: var(--text-xs); +} + +.leviathan-table th, +.leviathan-table td { + border-bottom: 1px solid var(--border-subtle); + padding: var(--space-2); + text-align: left; + vertical-align: top; +} + +.leviathan-table th { + color: var(--text-tertiary); + font-weight: var(--weight-semibold); +} + +.leviathan-tree-node { + border-bottom: 1px solid var(--border-subtle); + padding: var(--space-2) 0; + font-family: var(--font-mono); + font-size: var(--text-xs); +} + +.leviathan-tree-node span { + color: var(--text-tertiary); +} + +.leviathan-hex-grid { + display: grid; + gap: 2px; + margin-top: var(--space-3); + font-family: var(--font-mono); + font-size: var(--text-xs); +} + +.leviathan-hex-row { + display: grid; + grid-template-columns: 92px minmax(320px, 1fr) minmax(120px, 220px); + align-items: center; + gap: var(--space-3); +} + +.leviathan-offset { + color: var(--warning); +} + +.leviathan-byte-cells { + display: grid; + grid-template-columns: repeat(16, 28px); + gap: 3px; +} + +.leviathan-byte { + width: 28px; + min-height: 24px; + padding: 0; + text-align: center; + font-family: var(--font-mono); + font-size: var(--text-xs); +} + +.leviathan-ascii { + color: var(--text-tertiary); + white-space: pre; +} + +@media (max-width: 900px) { + .leviathan-panel-header { + flex-direction: column; + } + + .leviathan-hex-row { + grid-template-columns: 1fr; + gap: var(--space-1); + } +} diff --git a/frontend/src/css/sidebar.css b/frontend/src/css/sidebar.css index afe5c0b..24d9563 100644 --- a/frontend/src/css/sidebar.css +++ b/frontend/src/css/sidebar.css @@ -19,6 +19,8 @@ display: flex; align-items: center; gap: var(--space-2); + width: 100%; + min-width: 0; } .sidebar-logo svg { @@ -35,6 +37,51 @@ letter-spacing: 0.02em; } +.sidebar-product-switch { + display: inline-flex; + align-items: center; + gap: var(--space-2); + flex: 1; + min-width: 0; + -webkit-user-select: none; + user-select: none; +} + +.sidebar-mode-toggle { + appearance: none; + border: 0; + background: transparent; + padding: 0; + height: auto; + color: var(--text-tertiary); + cursor: pointer; + white-space: nowrap; + -webkit-user-select: none; + user-select: none; +} + +#mode-leviathan { + margin-left: auto; + text-align: right; +} + +.sidebar-mode-toggle.is-active { + color: var(--text-primary); +} + +.sidebar-mode-toggle:hover { + color: var(--text-secondary); +} + +.sidebar-mode-toggle.is-active:hover { + color: var(--text-primary); +} + +.mode-leviathan .sidebar-search, +.mode-leviathan .sidebar-footer { + display: none; +} + /* ----------------------------------------------------------------------- Sidebar search ----------------------------------------------------------------------- */ diff --git a/frontend/src/js/app.js b/frontend/src/js/app.js index c1aabd0..fca03a2 100644 --- a/frontend/src/js/app.js +++ b/frontend/src/js/app.js @@ -22,6 +22,7 @@ SnippetLibrary, SSHKeyManager, TriggerEngine, + LeviathanWorkbench, stateLabel, } = window.NexusTerminal; @@ -44,7 +45,9 @@ this.snippets = new SnippetLibrary(this); this.sshKeys = new SSHKeyManager(this); this.triggers = new TriggerEngine(this); + this.leviathan = new LeviathanWorkbench(this); this.updateInfo = null; + this.mode = "nexus"; } /** @@ -542,6 +545,8 @@ this._bindClick("btn-recordings", () => this.recordings.openBrowser()); this._bindClick("btn-new-tab", () => this.profileEditor.openNew()); this._bindClick("btn-broadcast", () => this.toggleBroadcast()); + this._bindClick("mode-nexus", () => this.setMode("nexus")); + this._bindClick("mode-leviathan", () => this.setMode("leviathan")); // Wire the "Local Machine" sidebar button this.sidebar.onConnectLocal = () => this.connectLocal(); @@ -589,6 +594,33 @@ }); } + // ── App mode switching ───────────────────────────────────────── + + /** + * Switches the sidebar between Nexus profiles and Leviathan tools. + * @param {"nexus"|"leviathan"} mode + */ + setMode(mode) { + this.mode = mode === "leviathan" ? "leviathan" : "nexus"; + const root = document.getElementById("app"); + if (root) { + root.classList.toggle("mode-leviathan", this.mode === "leviathan"); + } + document + .getElementById("mode-nexus") + ?.classList.toggle("is-active", this.mode === "nexus"); + document + .getElementById("mode-leviathan") + ?.classList.toggle("is-active", this.mode === "leviathan"); + + if (this.mode === "leviathan") { + this.leviathan.renderSidebar(); + this.leviathan.openTool("hex", { reuseExisting: true }); + } else { + this.sidebar.render(); + } + } + // ── Split pane ───────────────────────────────────────────────── /** diff --git a/frontend/src/js/leviathan.js b/frontend/src/js/leviathan.js new file mode 100644 index 0000000..7932581 --- /dev/null +++ b/frontend/src/js/leviathan.js @@ -0,0 +1,681 @@ +(function () { + const { createElement, escapeHtml, formatBytes, generateId } = + window.NexusTerminal; + + const tools = [ + { + group: "Editor", + id: "hex", + name: "Hex Editor", + ico: "HX", + desc: "Editable hex view over the loaded working copy.", + }, + { + group: "Triage", + id: "overview", + name: "Overview", + ico: "OV", + desc: "First-pass summary across hashes, entropy, strings, IOCs and carve candidates.", + }, + { group: "Triage", id: "hashes", name: "Hashes", ico: "#", desc: "CRC32, MD5, SHA-1, SHA-256 and SHA-512." }, + { group: "Triage", id: "entropy", name: "Entropy", ico: "EN", desc: "Windowed entropy for packed or encrypted regions." }, + { group: "Triage", id: "strings", name: "Strings", ico: "ST", desc: "ASCII and UTF-16 string extraction." }, + { group: "Triage", id: "iocs", name: "IOCs", ico: "IO", desc: "URLs, domains, paths, hashes and other indicators." }, + { group: "Find", id: "search", name: "Search", ico: "SR", desc: "Single text, hex or regular-expression search." }, + { group: "Find", id: "multi-search", name: "Multi-search", ico: "MS", desc: "Batch text, hex and regex searches." }, + { group: "Structure", id: "parse", name: "Structures", ico: "PX", desc: "Parsers for PE, ELF, Mach-O, ZIP, images, RIFF, ASN.1 and certs." }, + { group: "Structure", id: "disasm", name: "Disasm", ico: "DA", desc: "Heuristic disassembly for common CPU families." }, + { group: "Structure", id: "pdf", name: "PDF", ico: "PDF", desc: "PDF object, stream, xref and suspicious-surface inspection." }, + { group: "Structure", id: "squashfs", name: "SquashFS", ico: "SQ", desc: "SquashFS superblock and directory metadata." }, + { group: "Audio", id: "spectrogram-encode", name: "Spectrogram Encode", ico: "SE", desc: "Convert an image into WAV audio, or hide an image inside a carrier WAV/MP3." }, + { group: "Audio", id: "spectrogram-view", name: "Spectrogram View", ico: "SV", desc: "Render WAV or MP3 audio as a spectrogram image." }, + { group: "Compare", id: "diff", name: "Diff", ico: "DF", desc: "Changed byte regions against a compare file." }, + { group: "Recover", id: "carve", name: "Carve", ico: "CV", desc: "Embedded file carving candidates." }, + { group: "Recover", id: "keys", name: "Keys", ico: "KY", desc: "PEM, DER, OpenSSH and PKCS#12 artefact discovery." }, + { group: "Recover", id: "timestamps", name: "Timestamps", ico: "TS", desc: "UNIX, FILETIME, WebKit and OLE timestamp heuristics." }, + { group: "Recover", id: "stego", name: "Stego", ico: "SG", desc: "Appended payloads and embedded trailer formats." }, + { group: "Recover", id: "metadata", name: "Metadata", ico: "MD", desc: "PNG, JPEG and GIF textual metadata extraction." }, + { group: "Advanced", id: "binsnoop", name: "Binsnoop", ico: "BS", desc: "Deep signature scanning through the Binsnoop engine." }, + { group: "Advanced", id: "rules", name: "Rules", ico: "RL", desc: "YARA-like rule scans over the current document." }, + { group: "Edit", id: "patch-plan", name: "Patch Plan", ico: "PP", desc: "Validate patch scripts without modifying the working copy." }, + { group: "Edit", id: "patch-apply", name: "Patch Apply", ico: "PA", desc: "Apply a patch script to the working copy." }, + { group: "Edit", id: "transform", name: "Transforms", ico: "TR", desc: "Hex, Base64, URL, XOR, inflate and cipher pipelines." }, + { group: "Case", id: "case", name: "Case Export", ico: "CE", desc: "Build a case archive payload from the current working copy." }, + ]; + + const defaults = { + search: { mode: "text", query: "MZ", max: 1000 }, + "multi-search": { spec: "text MZ\nhex 7f454c46\nregex https?://", max: 1000 }, + parse: { kind: "auto" }, + disasm: { arch: "x86_64", start: 0, length: 4096, max: 1000 }, + entropy: { window: 4096, step: 4096 }, + strings: { min: 4, max: 5000 }, + iocs: { min: 5 }, + carve: { max: 256 }, + keys: { max: 200 }, + timestamps: { max: 1000 }, + rules: { source: 'rule mz { strings: $a = "MZ" condition: $a }', max: 1000 }, + "patch-plan": { source: "set 0x00 90 90" }, + "patch-apply": { source: "set 0x00 90 90" }, + transform: { pipeline: "hex.encode", start: 0, length: 1024 }, + binsnoop: { maxBytes: 67108864 }, + "spectrogram-view": { fftSize: 2048, height: 1024, maxWidth: 1400, aspectRatio: 1, minHz: 0, maxHz: 0, brightness: 1, contrast: 1 }, + hex: { start: 0, bytes: 2048, width: 16 }, + }; + + class LeviathanWorkbench { + constructor(app) { + this.app = app; + this.state = null; + this.lastResults = new Map(); + } + + renderSidebar() { + const el = document.getElementById("sidebar-profiles"); + if (!el) return; + el.innerHTML = ""; + + const actions = createElement("div", "leviathan-actions"); + const load = createElement("button", "btn btn-primary btn-sm", { + type: "button", + textContent: "Load", + }); + const save = createElement("button", "btn btn-secondary btn-sm", { + type: "button", + textContent: "Save", + }); + load.addEventListener("click", () => this.loadFile()); + save.addEventListener("click", () => this.saveFile()); + actions.appendChild(load); + actions.appendChild(save); + el.appendChild(actions); + + let lastGroup = ""; + for (const tool of tools) { + if (tool.group !== lastGroup) { + el.appendChild( + createElement("div", "sidebar-section-header leviathan-group", { + textContent: tool.group, + }), + ); + lastGroup = tool.group; + } + const item = createElement("div", "sidebar-item leviathan-tool", { + dataset: { toolId: tool.id }, + title: tool.desc, + }); + item.appendChild(createElement("span", "leviathan-tool-ico", { textContent: tool.ico })); + const label = createElement("span", "sidebar-item-label", { textContent: tool.name }); + item.appendChild(label); + item.addEventListener("click", () => this.openTool(tool.id)); + el.appendChild(item); + } + this._highlightActiveTool(); + } + + async loadFile() { + try { + const payload = await window.go.api.Service.LeviathanOpenFile(); + this.applyState(payload); + this.refreshOpenHexTabs(); + } catch (err) { + if (String(err || "").includes("cancel")) return; + this.app.notifications.error("Load failed: " + err); + } + } + + async saveFile() { + try { + const path = await window.go.api.Service.LeviathanSaveWorking(); + if (path) this.app.notifications.success("Saved working copy"); + const state = await window.go.api.Service.LeviathanState(); + this.applyState(state); + } catch (err) { + this.app.notifications.error("Save failed: " + err); + } + } + + openTool(toolId, options = {}) { + const tool = tools.find((t) => t.id === toolId) || tools[0]; + const existingId = options.reuseExisting + ? this._findExistingToolSession(tool.id) + : null; + const sessionId = existingId || `leviathan-${tool.id}-${generateId(8)}`; + + if (!existingId) { + const sessionInfo = { + id: sessionId, + profileId: `leviathan:${tool.id}`, + profileName: "Leviathan", + protocol: "leviathan", + host: "local", + port: 0, + state: "connected", + tabName: tool.name, + toolId: tool.id, + }; + this.app.state.addSession(sessionInfo); + this.app.tabs.addTab(sessionInfo); + const panel = document.getElementById(`panel-${sessionId}`); + if (panel) this.renderToolPanel(panel, sessionId, tool); + } else { + this.app.tabs.setActive(existingId); + } + this._highlightActiveTool(tool.id); + } + + renderToolPanel(panel, sessionId, tool) { + panel.classList.add("leviathan-panel"); + panel.innerHTML = ""; + + const header = createElement("div", "leviathan-panel-header"); + const titleWrap = createElement("div", "leviathan-title-wrap"); + titleWrap.appendChild(createElement("span", "leviathan-eyebrow", { textContent: this._stateLabel() })); + titleWrap.appendChild(createElement("h2", "", { textContent: tool.name })); + titleWrap.appendChild(createElement("p", "", { textContent: tool.desc })); + header.appendChild(titleWrap); + + const headerActions = createElement("div", "leviathan-header-actions"); + const undo = createElement("button", "btn btn-sm", { textContent: "Undo" }); + const redo = createElement("button", "btn btn-sm", { textContent: "Redo" }); + const revert = createElement("button", "btn btn-sm", { textContent: "Revert" }); + undo.addEventListener("click", () => this._stateAction("LeviathanUndo", sessionId)); + redo.addEventListener("click", () => this._stateAction("LeviathanRedo", sessionId)); + revert.addEventListener("click", () => this._stateAction("LeviathanRevert", sessionId)); + headerActions.appendChild(undo); + headerActions.appendChild(redo); + headerActions.appendChild(revert); + header.appendChild(headerActions); + panel.appendChild(header); + + const body = createElement("div", "leviathan-panel-body"); + panel.appendChild(body); + + if (tool.id === "hex") { + this.renderHexEditor(body, sessionId); + } else if (tool.id === "spectrogram-encode") { + this.renderSpectrogramEncode(body, sessionId); + } else if (tool.id === "spectrogram-view") { + this.renderSpectrogramView(body, sessionId); + } else { + this.renderToolRunner(body, sessionId, tool); + } + } + + renderHexEditor(body, sessionId) { + const toolbar = createElement("div", "leviathan-tool-toolbar"); + const start = this._input("start", "0"); + const bytes = this._input("bytes", "2048"); + const refresh = createElement("button", "btn btn-sm btn-primary", { textContent: "Refresh" }); + toolbar.append(start.row, bytes.row, refresh); + body.appendChild(toolbar); + + const status = createElement("div", "leviathan-status", { textContent: this._stateLabel() }); + const grid = createElement("div", "leviathan-hex-grid"); + body.append(status, grid); + + const load = async () => { + try { + status.textContent = "Loading hex..."; + const result = await window.go.api.Service.LeviathanRunTool("hex", { + start: Number(start.input.value || 0), + bytes: Number(bytes.input.value || 2048), + width: 16, + }); + this.lastResults.set(sessionId, result); + this._renderHexRows(grid, result.rows || [], status); + status.textContent = this._stateLabel(); + } catch (err) { + status.textContent = String(err); + } + }; + refresh.addEventListener("click", load); + load(); + } + + renderSpectrogramEncode(body, sessionId) { + const params = createElement("div", "leviathan-params"); + const fields = [ + this._paramInput(params, "durationSec", "Duration (s)", 5, { type: "number", min: "0.25", max: "60", step: "0.25" }), + this._paramInput(params, "sampleRate", "Sample rate", 44100, { type: "number", min: "8000", max: "96000", step: "1000" }), + this._paramInput(params, "minHz", "Min frequency", 200, { type: "number", min: "20", step: "10" }), + this._paramInput(params, "maxHz", "Max frequency", 20000, { type: "number", min: "120", step: "10" }), + this._paramInput(params, "bins", "Frequency bins", 160, { type: "number", min: "16", max: "384", step: "1" }), + this._paramInput(params, "columns", "Time columns", 768, { type: "number", min: "16", max: "2048", step: "1" }), + this._paramInput(params, "gain", "Output gain", 0.85, { type: "number", min: "0.05", max: "1", step: "0.05" }), + this._paramInput(params, "startSec", "Insert start (s)", 0, { type: "number", min: "0", step: "0.1" }), + this._paramInput(params, "mixLevel", "Insert mix", 0.35, { type: "number", min: "0.01", max: "1", step: "0.01" }), + this._paramInput(params, "preserveAspectRatio", "Preserve aspect", true, { type: "checkbox" }), + this._paramInput(params, "invert", "Invert image", true, { type: "checkbox" }), + ]; + body.appendChild(params); + + const imagePicker = createElement("div", "leviathan-file-picker"); + const chooseImage = createElement("button", "btn btn-sm", { type: "button", textContent: "Select Image" }); + const imagePath = createElement("span", "leviathan-file-path", { textContent: "No insertion image selected" }); + imagePicker.append(chooseImage, imagePath); + body.appendChild(imagePicker); + + let selectedImagePath = ""; + chooseImage.addEventListener("click", async () => { + try { + const path = await window.go.api.Service.SelectFile("Select Image to Insert", ""); + if (!path) return; + selectedImagePath = path; + imagePath.textContent = path; + } catch (err) { + this.app.notifications.error("Image selection failed: " + err); + } + }); + + const toolbar = createElement("div", "leviathan-tool-toolbar"); + const encode = createElement("button", "btn btn-sm btn-primary", { type: "button", textContent: "Encode Loaded Image" }); + const insert = createElement("button", "btn btn-sm", { type: "button", textContent: "Insert Into Loaded Audio" }); + toolbar.append(encode, insert); + body.appendChild(toolbar); + + const summary = createElement("div", "leviathan-summary"); + const results = createElement("div", "leviathan-results"); + body.append(summary, results); + results.appendChild(createElement("div", "leviathan-empty", { textContent: "Load an image to encode, or load WAV/MP3 audio and select an image to insert." })); + + const run = async (toolId) => { + try { + results.innerHTML = ""; + results.appendChild(createElement("div", "leviathan-empty", { textContent: "Rendering audio..." })); + const payload = {}; + for (const field of fields) payload[field.key] = this._fieldValue(field); + if (toolId === "spectrogram-insert") { + if (!selectedImagePath) throw new Error("Select an image to insert first."); + payload.imagePath = selectedImagePath; + } + const out = await window.go.api.Service.LeviathanRunTool(toolId, payload); + this.lastResults.set(sessionId, out); + summary.innerHTML = ""; + results.innerHTML = ""; + this._renderSummaryPills(summary, out.summary || out); + results.appendChild(createElement("pre", "leviathan-json", { textContent: JSON.stringify(out, null, 2) })); + if (out?.path) this.app.notifications.success("Saved " + out.path); + } catch (err) { + results.innerHTML = ""; + results.appendChild(createElement("pre", "leviathan-json", { textContent: String(err) })); + } + }; + encode.addEventListener("click", () => run("spectrogram-encode")); + insert.addEventListener("click", () => run("spectrogram-insert")); + } + + renderSpectrogramView(body, sessionId) { + const params = createElement("div", "leviathan-params"); + const fields = [ + this._paramInput(params, "fftSize", "FFT size", 2048, { type: "number", min: "256", max: "8192", step: "256" }), + this._paramInput(params, "height", "Base height", 1024, { type: "number", min: "64", max: "1024", step: "16" }), + this._paramInput(params, "maxWidth", "Max width", 1400, { type: "number", min: "64", max: "2400", step: "32" }), + this._paramInput(params, "aspectRatio", "Aspect ratio", 1, { type: "number", min: "0.25", max: "4", step: "0.25" }), + this._paramInput(params, "minHz", "Min frequency", 0, { type: "number", min: "0", step: "10" }), + this._paramInput(params, "maxHz", "Max frequency", 0, { type: "number", min: "0", step: "10" }), + this._paramInput(params, "brightness", "Brightness", 1, { type: "number", min: "0.1", max: "4", step: "0.1" }), + this._paramInput(params, "contrast", "Contrast", 1, { type: "number", min: "0.1", max: "4", step: "0.1" }), + ]; + body.appendChild(params); + + const toolbar = createElement("div", "leviathan-tool-toolbar"); + const run = createElement("button", "btn btn-sm btn-primary", { type: "button", textContent: "Render Spectrogram" }); + const exportBtn = createElement("button", "btn btn-sm", { type: "button", textContent: "Export JSON" }); + toolbar.append(run, exportBtn); + body.appendChild(toolbar); + + const summary = createElement("div", "leviathan-summary"); + const results = createElement("div", "leviathan-results"); + body.append(summary, results); + results.appendChild(createElement("div", "leviathan-empty", { textContent: "Load a WAV or MP3 file, then render its spectrogram." })); + + run.addEventListener("click", async () => { + try { + results.innerHTML = ""; + results.appendChild(createElement("div", "leviathan-empty", { textContent: "Rendering spectrogram..." })); + const payload = {}; + for (const field of fields) payload[field.key] = this._fieldValue(field); + const out = await window.go.api.Service.LeviathanRunTool("spectrogram-view", payload); + this.lastResults.set(sessionId, out); + summary.innerHTML = ""; + results.innerHTML = ""; + this._renderSummaryPills(summary, out.summary || out); + if (out?.pngBase64) { + const frame = createElement("div", "leviathan-spectrogram-frame"); + const img = createElement("img", "leviathan-spectrogram-image", { + alt: "Rendered audio spectrogram", + }); + img.src = `data:${out.mime || "image/png"};base64,${out.pngBase64}`; + frame.appendChild(img); + results.appendChild(frame); + } else { + results.appendChild(createElement("pre", "leviathan-json", { textContent: JSON.stringify(out, null, 2) })); + } + } catch (err) { + results.innerHTML = ""; + results.appendChild(createElement("pre", "leviathan-json", { textContent: String(err) })); + } + }); + exportBtn.addEventListener("click", async () => { + const out = this.lastResults.get(sessionId); + if (!out) return; + try { + await window.go.api.Service.LeviathanExportJSON("spectrogram-view.json", out); + this.app.notifications.success("Exported JSON"); + } catch (err) { + this.app.notifications.error("Export failed: " + err); + } + }); + } + + renderToolRunner(body, sessionId, tool) { + const toolbar = createElement("div", "leviathan-tool-toolbar"); + const params = createElement("div", "leviathan-params"); + const fields = this._renderParams(tool.id, params); + const run = createElement("button", "btn btn-sm btn-primary", { textContent: "Run" }); + const exportBtn = createElement("button", "btn btn-sm", { textContent: "Export JSON" }); + toolbar.append(run, exportBtn); + body.append(params, toolbar); + + const summary = createElement("div", "leviathan-summary"); + const results = createElement("div", "leviathan-results"); + body.append(summary, results); + + run.addEventListener("click", async () => { + try { + if (tool.id === "diff" && !this.state?.compare) { + const compare = await window.go.api.Service.LeviathanOpenCompare(); + this.applyState(compare); + } + results.innerHTML = ""; + results.appendChild(createElement("div", "leviathan-empty", { textContent: "Running..." })); + const payload = {}; + for (const field of fields) { + payload[field.key] = this._fieldValue(field); + } + const out = await window.go.api.Service.LeviathanRunTool(tool.id, payload); + this.lastResults.set(sessionId, out); + this._renderResult(summary, results, out); + if (out && out.state) this.applyState(out.state); + } catch (err) { + results.innerHTML = ""; + results.appendChild(createElement("pre", "leviathan-json", { textContent: String(err) })); + } + }); + exportBtn.addEventListener("click", async () => { + const out = this.lastResults.get(sessionId); + if (!out) return; + try { + await window.go.api.Service.LeviathanExportJSON(`${tool.id}.json`, out); + this.app.notifications.success("Exported JSON"); + } catch (err) { + this.app.notifications.error("Export failed: " + err); + } + }); + results.appendChild( + createElement("div", "leviathan-empty", { + textContent: "Run the tool to populate this tab.", + }), + ); + } + + applyState(payload) { + this.state = payload?.state || payload || null; + this._updateOpenHeaders(); + } + + destroyPanel(sessionId) { + this.lastResults.delete(sessionId); + } + + refreshOpenHexTabs() { + for (const session of this.app.state.getAllSessions()) { + if (session.protocol !== "leviathan" || session.toolId !== "hex") continue; + const panel = document.getElementById(`panel-${session.id}`); + if (panel) this.renderToolPanel(panel, session.id, tools[0]); + } + } + + _renderHexRows(grid, rows, status) { + grid.innerHTML = ""; + if (!rows.length) { + grid.appendChild(createElement("div", "leviathan-empty", { textContent: "Load a file to inspect bytes." })); + return; + } + for (const row of rows) { + const line = createElement("div", "leviathan-hex-row"); + line.appendChild(createElement("span", "leviathan-offset", { textContent: this._hexOffset(row.offset) })); + const bytes = createElement("span", "leviathan-byte-cells"); + const cells = row.bytes || String(row.hex || "").split(/\s+/).filter(Boolean); + cells.forEach((value, idx) => { + const cell = createElement("input", "leviathan-byte", { + value, + maxlength: "2", + "aria-label": `Byte ${this._hexOffset((row.offset || 0) + idx)}`, + }); + cell.addEventListener("change", async () => { + try { + await window.go.api.Service.LeviathanSetByte((row.offset || 0) + idx, cell.value); + cell.value = cell.value.trim().padStart(2, "0").slice(-2).toLowerCase(); + status.textContent = "Edited " + this._hexOffset((row.offset || 0) + idx); + } catch (err) { + status.textContent = String(err); + cell.value = value; + } + }); + bytes.appendChild(cell); + }); + line.appendChild(bytes); + line.appendChild(createElement("span", "leviathan-ascii", { textContent: row.ascii || "" })); + grid.appendChild(line); + } + } + + _renderParams(toolId, container) { + container.innerHTML = ""; + const cfg = defaults[toolId] || {}; + return Object.entries(cfg).map(([key, value]) => { + const row = createElement("label", "leviathan-param"); + row.appendChild(createElement("span", "", { textContent: key })); + let input; + if (key === "source" || key === "spec" || key === "pipeline") { + input = createElement("textarea", "form-input"); + } else if (key === "mode") { + input = createElement("select", "form-input"); + ["text", "hex", "regex"].forEach((v) => input.add(new Option(v, v))); + } else if (key === "kind") { + input = createElement("select", "form-input"); + ["auto", "png", "jpeg", "gif", "bmp", "zip", "elf", "pe", "macho", "riff", "asn1", "cert"].forEach((v) => input.add(new Option(v, v))); + } else if (key === "arch") { + input = createElement("select", "form-input"); + ["x86_64", "x86", "aarch64", "arm", "thumb", "mips", "mipsle"].forEach((v) => input.add(new Option(v, v))); + } else { + input = createElement("input", "form-input"); + } + input.value = value; + row.appendChild(input); + container.appendChild(row); + return { key, input }; + }); + } + + _renderResult(summary, results, out) { + summary.innerHTML = ""; + results.innerHTML = ""; + this._renderSummaryPills(summary, this._flatSummary(out)); + if (out?.tree) { + this._renderTree(results, out.tree); + return; + } + const arr = this._firstArray(out); + if (arr) { + this._renderTable(results, arr); + return; + } + results.appendChild(createElement("pre", "leviathan-json", { textContent: JSON.stringify(out, null, 2) })); + } + + _renderTable(container, rows) { + rows = rows.slice(0, 1000); + if (!rows.length) { + container.appendChild(createElement("div", "leviathan-empty", { textContent: "No rows." })); + return; + } + const preferred = ["offset", "length", "kind", "name", "category", "value", "sample", "summary", "iso", "type"]; + const all = Array.from(new Set(rows.flatMap((row) => Object.keys(row || {})))); + const cols = [...preferred.filter((c) => all.includes(c)), ...all.filter((c) => !preferred.includes(c))].slice(0, 8); + const table = createElement("table", "leviathan-table"); + const thead = createElement("thead"); + const headRow = createElement("tr"); + cols.forEach((c) => headRow.appendChild(createElement("th", "", { textContent: c }))); + thead.appendChild(headRow); + const tbody = createElement("tbody"); + rows.forEach((row) => { + const tr = createElement("tr"); + cols.forEach((c) => tr.appendChild(createElement("td", "", { textContent: this._formatCell(row[c]) }))); + tbody.appendChild(tr); + }); + table.append(thead, tbody); + container.appendChild(table); + } + + _renderTree(container, root) { + const wrap = createElement("div", "leviathan-tree"); + const walk = (node, depth = 0) => { + const row = createElement("div", "leviathan-tree-node"); + row.style.paddingLeft = `${depth * 16}px`; + row.appendChild(createElement("strong", "", { textContent: node.name || "" })); + row.appendChild(createElement("span", "", { textContent: ` ${node.type || ""} ${node.offset != null ? this._hexOffset(node.offset) : ""} +${node.length || 0}` })); + if (node.value) row.appendChild(createElement("div", "", { textContent: node.value })); + wrap.appendChild(row); + (node.children || []).forEach((child) => walk(child, depth + 1)); + }; + walk(root); + container.appendChild(wrap); + } + + _stateAction(method, sessionId) { + return window.go.api.Service[method]() + .then((state) => { + this.applyState(state); + const session = this.app.state.getSession(sessionId); + if (session?.toolId === "hex") { + const panel = document.getElementById(`panel-${sessionId}`); + if (panel) this.renderToolPanel(panel, sessionId, tools[0]); + } + }) + .catch((err) => this.app.notifications.error(String(err))); + } + + _input(label, value) { + const row = createElement("label", "leviathan-param-inline"); + row.appendChild(createElement("span", "", { textContent: label })); + const input = createElement("input", "form-input", { value }); + row.appendChild(input); + return { row, input }; + } + + _paramInput(container, key, label, value, attrs = {}) { + const row = createElement("label", "leviathan-param"); + row.appendChild(createElement("span", "", { textContent: label })); + const input = createElement("input", "form-input"); + for (const [attr, attrValue] of Object.entries(attrs)) { + input.setAttribute(attr, attrValue); + } + if (attrs.type === "checkbox") { + input.checked = Boolean(value); + input.classList.add("leviathan-checkbox"); + } else { + input.value = value; + } + row.appendChild(input); + container.appendChild(row); + return { key, input }; + } + + _fieldValue(field) { + if (field.input.type === "checkbox") { + return field.input.checked; + } + const raw = field.input.value; + const numeric = Number(raw); + const stringKeys = ["query", "source", "spec", "pipeline", "kind", "arch", "mode"]; + return raw !== "" && Number.isFinite(numeric) && !stringKeys.includes(field.key) + ? numeric + : raw; + } + + _renderSummaryPills(container, values) { + Object.entries(values || {}) + .slice(0, 10) + .forEach(([key, value]) => { + const item = createElement("div", "leviathan-pill"); + item.appendChild(createElement("span", "", { textContent: key })); + item.appendChild(createElement("strong", "", { textContent: this._formatCell(value) })); + container.appendChild(item); + }); + } + + _findExistingToolSession(toolId) { + const session = this.app.state + .getAllSessions() + .find((s) => s.protocol === "leviathan" && s.toolId === toolId); + return session?.id || null; + } + + _updateOpenHeaders() { + document.querySelectorAll(".leviathan-eyebrow").forEach((el) => { + el.textContent = this._stateLabel(); + }); + } + + _highlightActiveTool(toolId) { + document.querySelectorAll(".leviathan-tool").forEach((el) => { + el.classList.toggle("sidebar-item-active", el.dataset.toolId === toolId); + }); + } + + _stateLabel() { + if (!this.state?.name) return "No file loaded"; + const dirty = this.state.dirty ? "modified" : "clean"; + return `${this.state.name} · ${formatBytes(this.state.size || 0)} · ${dirty}`; + } + + _flatSummary(out) { + if (!out || typeof out !== "object") return {}; + const summary = {}; + for (const [key, value] of Object.entries(out)) { + if (Array.isArray(value)) summary[key] = value.length; + else if (value && typeof value === "object") { + if (key === "hashes") summary.hash = value.sha256 || value.md5 || "ready"; + else summary[key] = Object.keys(value).length + " keys"; + } else { + summary[key] = value; + } + } + return summary; + } + + _firstArray(out) { + if (!out || typeof out !== "object") return null; + for (const key of ["items", "hits", "all", "regions", "blocks", "entries", "metadata"]) { + if (Array.isArray(out[key])) return out[key]; + } + if (out.results?.Results && Array.isArray(out.results.Results)) return out.results.Results; + return null; + } + + _formatCell(value) { + return typeof value === "object" ? JSON.stringify(value) : String(value ?? ""); + } + + _hexOffset(value) { + return "0x" + Number(value || 0).toString(16).padStart(8, "0"); + } + } + + window.NexusTerminal = window.NexusTerminal || {}; + window.NexusTerminal.LeviathanWorkbench = LeviathanWorkbench; + window.NexusTerminal.LeviathanTools = tools; +})(); diff --git a/frontend/src/js/sidebar.js b/frontend/src/js/sidebar.js index cf76ab3..44076fb 100644 --- a/frontend/src/js/sidebar.js +++ b/frontend/src/js/sidebar.js @@ -59,6 +59,10 @@ */ render() { if (!this.el) return; + if (this.app.mode === "leviathan") { + this.app.leviathan.renderSidebar(); + return; + } this.el.innerHTML = ""; const profiles = this._filteredProfiles(); diff --git a/frontend/src/js/tabs.js b/frontend/src/js/tabs.js index 5feac49..e27bb3f 100644 --- a/frontend/src/js/tabs.js +++ b/frontend/src/js/tabs.js @@ -177,6 +177,7 @@ this.app.terminals.destroyTerminal(sessionId); this.app.rawViewer.destroyViewer(sessionId); window.NexusTerminal.GitHubViewer.destroyViewer(sessionId); + this.app.leviathan?.destroyPanel(sessionId); this.removeTab(sessionId); this.app.state.removeSession(sessionId); } diff --git a/frontend/src/js/terminal.js b/frontend/src/js/terminal.js index 3be3689..2cad1eb 100644 --- a/frontend/src/js/terminal.js +++ b/frontend/src/js/terminal.js @@ -182,21 +182,70 @@ this._handleFileDrop(sessionId, term, files); }); + let fitRaf = 0; + let forceNextFit = false; + let lastFitWidth = 0; + let lastFitHeight = 0; + + const cancelScheduledFit = () => { + if (!fitRaf) return; + cancelAnimationFrame(fitRaf); + fitRaf = 0; + }; + + const scheduleFit = (force = false) => { + forceNextFit = forceNextFit || force; + if (fitRaf) return; + + fitRaf = requestAnimationFrame(() => { + fitRaf = 0; + + if (!container.isConnected) { + forceNextFit = false; + return; + } + + const rect = container.getBoundingClientRect(); + const width = Math.round(rect.width); + const height = Math.round(rect.height); + if (width <= 0 || height <= 0) { + return; + } + + const dimensionsChanged = + width !== lastFitWidth || height !== lastFitHeight; + if (!forceNextFit && !dimensionsChanged) { + return; + } + + lastFitWidth = width; + lastFitHeight = height; + forceNextFit = false; + + try { + fitAddon.fit(); + } catch (_) { + /* noop */ + } + }); + }; + // Force a synchronous layout reflow so the container reports its // true dimensions, then fit the terminal to fill the space. // eslint-disable-next-line no-unused-expressions container.offsetHeight; - const doFit = () => { + scheduleFit(true); + // Belt-and-braces: retry once after a paint in case the initial + // reflow dimensions were still stale. + requestAnimationFrame(() => scheduleFit(true)); + + const refreshTerminal = () => { try { - fitAddon.fit(); + term.refresh(0, term.rows - 1); } catch (_) { /* noop */ } }; - doFit(); - // Belt-and-braces: retry once after a paint in case the initial - // reflow dimensions were still stale. - requestAnimationFrame(doFit); // Forward user keystrokes to the backend term.onData((data) => { @@ -288,13 +337,11 @@ } }); - // Re-fit when the container itself resizes + // Re-fit when the container itself resizes. ResizeObserver can fire + // repeatedly while xterm mutates its DOM, so schedule at most one fit + // per frame and skip work when the container dimensions are unchanged. const observer = new ResizeObserver(() => { - try { - fitAddon.fit(); - } catch (_) { - // Ignore transient errors during layout shifts - } + scheduleFit(); }); observer.observe(container); @@ -304,6 +351,9 @@ searchAddon, webLinksAddon, observer, + scheduleFit, + cancelScheduledFit, + refreshTerminal, decoder: typeof TextDecoder !== "undefined" ? new TextDecoder("utf-8") : null, }); @@ -346,6 +396,7 @@ if (!entry) return; entry.observer.disconnect(); + entry.cancelScheduledFit(); entry.term.dispose(); this.terminals.delete(sessionId); } @@ -365,13 +416,7 @@ // Re-fit the newly visible terminal const entry = this.terminals.get(sessionId); if (entry) { - requestAnimationFrame(() => { - try { - entry.fitAddon.fit(); - } catch (_) { - /* noop */ - } - }); + entry.scheduleFit(true); } } @@ -390,13 +435,7 @@ fitSession(sessionId) { const entry = this.terminals.get(sessionId); if (entry) { - requestAnimationFrame(() => { - try { - entry.fitAddon.fit(); - } catch (_) { - /* noop */ - } - }); + entry.scheduleFit(true); } } diff --git a/frontend/wailsjs/go/api/Service.d.ts b/frontend/wailsjs/go/api/Service.d.ts index 7067e3b..ff71dce 100755 --- a/frontend/wailsjs/go/api/Service.d.ts +++ b/frontend/wailsjs/go/api/Service.d.ts @@ -10,6 +10,8 @@ export function CancelReconnect(arg1:string):Promise; export function CheckForUpdates():Promise>; +export function CleanupRecordings(arg1:number,arg2:number):Promise>; + export function ClearSessionHistory(arg1:string):Promise; export function CloseAllSessions():Promise; @@ -64,6 +66,8 @@ export function GetProfile(arg1:string):Promise; export function GetRecording(arg1:string):Promise>; +export function GetRecordingStorageInfo():Promise>; + export function GetSSHPublicKey(arg1:string):Promise; export function GetSessionCwd(arg1:string):Promise; @@ -90,6 +94,28 @@ export function IsSecretStoreInitialised():Promise; export function IsSecretStoreUnlocked():Promise; +export function LeviathanExportJSON(arg1:string,arg2:Record):Promise; + +export function LeviathanLoadFile(arg1:string,arg2:boolean):Promise>; + +export function LeviathanOpenCompare():Promise>; + +export function LeviathanOpenFile():Promise>; + +export function LeviathanRedo():Promise; + +export function LeviathanRevert():Promise; + +export function LeviathanRunTool(arg1:string,arg2:Record):Promise>; + +export function LeviathanSaveWorking():Promise; + +export function LeviathanSetByte(arg1:number,arg2:string):Promise>; + +export function LeviathanState():Promise; + +export function LeviathanUndo():Promise; + export function ListAuditEntries(arg1:number):Promise>; export function ListFavourites():Promise>; @@ -150,8 +176,16 @@ export function SFTPStat(arg1:string,arg2:string):Promise>; export function SFTPUpload(arg1:string,arg2:string):Promise; +export function SFTPUploadCancel(arg1:string):Promise; + +export function SFTPUploadChunk(arg1:string,arg2:string):Promise; + export function SFTPUploadData(arg1:string,arg2:string,arg3:string,arg4:string):Promise; +export function SFTPUploadFinish(arg1:string):Promise; + +export function SFTPUploadStart(arg1:string,arg2:string,arg3:string,arg4:number):Promise; + export function SaveFile(arg1:string,arg2:string):Promise; export function SavePreferences(arg1:models.AppPreferences):Promise; diff --git a/frontend/wailsjs/go/api/Service.js b/frontend/wailsjs/go/api/Service.js index 6b150ac..e09b58e 100755 --- a/frontend/wailsjs/go/api/Service.js +++ b/frontend/wailsjs/go/api/Service.js @@ -14,6 +14,10 @@ export function CheckForUpdates() { return window['go']['api']['Service']['CheckForUpdates'](); } +export function CleanupRecordings(arg1, arg2) { + return window['go']['api']['Service']['CleanupRecordings'](arg1, arg2); +} + export function ClearSessionHistory(arg1) { return window['go']['api']['Service']['ClearSessionHistory'](arg1); } @@ -122,6 +126,10 @@ export function GetRecording(arg1) { return window['go']['api']['Service']['GetRecording'](arg1); } +export function GetRecordingStorageInfo() { + return window['go']['api']['Service']['GetRecordingStorageInfo'](); +} + export function GetSSHPublicKey(arg1) { return window['go']['api']['Service']['GetSSHPublicKey'](arg1); } @@ -174,6 +182,50 @@ export function IsSecretStoreUnlocked() { return window['go']['api']['Service']['IsSecretStoreUnlocked'](); } +export function LeviathanExportJSON(arg1, arg2) { + return window['go']['api']['Service']['LeviathanExportJSON'](arg1, arg2); +} + +export function LeviathanLoadFile(arg1, arg2) { + return window['go']['api']['Service']['LeviathanLoadFile'](arg1, arg2); +} + +export function LeviathanOpenCompare() { + return window['go']['api']['Service']['LeviathanOpenCompare'](); +} + +export function LeviathanOpenFile() { + return window['go']['api']['Service']['LeviathanOpenFile'](); +} + +export function LeviathanRedo() { + return window['go']['api']['Service']['LeviathanRedo'](); +} + +export function LeviathanRevert() { + return window['go']['api']['Service']['LeviathanRevert'](); +} + +export function LeviathanRunTool(arg1, arg2) { + return window['go']['api']['Service']['LeviathanRunTool'](arg1, arg2); +} + +export function LeviathanSaveWorking() { + return window['go']['api']['Service']['LeviathanSaveWorking'](); +} + +export function LeviathanSetByte(arg1, arg2) { + return window['go']['api']['Service']['LeviathanSetByte'](arg1, arg2); +} + +export function LeviathanState() { + return window['go']['api']['Service']['LeviathanState'](); +} + +export function LeviathanUndo() { + return window['go']['api']['Service']['LeviathanUndo'](); +} + export function ListAuditEntries(arg1) { return window['go']['api']['Service']['ListAuditEntries'](arg1); } @@ -294,10 +346,26 @@ export function SFTPUpload(arg1, arg2) { return window['go']['api']['Service']['SFTPUpload'](arg1, arg2); } +export function SFTPUploadCancel(arg1) { + return window['go']['api']['Service']['SFTPUploadCancel'](arg1); +} + +export function SFTPUploadChunk(arg1, arg2) { + return window['go']['api']['Service']['SFTPUploadChunk'](arg1, arg2); +} + export function SFTPUploadData(arg1, arg2, arg3, arg4) { return window['go']['api']['Service']['SFTPUploadData'](arg1, arg2, arg3, arg4); } +export function SFTPUploadFinish(arg1) { + return window['go']['api']['Service']['SFTPUploadFinish'](arg1); +} + +export function SFTPUploadStart(arg1, arg2, arg3, arg4) { + return window['go']['api']['Service']['SFTPUploadStart'](arg1, arg2, arg3, arg4); +} + export function SaveFile(arg1, arg2) { return window['go']['api']['Service']['SaveFile'](arg1, arg2); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index 66d2e19..d0008ae 100755 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -20,6 +20,48 @@ export namespace api { this.privateKeyPath = source["privateKeyPath"]; } } + export class leviathanDocState { + name: string; + size: number; + dirty: boolean; + undoLen: number; + redoLen: number; + compare?: leviathanDocState; + previewHex: any[]; + + static createFrom(source: any = {}) { + return new leviathanDocState(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.name = source["name"]; + this.size = source["size"]; + this.dirty = source["dirty"]; + this.undoLen = source["undoLen"]; + this.redoLen = source["redoLen"]; + this.compare = this.convertValues(source["compare"], leviathanDocState); + this.previewHex = source["previewHex"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } } diff --git a/go.mod b/go.mod index dba5106..7841ce1 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,12 @@ require ( github.com/creack/pty v1.1.24 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 + github.com/hajimehoshi/go-mp3 v0.3.4 + github.com/klauspost/compress v1.17.11 github.com/mattn/go-sqlite3 v1.14.44 github.com/pkg/sftp v1.13.10 github.com/pquerna/otp v1.5.0 + github.com/ulikunitz/xz v0.5.15 github.com/wailsapp/wails/v2 v2.12.0 github.com/zalando/go-keyring v0.2.8 golang.org/x/crypto v0.52.0 diff --git a/go.sum b/go.sum index a32e4b2..f1cd2dc 100644 --- a/go.sum +++ b/go.sum @@ -20,8 +20,13 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hajimehoshi/go-mp3 v0.3.4 h1:NUP7pBYH8OguP4diaTZ9wJbUbk3tC0KlfzsEpWmYj68= +github.com/hajimehoshi/go-mp3 v0.3.4/go.mod h1:fRtZraRFcWb0pu7ok0LqyFhCUrPeMsGRSVop0eemFmo= +github.com/hajimehoshi/oto/v2 v2.3.1/go.mod h1:seWLbgHH7AyUMYKfKYT9pg7PhUu9/SisyJvNTT+ASQo= github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1 h1:njuLRcjAuMKr7kI3D85AXWkw6/+v9PwtV6M6o11sWHQ= github.com/jchv/go-winloader v0.0.0-20250406163304-c1995be93bd1/go.mod h1:alcuEEnZsY1WQsagKhZDsoPCRoOijYqhZvPwLG0kzVs= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/labstack/echo/v4 v4.15.2 h1:nnh2sCzGCVYnU+wCisMPiYapEg/QVo/gcI9ePKg5/T4= @@ -70,6 +75,8 @@ github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tkrajina/go-reflector v0.5.8 h1:yPADHrwmUbMq4RGEyaOUpz2H90sRsETNVpjzo3DLVQQ= github.com/tkrajina/go-reflector v0.5.8/go.mod h1:ECbqLgccecY5kPmPmXg1MrHW585yMcDkVl6IvJe64T4= +github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY= +github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= @@ -85,13 +92,12 @@ github.com/zalando/go-keyring v0.2.8/go.mod h1:tsMo+VpRq5NGyKfxoBVjCuMrG47yj8cma golang.org/x/crypto v0.52.0 h1:RMs7fP2rXdep0CftQlK8Uf+kibLm7qkCcradZWYz988= golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGbc= golang.org/x/net v0.0.0-20210505024714-0287a6fb4125/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220712014510-0a85c31ab51e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= diff --git a/internal/api/api.go b/internal/api/api.go index 242bf77..978060d 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -48,6 +48,7 @@ const ( maxInlineSFTPUploadBytes = 25 * 1024 * 1024 maxSFTPUploadChunkBytes = 1024 * 1024 maxStreamingSFTPUploadSize = int64(2 * 1024 * 1024 * 1024) + maxSSHPublicKeyBytes = 64 * 1024 ) // Service is the main API surface exposed to the frontend via Wails bindings. @@ -66,6 +67,8 @@ type Service struct { uploadMu sync.Mutex uploads map[string]*sftpUpload + + leviathan *leviathanState } type sftpUpload struct { @@ -93,6 +96,7 @@ func NewService( auditLog: auditLog, prefs: prefs, uploads: make(map[string]*sftpUpload), + leviathan: &leviathanState{}, } } @@ -1763,7 +1767,7 @@ func (s *Service) ListSSHKeyPairs() ([]SSHKeyInfo, error) { seen[baseName] = true // Read public key content - pubBytes, err := os.ReadFile(filepath.Join(sshDir, name)) + pubBytes, err := readSSHPublicKeyFile(filepath.Join(sshDir, name)) if err != nil { continue } @@ -1849,7 +1853,7 @@ func (s *Service) GetSSHPublicKey(name string) (string, error) { if err != nil { return "", err } - data, err := os.ReadFile(pubPath) + data, err := readSSHPublicKeyFile(pubPath) if err != nil { return "", fmt.Errorf("unable to read public key: %w", err) } @@ -1891,6 +1895,20 @@ func sshKeyPairPaths(name string) (string, string, error) { return privPath, pubPath, nil } +func readSSHPublicKeyFile(path string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("public key path is a directory") + } + if info.Size() > maxSSHPublicKeyBytes { + return nil, fmt.Errorf("public key file is too large: %d bytes exceeds %d bytes", info.Size(), maxSSHPublicKeyBytes) + } + return os.ReadFile(path) +} + func validateSSHKeyName(name string) (string, error) { name = strings.TrimSpace(name) if name == "" { diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 6bf3d21..d95c4eb 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -2,11 +2,13 @@ package api import ( "encoding/base64" + "os" "path/filepath" "strings" "testing" "github.com/andydixon/nexus/internal/audit" + "github.com/andydixon/nexus/internal/leviathan/document" "github.com/andydixon/nexus/internal/models" "github.com/andydixon/nexus/internal/storage" ) @@ -169,6 +171,63 @@ func TestImportProfilesNormalizesSparseRAWProfile(t *testing.T) { } } +func TestLeviathanDocumentBytesRejectsOversizedDocument(t *testing.T) { + doc := document.New("large.bin", []byte("abcd")) + _, err := leviathanDocumentBytes(doc, 3, "test") + if err == nil { + t.Fatal("expected oversized document to be rejected") + } + if !strings.Contains(err.Error(), "too large") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteLeviathanDocumentFileWritesPatchedBytes(t *testing.T) { + doc := document.New("patched.bin", []byte("abcd")) + if _, err := doc.Set(1, 'Z'); err != nil { + t.Fatalf("Set() failed: %v", err) + } + path := filepath.Join(t.TempDir(), "out.bin") + if err := writeLeviathanDocumentFile(path, doc); err != nil { + t.Fatalf("writeLeviathanDocumentFile() failed: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatalf("ReadFile() failed: %v", err) + } + if string(got) != "aZcd" { + t.Fatalf("written bytes = %q, want %q", string(got), "aZcd") + } +} + +func TestReadLeviathanSmallFileRejectsOversizedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "big.img") + if err := os.WriteFile(path, []byte("abcd"), 0600); err != nil { + t.Fatalf("WriteFile() failed: %v", err) + } + _, err := readLeviathanSmallFile(path, 3, "image") + if err == nil { + t.Fatal("expected oversized file to be rejected") + } + if !strings.Contains(err.Error(), "too large") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestReadSSHPublicKeyFileRejectsOversizedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "big.pub") + if err := os.WriteFile(path, []byte(strings.Repeat("x", maxSSHPublicKeyBytes+1)), 0600); err != nil { + t.Fatalf("WriteFile() failed: %v", err) + } + _, err := readSSHPublicKeyFile(path) + if err == nil { + t.Fatal("expected oversized public key to be rejected") + } + if !strings.Contains(err.Error(), "too large") { + t.Fatalf("unexpected error: %v", err) + } +} + func TestCreateProfileNormalizesLegacyVNCSecretMode(t *testing.T) { svc := newTestService(t) profile := models.NewProfile(models.ProtocolVNC) diff --git a/internal/api/leviathan.go b/internal/api/leviathan.go new file mode 100644 index 0000000..e1e7ba8 --- /dev/null +++ b/internal/api/leviathan.go @@ -0,0 +1,907 @@ +package api + +import ( + "archive/zip" + "encoding/base64" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "time" + + wailsRuntime "github.com/wailsapp/wails/v2/pkg/runtime" + + "github.com/andydixon/nexus/internal/leviathan/analysis" + "github.com/andydixon/nexus/internal/leviathan/binsnoop" + "github.com/andydixon/nexus/internal/leviathan/disasm" + "github.com/andydixon/nexus/internal/leviathan/document" + "github.com/andydixon/nexus/internal/leviathan/parsers" + "github.com/andydixon/nexus/internal/leviathan/patch" + "github.com/andydixon/nexus/internal/leviathan/pdf" + "github.com/andydixon/nexus/internal/leviathan/spectrogram" + "github.com/andydixon/nexus/internal/leviathan/squashfs" + "github.com/andydixon/nexus/internal/leviathan/stego" + "github.com/andydixon/nexus/internal/leviathan/transforms" +) + +type leviathanState struct { + mu sync.Mutex + doc *document.Document + cmp *document.Document +} + +type leviathanDocState struct { + Name string `json:"name"` + Size int64 `json:"size"` + Dirty bool `json:"dirty"` + UndoLen int `json:"undoLen"` + RedoLen int `json:"redoLen"` + Compare *leviathanDocState `json:"compare,omitempty"` + PreviewHex []map[string]any `json:"previewHex"` +} + +func (s *Service) leviathanEngine() *leviathanState { + if s.leviathan == nil { + s.leviathan = &leviathanState{} + } + return s.leviathan +} + +// LeviathanOpenFile prompts for a binary and loads it into the Leviathan workbench. +func (s *Service) LeviathanOpenFile() (map[string]any, error) { + if s.ctx == nil { + return nil, fmt.Errorf("application context not set") + } + path, err := wailsRuntime.OpenFileDialog(s.ctx, wailsRuntime.OpenDialogOptions{ + Title: "Open Binary", + }) + if err != nil || path == "" { + return nil, err + } + return s.LeviathanLoadFile(path, false) +} + +// LeviathanOpenCompare prompts for a binary to compare against the working file. +func (s *Service) LeviathanOpenCompare() (map[string]any, error) { + if s.ctx == nil { + return nil, fmt.Errorf("application context not set") + } + path, err := wailsRuntime.OpenFileDialog(s.ctx, wailsRuntime.OpenDialogOptions{ + Title: "Open Compare Binary", + }) + if err != nil || path == "" { + return nil, err + } + return s.LeviathanLoadFile(path, true) +} + +// LeviathanLoadFile loads a path supplied by drag/drop or the native file picker. +func (s *Service) LeviathanLoadFile(path string, compare bool) (map[string]any, error) { + d, err := document.Open(path) + if err != nil { + return nil, err + } + engine := s.leviathanEngine() + engine.mu.Lock() + if compare { + engine.cmp = d + } else { + engine.doc = d + } + state := engine.stateLocked() + engine.mu.Unlock() + return map[string]any{"state": state}, nil +} + +// LeviathanState returns the current binary workbench state. +func (s *Service) LeviathanState() (leviathanDocState, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + defer engine.mu.Unlock() + return engine.stateLocked(), nil +} + +// LeviathanRunTool runs one Leviathan analysis/editing tool. +func (s *Service) LeviathanRunTool(tool string, params map[string]any) (map[string]any, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + if engine.doc == nil { + engine.mu.Unlock() + return nil, errors.New("open a file first") + } + if tool == "spectrogram-encode" || tool == "spectrogram-insert" || tool == "spectrogram-view" || tool == "case" { + doc := engine.doc + engine.mu.Unlock() + switch tool { + case "spectrogram-encode": + return s.leviathanSpectrogramEncode(doc, params) + case "spectrogram-insert": + return s.leviathanSpectrogramInsert(doc, params) + case "case": + return s.leviathanExportCase(doc) + default: + return leviathanSpectrogramView(doc, params) + } + } + defer engine.mu.Unlock() + return engine.runToolLocked(tool, params) +} + +// LeviathanSetByte patches one byte in the working document. +func (s *Service) LeviathanSetByte(offset int64, value string) (map[string]any, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + defer engine.mu.Unlock() + if engine.doc == nil { + return nil, errors.New("open a file first") + } + value = strings.TrimSpace(value) + value = strings.TrimPrefix(strings.TrimPrefix(value, "0x"), "0X") + if len(value) == 1 { + value = "0" + value + } + if len(value) != 2 { + return nil, errors.New("byte value must be two hex digits") + } + b, err := hex.DecodeString(value) + if err != nil { + return nil, err + } + if _, err := engine.doc.Set(offset, b[0]); err != nil { + return nil, err + } + return map[string]any{"state": engine.stateLocked()}, nil +} + +// LeviathanUndo reverts the latest byte edit. +func (s *Service) LeviathanUndo() (leviathanDocState, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + defer engine.mu.Unlock() + if engine.doc == nil { + return leviathanDocState{}, errors.New("open a file first") + } + engine.doc.Undo() + return engine.stateLocked(), nil +} + +// LeviathanRedo reapplies the latest undone byte edit. +func (s *Service) LeviathanRedo() (leviathanDocState, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + defer engine.mu.Unlock() + if engine.doc == nil { + return leviathanDocState{}, errors.New("open a file first") + } + engine.doc.Redo() + return engine.stateLocked(), nil +} + +// LeviathanRevert clears all working edits. +func (s *Service) LeviathanRevert() (leviathanDocState, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + defer engine.mu.Unlock() + if engine.doc == nil { + return leviathanDocState{}, errors.New("open a file first") + } + engine.doc.Revert() + return engine.stateLocked(), nil +} + +// LeviathanSaveWorking prompts for a destination and writes the patched bytes. +func (s *Service) LeviathanSaveWorking() (string, error) { + engine := s.leviathanEngine() + engine.mu.Lock() + if engine.doc == nil { + engine.mu.Unlock() + return "", errors.New("open a file first") + } + doc := engine.doc + engine.mu.Unlock() + if s.ctx == nil { + return "", fmt.Errorf("application context not set") + } + path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{ + Title: "Save Working Copy", + DefaultFilename: doc.Name() + ".patched", + }) + if err != nil || path == "" { + return "", err + } + return path, writeLeviathanDocumentFile(path, doc) +} + +// LeviathanExportJSON writes a tool result to disk. +func (s *Service) LeviathanExportJSON(name string, value map[string]any) (string, error) { + if s.ctx == nil { + return "", fmt.Errorf("application context not set") + } + path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{ + Title: "Export JSON", + DefaultFilename: safeExportName(name, "leviathan-result.json"), + }) + if err != nil || path == "" { + return "", err + } + b, err := json.MarshalIndent(value, "", " ") + if err != nil { + return "", err + } + return path, os.WriteFile(path, b, 0600) +} + +func (l *leviathanState) stateLocked() leviathanDocState { + state := leviathanDocState{} + if l.doc != nil { + state = leviathanDocStateFor(l.doc, true) + } + if l.cmp != nil { + cmp := leviathanDocStateFor(l.cmp, false) + state.Compare = &cmp + } + return state +} + +func leviathanDocStateFor(d *document.Document, preview bool) leviathanDocState { + s := leviathanDocState{ + Name: d.Name(), + Size: d.Size(), + Dirty: d.IsDirty(), + UndoLen: d.HistoryLen(), + RedoLen: d.RedoLen(), + } + if preview { + s.PreviewHex = leviathanHexRows(d, 0, 512, 8) + } + return s +} + +func (l *leviathanState) runToolLocked(tool string, params map[string]any) (map[string]any, error) { + start := int64Param(params, "start", 0) + length := int64Param(params, "length", -1) + switch tool { + case "overview": + return analysis.Overview(l.doc) + case "hashes": + return analysis.Hashes(l.doc, start, length) + case "entropy": + return analysis.Entropy(l.doc, start, length, int64Param(params, "window", 4096), int64Param(params, "step", 4096)) + case "strings": + items, err := analysis.ExtractStrings(l.doc, start, length, intParam(params, "min", 4), []string{"ascii", "utf16le", "utf16be"}, intParam(params, "max", 5000)) + return map[string]any{"items": items}, err + case "iocs": + return analysis.ExtractIOCs(l.doc, start, length, intParam(params, "min", 5)) + case "carve": + items, err := analysis.Carve(l.doc, start, length, intParam(params, "max", 256)) + return map[string]any{"items": items}, err + case "keys": + items, err := analysis.ScanKeys(l.doc) + return map[string]any{"items": items}, err + case "timestamps": + return analysis.ScanTimestamps(l.doc, start, length, intParam(params, "max", 1000)) + case "search": + return l.runSearch(params, start, length) + case "multi-search": + return analysis.MultiSearch(l.doc, start, length, stringParam(params, "spec", ""), intParam(params, "max", 1000)) + case "parse": + name, tree, err := parseLeviathanDocument(l.doc, stringParam(params, "kind", "auto")) + if err != nil { + return nil, err + } + return map[string]any{"parser": name, "tree": tree.ToMap()}, nil + case "disasm": + return l.runDisasm(params, start, length) + case "diff": + if l.cmp == nil { + return nil, errors.New("open a compare file first") + } + return analysis.Diff(l.doc, l.cmp, start, length) + case "stego": + out, err := stego.AnalyseAppended(l.doc) + if err != nil { + return nil, err + } + meta, _ := stego.ScanMetadata(l.doc) + out["metadata"] = meta + return out, nil + case "metadata": + items, err := stego.ScanMetadata(l.doc) + return map[string]any{"items": items}, err + case "pdf": + return pdf.Parse(l.doc) + case "squashfs": + fs, err := squashfs.Open(l.doc) + if err != nil { + return nil, err + } + entries, err := fs.Walk() + return map[string]any{"superblock": fs.Superblock(), "compression": fs.CompressionName(), "entries": entries}, err + case "binsnoop": + buf, err := l.rangeBytes(0, min64(l.doc.Size(), int64Param(params, "maxBytes", 64<<20))) + if err != nil { + return nil, err + } + scanner := binsnoop.New() + return map[string]any{"results": scanner.AnalyzeBytes(buf, l.doc.Name(), false), "signatures": binsnoop.ListSignatures()}, nil + case "rules": + rules, err := analysis.ParseRules(stringParam(params, "source", "")) + if err != nil { + return nil, err + } + items, err := analysis.RunRules(l.doc, start, length, rules, intParam(params, "max", 1000)) + return map[string]any{"items": items}, err + case "patch-plan": + steps, err := patch.Parse(stringParam(params, "source", "")) + if err != nil { + return nil, err + } + plan, err := patch.Plan(l.doc, steps) + return map[string]any{"plan": plan}, err + case "patch-apply": + steps, err := patch.Parse(stringParam(params, "source", "")) + if err != nil { + return nil, err + } + if _, err := patch.Plan(l.doc, steps); err != nil { + return nil, err + } + return map[string]any{"state": l.stateLocked()}, patch.Apply(l.doc, steps) + case "transform": + return l.runTransform(params, start, length) + case "hex": + return map[string]any{"rows": leviathanHexRows(l.doc, start, int64Param(params, "bytes", 1024), intParam(params, "width", 16))}, nil + case "case": + return nil, errors.New("case export must run through the service") + default: + return nil, errors.New("unknown tool: " + tool) + } +} + +func (l *leviathanState) runSearch(params map[string]any, start, length int64) (map[string]any, error) { + mode := stringParam(params, "mode", "text") + query := stringParam(params, "query", "") + max := intParam(params, "max", 1000) + switch mode { + case "hex": + b, err := hex.DecodeString(strings.ReplaceAll(query, " ", "")) + if err != nil { + return nil, err + } + hits, err := analysis.SearchBytes(l.doc, start, length, b, max) + return map[string]any{"hits": analysis.HitsToMaps(hits)}, err + case "regex": + hits, err := analysis.SearchRegex(l.doc, start, length, query, max) + return map[string]any{"hits": analysis.HitsToMaps(hits)}, err + default: + hits, err := analysis.SearchBytes(l.doc, start, length, []byte(query), max) + return map[string]any{"hits": analysis.HitsToMaps(hits)}, err + } +} + +func (l *leviathanState) runDisasm(params map[string]any, start, length int64) (map[string]any, error) { + if length < 0 || length > 65536 { + length = 4096 + } + buf, err := l.rangeBytes(start, length) + if err != nil { + return nil, err + } + return map[string]any{"items": disasm.Disassemble(buf, start, stringParam(params, "arch", "x86_64"), intParam(params, "max", 1000))}, nil +} + +func (l *leviathanState) runTransform(params map[string]any, start, length int64) (map[string]any, error) { + if length < 0 || length > 8<<20 { + length = min64(l.doc.Size()-start, 1<<20) + } + buf, err := l.rangeBytes(start, length) + if err != nil { + return nil, err + } + steps := parseLeviathanTransformPipeline(stringParam(params, "pipeline", "hex.encode")) + results, err := transforms.ApplyPipeline(buf, steps) + if err != nil { + return nil, err + } + out := make([]map[string]any, 0, len(results)) + for i, r := range results { + out = append(out, map[string]any{"step": i, "length": len(r), "preview": previewLeviathanBytes(r)}) + } + return map[string]any{"steps": steps, "results": out}, nil +} + +func leviathanSpectrogramView(doc *document.Document, params map[string]any) (map[string]any, error) { + buf, err := leviathanDocumentBytes(doc, spectrogram.MaxAudioBytes, "audio") + if err != nil { + return nil, err + } + pngBytes, summary, err := spectrogram.AudioSpectrogramPNG(buf, doc.Name(), leviathanSpectrogramViewOptions(params)) + if err != nil { + return nil, err + } + return map[string]any{ + "mime": "image/png", + "pngBase64": base64.StdEncoding.EncodeToString(pngBytes), + "summary": summary, + }, nil +} + +func (s *Service) leviathanExportCase(doc *document.Document) (map[string]any, error) { + if s.ctx == nil { + return nil, fmt.Errorf("application context not set") + } + filename := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".case.zip" + path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{ + Title: "Export Leviathan Case", + DefaultFilename: filename, + }) + if err != nil || path == "" { + return nil, err + } + + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return nil, err + } + defer f.Close() + + zw := zip.NewWriter(f) + add := func(name string, data []byte) error { + w, err := zw.Create(name) + if err != nil { + return err + } + _, err = w.Write(data) + return err + } + hashes, _ := analysis.Hashes(doc, 0, -1) + overview, _ := analysis.Overview(doc) + manifest, _ := json.MarshalIndent(map[string]any{"name": doc.Name(), "size": doc.Size(), "exportedAt": time.Now().Format(time.RFC3339), "hashes": hashes}, "", " ") + overviewJSON, _ := json.MarshalIndent(overview, "", " ") + patchesJSON, _ := json.MarshalIndent(doc.Patches(), "", " ") + if err := add("manifest.json", manifest); err != nil { + return nil, err + } + if err := addDocument("working.bin", doc, zw); err != nil { + return nil, err + } + if err := add("overview.json", overviewJSON); err != nil { + return nil, err + } + if err := add("patches.json", patchesJSON); err != nil { + return nil, err + } + if err := zw.Close(); err != nil { + return nil, err + } + if err := f.Sync(); err != nil { + return nil, err + } + info, err := f.Stat() + if err != nil { + return nil, err + } + return map[string]any{"path": path, "bytes": info.Size()}, nil +} + +func (s *Service) leviathanSpectrogramEncode(doc *document.Document, params map[string]any) (map[string]any, error) { + if s.ctx == nil { + return nil, fmt.Errorf("application context not set") + } + imageBytes, err := leviathanDocumentBytes(doc, spectrogram.MaxImageBytes, "image") + if err != nil { + return nil, err + } + wavBytes, summary, err := spectrogram.ImageToWAV(imageBytes, leviathanSpectrogramEncodeOptions(params)) + if err != nil { + return nil, err + } + defaultName := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".spectrogram.wav" + path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{ + Title: "Save Spectrogram WAV", + DefaultFilename: safeExportName(defaultName, "spectrogram.wav"), + Filters: []wailsRuntime.FileFilter{ + {DisplayName: "WAV Audio (*.wav)", Pattern: "*.wav"}, + }, + }) + if err != nil || path == "" { + return nil, err + } + if filepath.Ext(strings.ToLower(path)) != ".wav" { + path += ".wav" + } + if err := os.WriteFile(path, wavBytes, 0600); err != nil { + return nil, err + } + summary["path"] = path + return map[string]any{"path": path, "summary": summary}, nil +} + +func (s *Service) leviathanSpectrogramInsert(doc *document.Document, params map[string]any) (map[string]any, error) { + if s.ctx == nil { + return nil, fmt.Errorf("application context not set") + } + imagePath := strings.TrimSpace(stringParam(params, "imagePath", "")) + if imagePath == "" { + return nil, errors.New("select an image to insert") + } + imageBytes, err := readLeviathanSmallFile(imagePath, spectrogram.MaxImageBytes, "image") + if err != nil { + return nil, fmt.Errorf("read image: %w", err) + } + audioBytes, err := leviathanDocumentBytes(doc, spectrogram.MaxAudioBytes, "audio") + if err != nil { + return nil, err + } + wavBytes, summary, err := spectrogram.InsertImageIntoAudio(imageBytes, audioBytes, doc.Name(), leviathanSpectrogramInsertOptions(params)) + if err != nil { + return nil, err + } + defaultName := strings.TrimSuffix(doc.Name(), filepath.Ext(doc.Name())) + ".with-spectrogram.wav" + path, err := wailsRuntime.SaveFileDialog(s.ctx, wailsRuntime.SaveDialogOptions{ + Title: "Save Audio With Spectrogram Image", + DefaultFilename: safeExportName(defaultName, "with-spectrogram.wav"), + Filters: []wailsRuntime.FileFilter{ + {DisplayName: "WAV Audio (*.wav)", Pattern: "*.wav"}, + }, + }) + if err != nil || path == "" { + return nil, err + } + if filepath.Ext(strings.ToLower(path)) != ".wav" { + path += ".wav" + } + if err := os.WriteFile(path, wavBytes, 0600); err != nil { + return nil, err + } + summary["path"] = path + summary["imagePath"] = imagePath + return map[string]any{"path": path, "summary": summary}, nil +} + +func writeLeviathanDocumentFile(path string, doc *document.Document) error { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return err + } + defer f.Close() + if err := doc.ForEachChunk(0, doc.Size(), func(_ int64, chunk []byte) error { + _, err := f.Write(chunk) + return err + }); err != nil { + return err + } + return f.Sync() +} + +func addDocument(name string, doc *document.Document, zw *zip.Writer) error { + w, err := zw.Create(name) + if err != nil { + return err + } + return doc.ForEachChunk(0, doc.Size(), func(_ int64, chunk []byte) error { + _, err := w.Write(chunk) + return err + }) +} + +func leviathanDocumentBytes(doc *document.Document, maxBytes int64, label string) ([]byte, error) { + size := doc.Size() + if maxBytes > 0 && size > maxBytes { + return nil, fmt.Errorf("%s file is too large for this operation: %s exceeds %s", label, humanBytes(size), humanBytes(maxBytes)) + } + return doc.Range(0, size) +} + +func readLeviathanSmallFile(path string, maxBytes int64, label string) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("%s path is a directory", label) + } + if maxBytes > 0 && info.Size() > maxBytes { + return nil, fmt.Errorf("%s file is too large for this operation: %s exceeds %s", label, humanBytes(info.Size()), humanBytes(maxBytes)) + } + return os.ReadFile(path) +} + +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + value := float64(n) + for _, suffix := range []string{"KiB", "MiB", "GiB", "TiB"} { + value /= unit + if value < unit { + return fmt.Sprintf("%.1f %s", value, suffix) + } + } + return fmt.Sprintf("%.1f PiB", value/unit) +} + +func (l *leviathanState) rangeBytes(start, length int64) ([]byte, error) { + if start < 0 { + start = 0 + } + if length < 0 || start+length > l.doc.Size() { + length = l.doc.Size() - start + } + if length < 0 { + length = 0 + } + return l.doc.Range(start, length) +} + +func parseLeviathanDocument(d *document.Document, kind string) (string, *parsers.Field, error) { + kind = strings.ToLower(strings.TrimSpace(kind)) + if kind == "" || kind == "auto" { + return parsers.DetectAndParse(d) + } + var tree *parsers.Field + var err error + switch kind { + case "png": + tree, err = parsers.ParsePNG(d) + case "jpeg", "jpg": + tree, err = parsers.ParseJPEG(d) + kind = "jpeg" + case "gif": + tree, err = parsers.ParseGIF(d) + case "bmp": + tree, err = parsers.ParseBMP(d) + case "zip": + tree, err = parsers.ParseZIP(d) + case "elf": + tree, err = parsers.ParseELF(d) + case "pe": + tree, err = parsers.ParsePE(d) + case "macho": + tree, err = parsers.ParseMachO(d) + case "riff": + tree, err = parsers.ParseRIFF(d) + case "asn1": + tree, err = parsers.ParseASN1(d, 0, d.Size()) + case "cert": + tree, err = parsers.ParseCerts(d) + default: + return "", nil, errors.New("unknown parser: " + kind) + } + return kind, tree, err +} + +func leviathanHexRows(d *document.Document, start, length int64, width int) []map[string]any { + if width <= 0 { + width = 16 + } + if length < 0 || length > 8192 { + length = 8192 + } + rows := []map[string]any{} + for off := start; off < start+length && off < d.Size(); off += int64(width) { + n := int64(width) + if off+n > d.Size() { + n = d.Size() - off + } + buf, err := d.Range(off, n) + if err != nil { + break + } + rows = append(rows, map[string]any{"offset": off, "hex": spacedLeviathanHex(buf), "bytes": byteHexCells(buf), "ascii": asciiLeviathanPreview(buf)}) + } + return rows +} + +func parseLeviathanTransformPipeline(src string) []transforms.Step { + parts := strings.Split(src, "|") + steps := []transforms.Step{} + for _, part := range parts { + fields := strings.Fields(strings.TrimSpace(part)) + if len(fields) == 0 { + continue + } + op := fields[0] + arg := "" + if len(fields) > 1 { + arg = strings.Join(fields[1:], " ") + } + switch op { + case "hex": + op = "hex.encode" + case "unhex": + op = "hex.decode" + case "base64", "b64": + op = "base64.encode" + case "unbase64", "unb64": + op = "base64.decode" + case "url": + op = "url.encode" + case "unurl": + op = "url.decode" + } + steps = append(steps, transforms.Step{Op: op, Arg: arg}) + } + return steps +} + +func spacedLeviathanHex(buf []byte) string { + return strings.Join(byteHexCells(buf), " ") +} + +func byteHexCells(buf []byte) []string { + parts := make([]string, len(buf)) + for i, b := range buf { + parts[i] = fmt.Sprintf("%02x", b) + } + return parts +} + +func asciiLeviathanPreview(buf []byte) string { + var b strings.Builder + for _, c := range buf { + if c >= 32 && c < 127 { + b.WriteByte(c) + } else { + b.WriteByte('.') + } + } + return b.String() +} + +func previewLeviathanBytes(buf []byte) string { + if len(buf) > 2048 { + buf = buf[:2048] + } + if isMostlyLeviathanText(buf) { + return string(buf) + } + return spacedLeviathanHex(buf) +} + +func isMostlyLeviathanText(buf []byte) bool { + if len(buf) == 0 { + return true + } + printable := 0 + for _, b := range buf { + if b == '\n' || b == '\r' || b == '\t' || (b >= 32 && b < 127) { + printable++ + } + } + return printable*100/len(buf) > 85 +} + +func intParam(m map[string]any, key string, def int) int { + return int(int64Param(m, key, int64(def))) +} + +func int64Param(m map[string]any, key string, def int64) int64 { + v, ok := m[key] + if !ok || v == nil { + return def + } + switch n := v.(type) { + case int64: + return n + case int: + return int64(n) + case float64: + return int64(n) + case string: + if n == "" { + return def + } + if parsed, err := strconv.ParseInt(n, 0, 64); err == nil { + return parsed + } + } + return def +} + +func stringParam(m map[string]any, key, def string) string { + if v, ok := m[key]; ok && v != nil { + return fmt.Sprint(v) + } + return def +} + +func floatParam(m map[string]any, key string, def float64) float64 { + v, ok := m[key] + if !ok || v == nil { + return def + } + switch n := v.(type) { + case float64: + return n + case float32: + return float64(n) + case int: + return float64(n) + case int64: + return float64(n) + case string: + if n == "" { + return def + } + if parsed, err := strconv.ParseFloat(n, 64); err == nil { + return parsed + } + } + return def +} + +func boolParam(m map[string]any, key string, def bool) bool { + v, ok := m[key] + if !ok || v == nil { + return def + } + switch b := v.(type) { + case bool: + return b + case string: + if b == "" { + return def + } + if parsed, err := strconv.ParseBool(b); err == nil { + return parsed + } + } + return def +} + +func leviathanSpectrogramEncodeOptions(params map[string]any) spectrogram.EncodeOptions { + return spectrogram.EncodeOptions{ + DurationSec: floatParam(params, "durationSec", 5), + SampleRate: intParam(params, "sampleRate", 44100), + MinHz: floatParam(params, "minHz", 200), + MaxHz: floatParam(params, "maxHz", 20000), + Bins: intParam(params, "bins", 160), + Columns: intParam(params, "columns", 768), + Invert: boolParam(params, "invert", true), + PreserveAR: boolParam(params, "preserveAspectRatio", true), + Gain: floatParam(params, "gain", 0.85), + } +} + +func leviathanSpectrogramInsertOptions(params map[string]any) spectrogram.InsertOptions { + return spectrogram.InsertOptions{ + EncodeOptions: leviathanSpectrogramEncodeOptions(params), + StartSec: floatParam(params, "startSec", 0), + MixLevel: floatParam(params, "mixLevel", 0.35), + } +} + +func leviathanSpectrogramViewOptions(params map[string]any) spectrogram.ViewOptions { + return spectrogram.ViewOptions{ + FFTSize: intParam(params, "fftSize", 2048), + Height: intParam(params, "height", 1024), + MaxWidth: intParam(params, "maxWidth", 1400), + Aspect: floatParam(params, "aspectRatio", 1), + MinHz: floatParam(params, "minHz", 0), + MaxHz: floatParam(params, "maxHz", 0), + Brightness: floatParam(params, "brightness", 1), + Contrast: floatParam(params, "contrast", 1), + } +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +func safeExportName(name, fallback string) string { + name = filepath.Base(strings.TrimSpace(name)) + if name == "" || name == "." || strings.ContainsRune(name, 0) { + return fallback + } + return name +} diff --git a/internal/githubclient/githubclient.go b/internal/githubclient/githubclient.go index cd8fab9..2b22769 100644 --- a/internal/githubclient/githubclient.go +++ b/internal/githubclient/githubclient.go @@ -19,6 +19,11 @@ import ( "github.com/andydixon/nexus/internal/models" ) +const ( + maxGitHubJSONBodyBytes = 4 << 20 + maxGitHubErrorBodyBytes = 64 << 10 +) + // Client wraps the GitHub REST API for repository listing. type Client struct { baseURL string @@ -41,19 +46,19 @@ func New(baseURL, token string) *Client { // apiRepo is the subset of the GitHub API response we care about. type apiRepo struct { - FullName string `json:"full_name"` - Name string `json:"name"` - Description string `json:"description"` - HTMLURL string `json:"html_url"` - DefaultBranch string `json:"default_branch"` - Private bool `json:"private"` - Fork bool `json:"fork"` - Archived bool `json:"archived"` - Language string `json:"language"` - Stars int `json:"stargazers_count"` - PushedAt time.Time `json:"pushed_at"` - UpdatedAt time.Time `json:"updated_at"` - Owner struct { + FullName string `json:"full_name"` + Name string `json:"name"` + Description string `json:"description"` + HTMLURL string `json:"html_url"` + DefaultBranch string `json:"default_branch"` + Private bool `json:"private"` + Fork bool `json:"fork"` + Archived bool `json:"archived"` + Language string `json:"language"` + Stars int `json:"stargazers_count"` + PushedAt time.Time `json:"pushed_at"` + UpdatedAt time.Time `json:"updated_at"` + Owner struct { Login string `json:"login"` } `json:"owner"` } @@ -90,7 +95,7 @@ func (c *Client) fetchRepos(ctx context.Context, progress func(done int, total i return nil, fmt.Errorf("fetching repos page %d: %w", page, err) } - body, readErr := io.ReadAll(resp.Body) + body, readErr := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes) resp.Body.Close() if readErr != nil { return nil, fmt.Errorf("reading response body: %w", readErr) @@ -128,19 +133,19 @@ func (c *Client) fetchRepos(ctx context.Context, progress func(done int, total i result := make([]models.GitHubRepo, len(all)) for i, r := range all { result[i] = models.GitHubRepo{ - FullName: r.FullName, - Name: r.Name, - Owner: r.Owner.Login, - Description: r.Description, - HTMLURL: r.HTMLURL, + FullName: r.FullName, + Name: r.Name, + Owner: r.Owner.Login, + Description: r.Description, + HTMLURL: r.HTMLURL, DefaultBranch: r.DefaultBranch, - Private: r.Private, - Fork: r.Fork, - Archived: r.Archived, - Language: r.Language, - Stars: r.Stars, - PushedAt: r.PushedAt.Format(time.RFC3339), - UpdatedAt: r.UpdatedAt.Format(time.RFC3339), + Private: r.Private, + Fork: r.Fork, + Archived: r.Archived, + Language: r.Language, + Stars: r.Stars, + PushedAt: r.PushedAt.Format(time.RFC3339), + UpdatedAt: r.UpdatedAt.Format(time.RFC3339), } } @@ -208,8 +213,7 @@ func (c *Client) FetchLatestCommit(ctx context.Context, fullName string, ref str return "", "", fmt.Errorf("authentication failed (HTTP %d)", resp.StatusCode) } if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return "", "", fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, string(body)) + return "", "", fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, readGitHubErrorBody(resp.Body)) } var payload struct { @@ -225,7 +229,11 @@ func (c *Client) FetchLatestCommit(ctx context.Context, fullName string, ref str } `json:"commit"` } - if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + body, err := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes) + if err != nil { + return "", "", fmt.Errorf("reading commit response: %w", err) + } + if err := json.Unmarshal(body, &payload); err != nil { return "", "", fmt.Errorf("decoding commit: %w", err) } @@ -270,8 +278,7 @@ func (c *Client) DownloadRepoArchive(ctx context.Context, fullName string, ref s return fmt.Errorf("authentication failed (HTTP %d) — check your access token", resp.StatusCode) } if resp.StatusCode != http.StatusOK { - body, _ := io.ReadAll(resp.Body) - return fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, string(body)) + return fmt.Errorf("GitHub API returned HTTP %d: %s", resp.StatusCode, readGitHubErrorBody(resp.Body)) } file, err := os.Create(destPath) @@ -310,8 +317,31 @@ func (c *Client) ValidateToken(ctx context.Context) (string, error) { var user struct { Login string `json:"login"` } - if err := json.NewDecoder(resp.Body).Decode(&user); err != nil { + body, err := readLimitedGitHubBody(resp.Body, maxGitHubJSONBodyBytes) + if err != nil { + return "", fmt.Errorf("reading user response: %w", err) + } + if err := json.Unmarshal(body, &user); err != nil { return "", fmt.Errorf("decoding user: %w", err) } return user.Login, nil } + +func readLimitedGitHubBody(r io.Reader, maxBytes int64) ([]byte, error) { + body, err := io.ReadAll(io.LimitReader(r, maxBytes+1)) + if err != nil { + return nil, err + } + if int64(len(body)) > maxBytes { + return nil, fmt.Errorf("response body exceeds %d bytes", maxBytes) + } + return body, nil +} + +func readGitHubErrorBody(r io.Reader) string { + body, err := readLimitedGitHubBody(r, maxGitHubErrorBodyBytes) + if err != nil { + return "" + } + return strings.TrimSpace(string(body)) +} diff --git a/internal/githubclient/githubclient_test.go b/internal/githubclient/githubclient_test.go new file mode 100644 index 0000000..4b07187 --- /dev/null +++ b/internal/githubclient/githubclient_test.go @@ -0,0 +1,26 @@ +package githubclient + +import ( + "strings" + "testing" +) + +func TestReadLimitedGitHubBodyRejectsOversizedResponse(t *testing.T) { + _, err := readLimitedGitHubBody(strings.NewReader("abcd"), 3) + if err == nil { + t.Fatal("expected oversized response to be rejected") + } + if !strings.Contains(err.Error(), "response body exceeds") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestReadLimitedGitHubBodyAllowsResponseAtLimit(t *testing.T) { + got, err := readLimitedGitHubBody(strings.NewReader("abc"), 3) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if string(got) != "abc" { + t.Fatalf("body = %q, want abc", string(got)) + } +} diff --git a/internal/leviathan/analysis/carve.go b/internal/leviathan/analysis/carve.go new file mode 100644 index 0000000..55296b6 --- /dev/null +++ b/internal/leviathan/analysis/carve.go @@ -0,0 +1,876 @@ +package analysis + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/binary" + "io" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Signature describes a carveable object. `validate` is run against +// the candidate start and returns the object length if it is a real +// object, or 0 to reject. `trailer`, if present, must appear before +// the object ends (for simple trailer-based terminations). +type Signature struct { + Kind string + Magic []byte + Extension string + // Anchor is how many bytes into the object the magic appears. It is 0 for + // almost every format (magic at offset 0), but a few — notably ISO-BMFF + // (MP4/MOV), whose `ftyp` magic follows a 4-byte box size — carry it. + Anchor int64 + Validate func(d *document.Document, start, max int64) (length int64, confidence float64) +} + +// carveSignatures is the phase-1 signature set. We stay conservative +// and only declare a carve when we can determine length reliably. +var carveSignatures = []Signature{ + {Kind: "png", Magic: []byte{0x89, 'P', 'N', 'G', 0x0d, 0x0a, 0x1a, 0x0a}, Extension: "png", Validate: validatePNG}, + {Kind: "jpeg", Magic: []byte{0xff, 0xd8, 0xff}, Extension: "jpg", Validate: validateJPEG}, + {Kind: "gif87a", Magic: []byte("GIF87a"), Extension: "gif", Validate: validateGIF}, + {Kind: "gif89a", Magic: []byte("GIF89a"), Extension: "gif", Validate: validateGIF}, + {Kind: "zip", Magic: []byte{'P', 'K', 0x03, 0x04}, Extension: "zip", Validate: validateZIP}, + {Kind: "pdf", Magic: []byte("%PDF-"), Extension: "pdf", Validate: validatePDF}, + {Kind: "elf", Magic: []byte{0x7f, 'E', 'L', 'F'}, Extension: "elf", Validate: validateELF}, + {Kind: "mz", Magic: []byte{'M', 'Z'}, Extension: "exe", Validate: validatePE}, + {Kind: "gzip", Magic: []byte{0x1f, 0x8b, 0x08}, Extension: "gz", Validate: validateGZIP}, + {Kind: "bzip2", Magic: []byte{'B', 'Z', 'h'}, Extension: "bz2", Validate: validateBzip2}, + {Kind: "sevenz", Magic: []byte{'7', 'z', 0xbc, 0xaf, 0x27, 0x1c}, Extension: "7z", Validate: validate7z}, + {Kind: "rar4", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x00}, Extension: "rar", Validate: validateRAR}, + {Kind: "rar5", Magic: []byte{'R', 'a', 'r', '!', 0x1a, 0x07, 0x01, 0x00}, Extension: "rar", Validate: validateRAR}, + {Kind: "wav", Magic: []byte("RIFF"), Extension: "wav", Validate: validateRIFF}, + {Kind: "mach-o-le32", Magic: []byte{0xce, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO}, + {Kind: "mach-o-le64", Magic: []byte{0xcf, 0xfa, 0xed, 0xfe}, Extension: "macho", Validate: validateMachO}, + {Kind: "mach-o-be32", Magic: []byte{0xfe, 0xed, 0xfa, 0xce}, Extension: "macho", Validate: validateMachO}, + {Kind: "mach-o-be64", Magic: []byte{0xfe, 0xed, 0xfa, 0xcf}, Extension: "macho", Validate: validateMachO}, + {Kind: "sqlite", Magic: []byte("SQLite format 3\x00"), Extension: "sqlite", Validate: validateSQLite}, + {Kind: "mp3", Magic: []byte("ID3"), Extension: "mp3", Validate: validateID3}, + {Kind: "ogg", Magic: []byte("OggS"), Extension: "ogg", Validate: validateOgg}, + {Kind: "matroska", Magic: []byte{0x1a, 0x45, 0xdf, 0xa3}, Extension: "mkv", Validate: validateMatroska}, + {Kind: "flac", Magic: []byte("fLaC"), Extension: "flac", Validate: validateFLAC}, + {Kind: "bmp", Magic: []byte{'B', 'M'}, Extension: "bmp", Validate: validateBMP}, + {Kind: "mp4", Magic: []byte("ftyp"), Anchor: 4, Extension: "mp4", Validate: validateMP4}, +} + +// Carve scans the document for known signatures and returns a list of +// discovered objects with offset, length, kind, and confidence. +func Carve(d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) { + return CarveContext(context.Background(), d, start, length, maxHits) +} + +func CarveContext(ctx context.Context, d *document.Document, start, length int64, maxHits int) ([]map[string]any, error) { + if length < 0 { + length = d.Size() - start + } + if maxHits <= 0 { + maxHits = 10_000 + } + results := make([]map[string]any, 0, 64) + + // Build a flat list of (magicBytes, sigIndex) for a simple first-pass scan. + // Phase 1 uses per-signature naive scan chunks. + for idx, sig := range carveSignatures { + if err := ctx.Err(); err != nil { + return nil, err + } + hits, err := SearchBytesContext(ctx, d, start, length, sig.Magic, maxHits) + if err != nil { + return nil, err + } + for _, h := range hits { + if err := ctx.Err(); err != nil { + return nil, err + } + // The object begins Anchor bytes before the magic (0 for most formats). + objStart := h.Offset - sig.Anchor + if objStart < start { + continue + } + ln, conf := int64(0), 0.4 + if sig.Validate != nil { + ln, conf = sig.Validate(d, objStart, start+length) + } + if ln <= 0 { + // record low-confidence magic hit only for kinds where length is not determinable + if sig.Kind == "mz" || sig.Kind == "elf" || sig.Kind == "mach-o-le32" || + sig.Kind == "mach-o-le64" || sig.Kind == "mach-o-be32" || sig.Kind == "mach-o-be64" || + sig.Kind == "sqlite" || sig.Kind == "mp3" || sig.Kind == "matroska" || + sig.Kind == "flac" || sig.Kind == "bzip2" || sig.Kind == "rar4" || sig.Kind == "rar5" { + results = append(results, map[string]any{ + "kind": sig.Kind, + "extension": sig.Extension, + "offset": objStart, + "length": int64(0), + "confidence": 0.3, + "status": "magic-only", + "sigIndex": idx, + }) + } + continue + } + results = append(results, map[string]any{ + "kind": sig.Kind, + "extension": sig.Extension, + "offset": objStart, + "length": ln, + "confidence": conf, + "status": "valid", + "sigIndex": idx, + }) + if len(results) >= maxHits { + return results, nil + } + } + } + return results, nil +} + +// --- validators --- + +func validatePNG(d *document.Document, start, max int64) (int64, float64) { + // Walk chunks until IEND + off := start + 8 + for off < max { + hdr, err := d.Range(off, 8) + if err != nil || len(hdr) < 8 { + return 0, 0 + } + size := int64(binary.BigEndian.Uint32(hdr[0:4])) + typ := string(hdr[4:8]) + // Skip payload + CRC (4 bytes) + off += 8 + size + 4 + if typ == "IEND" { + return off - start, 0.95 + } + if off > max { + return 0, 0 + } + } + return 0, 0 +} + +func validateJPEG(d *document.Document, start, max int64) (int64, float64) { + // Walk segments until EOI (0xFFD9) + off := start + 2 + for off < max-1 { + buf, err := d.Range(off, 2) + if err != nil || len(buf) < 2 { + return 0, 0 + } + if buf[0] != 0xff { + return 0, 0 + } + marker := buf[1] + off += 2 + // No-length markers + if marker == 0xd9 { // EOI + return off - start, 0.95 + } + if marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) || marker == 0x00 { + continue + } + // Read length + lb, err := d.Range(off, 2) + if err != nil || len(lb) < 2 { + return 0, 0 + } + ln := int64(binary.BigEndian.Uint16(lb)) + off += ln + if marker == 0xda { // SOS, scan until next 0xff!=0 + for off < max { + b, err := d.Range(off, 2) + if err != nil || len(b) < 2 { + return 0, 0 + } + if b[0] == 0xff && b[1] != 0x00 && !(b[1] >= 0xd0 && b[1] <= 0xd7) { + break + } + off++ + } + } + } + return 0, 0 +} + +func validateGIF(d *document.Document, start, max int64) (int64, float64) { + // Minimal: locate trailer 0x3B + buf, err := d.Range(start, min64(max-start, 1<<20)) + if err != nil { + return 0, 0 + } + idx := bytes.IndexByte(buf[13:], 0x3b) + if idx < 0 { + return 0, 0 + } + return int64(idx + 13 + 1), 0.7 +} + +func validateZIP(d *document.Document, start, max int64) (int64, float64) { + // Search forward for End of Central Directory record 0x06054b50. + const sig = "\x50\x4b\x05\x06" + limit := min64(max-start, 64<<20) + buf, err := d.Range(start, limit) + if err != nil { + return 0, 0 + } + idx := bytes.LastIndex(buf, []byte(sig)) + if idx < 0 { + return 0, 0 + } + // EOCD record is at least 22 bytes; read comment length. + if idx+22 > len(buf) { + return 0, 0 + } + commentLen := int(binary.LittleEndian.Uint16(buf[idx+20 : idx+22])) + end := int64(idx + 22 + commentLen) + return end, 0.9 +} + +func validatePDF(d *document.Document, start, max int64) (int64, float64) { + limit := min64(max-start, 64<<20) + buf, err := d.Range(start, limit) + if err != nil { + return 0, 0 + } + idx := bytes.LastIndex(buf, []byte("%%EOF")) + if idx < 0 { + return 0, 0 + } + return int64(idx + 5), 0.85 +} + +func validateELF(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 64) + if err != nil || len(hdr) < 52 { + return 0, 0 + } + if hdr[4] != 1 && hdr[4] != 2 { + return 0, 0 + } + is64 := hdr[4] == 2 + little := hdr[5] == 1 + var end int64 + if is64 { + if len(hdr) < 64 { + return 0, 0 + } + // e_shoff at 40, e_shentsize at 58, e_shnum at 60 + off := binary.LittleEndian.Uint64(hdr[40:48]) + entSz := uint64(binary.LittleEndian.Uint16(hdr[58:60])) + num := uint64(binary.LittleEndian.Uint16(hdr[60:62])) + if !little { + off = binary.BigEndian.Uint64(hdr[40:48]) + entSz = uint64(binary.BigEndian.Uint16(hdr[58:60])) + num = uint64(binary.BigEndian.Uint16(hdr[60:62])) + } + end = int64(off + entSz*num) + } else { + off := binary.LittleEndian.Uint32(hdr[32:36]) + entSz := uint32(binary.LittleEndian.Uint16(hdr[46:48])) + num := uint32(binary.LittleEndian.Uint16(hdr[48:50])) + if !little { + off = binary.BigEndian.Uint32(hdr[32:36]) + entSz = uint32(binary.BigEndian.Uint16(hdr[46:48])) + num = uint32(binary.BigEndian.Uint16(hdr[48:50])) + } + end = int64(off) + int64(entSz)*int64(num) + } + if end <= 0 || start+end > max { + return 0, 0.5 + } + return end, 0.9 +} + +func validatePE(d *document.Document, start, max int64) (int64, float64) { + // e_lfanew at offset 0x3C points to PE header + hdr, err := d.Range(start, 64) + if err != nil || len(hdr) < 64 { + return 0, 0 + } + peOff := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40])) + if peOff < 0x40 || start+peOff+4 > max { + return 0, 0 + } + sig, err := d.Range(start+peOff, 4) + if err != nil || string(sig) != "PE\x00\x00" { + return 0, 0.3 + } + coff, err := d.Range(start+peOff+4, 20) + if err != nil || len(coff) < 20 { + return 0, 0.4 + } + numSections := int(binary.LittleEndian.Uint16(coff[2:4])) + sizeOpt := int64(binary.LittleEndian.Uint16(coff[16:18])) + if numSections <= 0 || numSections > 96 || sizeOpt < 0 || sizeOpt > 4096 { + return 0, 0.5 + } + sectOff := start + peOff + 24 + sizeOpt + end := peOff + 24 + sizeOpt + int64(numSections*40) + for i := 0; i < numSections; i++ { + sh, err := d.Range(sectOff+int64(i*40), 40) + if err != nil || len(sh) < 40 { + return 0, 0.5 + } + rawSize := int64(binary.LittleEndian.Uint32(sh[16:20])) + rawPtr := int64(binary.LittleEndian.Uint32(sh[20:24])) + if rawSize < 0 || rawPtr < 0 { + return 0, 0.5 + } + if rawSize > 0 && rawPtr+rawSize > end { + end = rawPtr + rawSize + } + } + if end <= peOff || start+end > max { + return 0, 0.5 + } + return end, 0.85 +} + +func validateGZIP(d *document.Document, start, max int64) (int64, float64) { + limit := min64(max-start, 64<<20) + buf, err := d.Range(start, limit) + if err != nil || len(buf) < 18 { + return 0, 0 + } + br := bytes.NewReader(buf) + zr, err := gzip.NewReader(br) + if err != nil { + return 0, 0 + } + zr.Multistream(false) + if _, err := io.Copy(io.Discard, io.LimitReader(zr, 512<<20)); err != nil { + _ = zr.Close() + return 0, 0 + } + if err := zr.Close(); err != nil { + return 0, 0 + } + consumed := int64(len(buf)) - int64(br.Len()) + if consumed <= 10 || start+consumed > max { + return 0, 0 + } + return consumed, 0.9 +} + +// validateBzip2 locates the bit-aligned end-of-stream marker (0x177245385090, +// the "sqrt(pi)" magic) and returns the stream length including the trailing +// 32-bit combined CRC. bzip2 carries no length in its header, so this is the +// only reliable way to bound the object. +func validateBzip2(d *document.Document, start, max int64) (int64, float64) { + if start+4 > max { + return 0, 0.4 + } + hdr, err := d.Range(start, 4) + if err != nil || len(hdr) < 4 || hdr[0] != 'B' || hdr[1] != 'Z' || hdr[2] != 'h' || hdr[3] < '1' || hdr[3] > '9' { + return 0, 0 + } + limit := min64(max-start, 64<<20) + buf, err := d.Range(start, limit) + if err != nil || len(buf) < 14 { + return 0, 0.4 + } + const eos uint64 = 0x177245385090 + const mask48 uint64 = (1 << 48) - 1 + var w uint64 + for i := 0; i < len(buf); i++ { + w = (w << 8) | uint64(buf[i]) + // Check all eight bit alignments ending at or before this byte. + for s := 0; s < 8; s++ { + if (w>>uint(s))&mask48 == eos { + endBit := (int64(i)+1)*8 - int64(s) + 32 // include stream CRC + endByte := (endBit + 7) / 8 + if endByte > 14 && start+endByte <= max { + return endByte, 0.85 + } + } + } + } + return 0, 0.4 +} + +func validate7z(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 32) + if err != nil || len(hdr) < 32 { + return 0, 0 + } + nextHdrOff := binary.LittleEndian.Uint64(hdr[12:20]) + nextHdrSz := binary.LittleEndian.Uint64(hdr[20:28]) + end := int64(32 + nextHdrOff + nextHdrSz) + if end <= 0 || start+end > max { + return 0, 0.5 + } + return end, 0.85 +} + +// validateRAR walks the block headers of a RAR4 or RAR5 archive to the +// end-of-archive marker. The two formats differ structurally, so the version +// is detected from the signature byte at offset 6. +func validateRAR(d *document.Document, start, max int64) (int64, float64) { + sig, err := d.Range(start, 8) + if err != nil || len(sig) < 8 { + return 0, 0 + } + if sig[6] == 0x01 { + return validateRAR5(d, start, max) + } + return validateRAR4(d, start, max) +} + +func validateRAR4(d *document.Document, start, max int64) (int64, float64) { + pos := start + 7 // 7-byte signature + for n := 0; n < 100000; n++ { + h, err := d.Range(pos, 11) + if err != nil || len(h) < 7 { + return 0, 0.4 + } + flags := binary.LittleEndian.Uint16(h[3:5]) + headSize := int64(binary.LittleEndian.Uint16(h[5:7])) + if headSize < 7 { + return 0, 0.4 + } + blockSize := headSize + if flags&0x8000 != 0 { // ADD_SIZE present + if len(h) < 11 { + return 0, 0.4 + } + blockSize += int64(binary.LittleEndian.Uint32(h[7:11])) + } + pos += blockSize + if h[2] == 0x7b { // terminator block + if pos > start && pos <= max { + return pos - start, 0.85 + } + return 0, 0.4 + } + if pos > max { + return 0, 0.4 + } + } + return 0, 0.4 +} + +func validateRAR5(d *document.Document, start, max int64) (int64, float64) { + pos := start + 8 // 8-byte signature + for n := 0; n < 100000; n++ { + // Each block: CRC32(4) + vint(header_size) + header_size bytes [+ data]. + h, err := d.Range(pos, 4+11) + if err != nil || len(h) < 5 { + return 0, 0.4 + } + headerSize, vlen := rar5Vint(h, 4) + if vlen == 0 || headerSize == 0 { + return 0, 0.4 + } + hdrStart := pos + 4 + int64(vlen) + body, err := d.Range(hdrStart, int64(headerSize)) + if err != nil || int64(len(body)) < int64(headerSize) { + return 0, 0.4 + } + htype, t1 := rar5Vint(body, 0) + hflags, t2 := rar5Vint(body, t1) + p := t1 + t2 + var dataSize uint64 + if hflags&0x0001 != 0 { // extra area present + _, te := rar5Vint(body, p) + p += te + } + if hflags&0x0002 != 0 { // data area present + dataSize, _ = rar5Vint(body, p) + } + pos = hdrStart + int64(headerSize) + int64(dataSize) + if htype == 5 { // end-of-archive header + if pos > start && pos <= max { + return pos - start, 0.85 + } + return 0, 0.4 + } + if pos > max { + return 0, 0.4 + } + } + return 0, 0.4 +} + +// rar5Vint decodes a RAR5 little-endian base-128 variable integer. +func rar5Vint(buf []byte, pos int) (uint64, int) { + var v uint64 + shift := uint(0) + for n := 0; pos+n < len(buf) && n < 10; n++ { + b := buf[pos+n] + v |= uint64(b&0x7f) << shift + if b&0x80 == 0 { + return v, n + 1 + } + shift += 7 + } + return 0, 0 +} + +func validateRIFF(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 12) + if err != nil || len(hdr) < 12 { + return 0, 0 + } + size := int64(binary.LittleEndian.Uint32(hdr[4:8])) + return size + 8, 0.9 +} + +func validateMachO(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 32) + if err != nil || len(hdr) < 28 { + return 0, 0 + } + var ( + bo binary.ByteOrder + is64 bool + hdrSize int64 = 28 + ) + switch binary.BigEndian.Uint32(hdr[0:4]) { + case 0xfeedface: + bo = binary.BigEndian + case 0xfeedfacf: + bo = binary.BigEndian + is64 = true + hdrSize = 32 + case 0xcefaedfe: + bo = binary.LittleEndian + case 0xcffaedfe: + bo = binary.LittleEndian + is64 = true + hdrSize = 32 + default: + return 0, 0 + } + if is64 && len(hdr) < 32 { + return 0, 0 + } + ncmds := bo.Uint32(hdr[16:20]) + sizeofcmds := bo.Uint32(hdr[20:24]) + if ncmds == 0 || ncmds > 10000 || sizeofcmds > 128<<20 { + return 0, 0 + } + cmdBytes, err := d.Range(start+hdrSize, int64(sizeofcmds)) + if err != nil || len(cmdBytes) < int(sizeofcmds) { + return 0, 0 + } + end := hdrSize + int64(sizeofcmds) + off := 0 + for i := uint32(0); i < ncmds; i++ { + if off+8 > len(cmdBytes) { + return 0, 0 + } + cmd := bo.Uint32(cmdBytes[off : off+4]) + cmdSize := int(bo.Uint32(cmdBytes[off+4 : off+8])) + if cmdSize < 8 || off+cmdSize > len(cmdBytes) { + return 0, 0 + } + switch cmd { + case 0x1: // LC_SEGMENT + if cmdSize >= 56 { + fileOff := int64(bo.Uint32(cmdBytes[off+32 : off+36])) + fileSize := int64(bo.Uint32(cmdBytes[off+36 : off+40])) + if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end { + end = fileOff + fileSize + } + } + case 0x19: // LC_SEGMENT_64 + if cmdSize >= 72 { + fileOff := int64(bo.Uint64(cmdBytes[off+40 : off+48])) + fileSize := int64(bo.Uint64(cmdBytes[off+48 : off+56])) + if fileOff >= 0 && fileSize >= 0 && fileOff+fileSize > end { + end = fileOff + fileSize + } + } + } + off += cmdSize + } + if end <= hdrSize || start+end > max { + return 0, 0 + } + return end, 0.85 +} + +func validateID3(d *document.Document, start, max int64) (int64, float64) { + // ID3v2 header: "ID3" + ver(2) + flags(1) + size(4, synchsafe). + hdr, err := d.Range(start, 10) + if err != nil || len(hdr) < 10 { + return 0, 0.3 + } + if hdr[3] == 0xff || hdr[4] == 0xff { + return 0, 0.3 + } + // Synchsafe 28-bit size (7 bits per byte). + sz := (int64(hdr[6]&0x7f) << 21) | (int64(hdr[7]&0x7f) << 14) | + (int64(hdr[8]&0x7f) << 7) | int64(hdr[9]&0x7f) + if sz < 0 || sz > 32*1024*1024 { + return 0, 0.4 + } + // Skip the ID3v2 tag (header + synchsafe body, plus a 10-byte footer if the + // footer-present flag is set), then walk MPEG audio frames to the end. + audioStart := start + 10 + sz + if hdr[5]&0x10 != 0 { + audioStart += 10 + } + if end := walkMP3Frames(d, audioStart, max); end > audioStart { + return end - start, 0.85 + } + return 0, 0.7 +} + +// walkMP3Frames follows MPEG-1/2/2.5 Layer I/II/III frame headers from start +// until a byte that is not a valid frame sync, returning the offset just past +// the last frame. APE/Lyrics3 trailers are not stripped — the audio extent is +// what matters for carving. +func walkMP3Frames(d *document.Document, start, max int64) int64 { + pos := start + frames := 0 + for pos+4 <= max && frames < 5_000_000 { + h, err := d.Range(pos, 4) + if err != nil || len(h) < 4 { + break + } + ln := mp3FrameLen(h) + if ln <= 0 { + break + } + pos += ln + frames++ + } + if frames == 0 { + return 0 + } + if pos > max { + return max + } + return pos +} + +var mp3BitrateV1 = [16]int{0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0} +var mp3BitrateV2 = [16]int{0, 8, 16, 24, 32, 40, 48, 56, 64, 80, 96, 112, 128, 144, 160, 0} +var mp3SampleRate = map[int][3]int{ // [version]{rate index 0,1,2} + 3: {44100, 48000, 32000}, // MPEG1 + 2: {22050, 24000, 16000}, // MPEG2 + 0: {11025, 12000, 8000}, // MPEG2.5 +} + +// mp3FrameLen returns the byte length of the MPEG audio frame whose 4-byte +// header is h, or 0 if h is not a valid frame header. +func mp3FrameLen(h []byte) int64 { + if h[0] != 0xff || h[1]&0xe0 != 0xe0 { + return 0 + } + version := int(h[1] >> 3 & 0x03) // 3=MPEG1, 2=MPEG2, 0=MPEG2.5 + layer := int(h[1] >> 1 & 0x03) // 3=I, 2=II, 1=III + if version == 1 || layer == 0 { + return 0 // reserved + } + brIndex := int(h[2] >> 4 & 0x0f) + srIndex := int(h[2] >> 2 & 0x03) + padding := int(h[2] >> 1 & 0x01) + if brIndex == 0 || brIndex == 15 || srIndex == 3 { + return 0 + } + var bitrate int + if version == 3 { + bitrate = mp3BitrateV1[brIndex] + } else { + bitrate = mp3BitrateV2[brIndex] + } + rates, ok := mp3SampleRate[version] + if !ok || bitrate == 0 { + return 0 + } + sampleRate := rates[srIndex] + if sampleRate == 0 { + return 0 + } + bps := bitrate * 1000 + if layer == 3 { // Layer I + return int64(((12 * bps / sampleRate) + padding) * 4) + } + // Layer II & III. MPEG2/2.5 Layer III uses 72 samples-per-byte factor. + coef := 144 + if version != 3 && layer == 1 { + coef = 72 + } + return int64(coef*bps/sampleRate + padding) +} + +func validateOgg(d *document.Document, start, max int64) (int64, float64) { + off := start + first := true + for off+27 <= max { + hdr, err := d.Range(off, 27) + if err != nil || len(hdr) < 27 || string(hdr[:4]) != "OggS" || hdr[4] != 0 { + return 0, 0 + } + headerType := hdr[5] + if first && headerType&0x02 == 0 { + return 0, 0.4 + } + segCount := int(hdr[26]) + lacing, err := d.Range(off+27, int64(segCount)) + if err != nil || len(lacing) < segCount { + return 0, 0 + } + payload := int64(0) + for _, v := range lacing { + payload += int64(v) + } + pageLen := int64(27 + segCount) + if payload < 0 || off+pageLen+payload > max { + return 0, 0 + } + off += pageLen + payload + if headerType&0x04 != 0 { + return off - start, 0.9 + } + first = false + } + return 0, 0.6 +} + +// validateMatroska reads the EBML header element and the following Segment +// element's size to bound a Matroska/WebM stream. A Segment with "unknown size" +// (all size bits set, as live-streamed files use) cannot be bounded this way. +func validateMatroska(d *document.Document, start, max int64) (int64, float64) { + buf, err := d.Range(start, min64(max-start, 4096)) + if err != nil || len(buf) < 8 { + return 0, 0 + } + // EBML header: ID 0x1A45DFA3, then a vint size. + if !(buf[0] == 0x1a && buf[1] == 0x45 && buf[2] == 0xdf && buf[3] == 0xa3) { + return 0, 0 + } + hsz, n, unknown := ebmlSize(buf, 4) + if n == 0 || unknown { + return 0, 0.6 + } + pos := 4 + n + int(hsz) // start of the Segment element + if pos+4 > len(buf) { + return 0, 0.6 + } + // Segment: ID 0x18538067, then a vint size. + if !(buf[pos] == 0x18 && buf[pos+1] == 0x53 && buf[pos+2] == 0x80 && buf[pos+3] == 0x67) { + return 0, 0.6 + } + segSz, sn, segUnknown := ebmlSize(buf, pos+4) + if sn == 0 || segUnknown { + return 0, 0.6 // live/streamed segment without a known size + } + end := int64(pos+4+sn) + segSz + if end <= 0 || start+end > max { + return 0, 0.6 + } + return end, 0.9 +} + +// ebmlSize decodes an EBML variable-length integer (data size form). The length +// is the number of leading zero bits in the first byte plus one; the value is +// the remaining bits. unknown is true when every value bit is set. +func ebmlSize(buf []byte, pos int) (val int64, n int, unknown bool) { + if pos >= len(buf) || buf[pos] == 0 { + return 0, 0, false + } + first := buf[pos] + mask := byte(0x80) + length := 1 + for first&mask == 0 { + mask >>= 1 + length++ + } + if length > 8 || pos+length > len(buf) { + return 0, 0, false + } + low := mask - 1 + v := int64(first & low) + allOnes := (first & low) == low + for i := 1; i < length; i++ { + b := buf[pos+i] + v = (v << 8) | int64(b) + if b != 0xff { + allOnes = false + } + } + return v, length, allOnes +} + +func validateFLAC(d *document.Document, start, max int64) (int64, float64) { + return 0, 0.8 +} + +// validateBMP reads the BITMAPFILEHEADER size field (bfSize), sanity-checking +// the embedded pixel-data offset against the declared length. +func validateBMP(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 14) + if err != nil || len(hdr) < 14 { + return 0, 0 + } + size := int64(binary.LittleEndian.Uint32(hdr[2:6])) + dataOff := int64(binary.LittleEndian.Uint32(hdr[10:14])) + if size < 26 || dataOff < 26 || dataOff > size || start+size > max { + return 0, 0 + } + return size, 0.8 +} + +// validateMP4 walks the top-level boxes of an ISO base-media file (MP4/MOV/M4A) +// from the file start (the ftyp box). 64-bit box sizes are honoured. +func validateMP4(d *document.Document, start, max int64) (int64, float64) { + pos := start + boxes := 0 + for pos+8 <= max && boxes < 100000 { + h, err := d.Range(pos, 16) + if err != nil || len(h) < 8 { + break + } + size := int64(binary.BigEndian.Uint32(h[0:4])) + switch { + case size == 1: // 64-bit largesize follows the type + if len(h) < 16 { + return 0, 0.4 + } + size = int64(binary.BigEndian.Uint64(h[8:16])) + case size == 0: // box extends to end of file + return max - start, 0.8 + } + if size < 8 { + break + } + pos += size + boxes++ + if pos > max { + return 0, 0.4 + } + } + if pos > start && pos <= max && boxes > 0 { + return pos - start, 0.85 + } + return 0, 0.4 +} + +func validateSQLite(d *document.Document, start, max int64) (int64, float64) { + hdr, err := d.Range(start, 32) + if err != nil || len(hdr) < 32 { + return 0, 0 + } + pageSize := int64(binary.BigEndian.Uint16(hdr[16:18])) + if pageSize == 1 { + pageSize = 65536 + } + pages := int64(binary.BigEndian.Uint32(hdr[28:32])) + if pages == 0 || pageSize == 0 { + return 0, 0.5 + } + return pageSize * pages, 0.9 +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/internal/leviathan/analysis/diff.go b/internal/leviathan/analysis/diff.go new file mode 100644 index 0000000..341798f --- /dev/null +++ b/internal/leviathan/analysis/diff.go @@ -0,0 +1,244 @@ +package analysis + +import ( + "context" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// DiffRegion describes a contiguous run of differing bytes. +type DiffRegion struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + // kinds: "edit" (both sides have bytes), "add" (only B), "remove" (only A) + Kind string `json:"kind"` +} + +type DiffBlock struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + ChangedBytes int64 `json:"changedBytes"` + EditBytes int64 `json:"editBytes"` + AddBytes int64 `json:"addBytes"` + RemoveBytes int64 `json:"removeBytes"` + Similarity float64 `json:"similarity"` +} + +// Diff compares two documents by byte. It handles documents of +// different sizes by treating the tail of the longer as add/remove. +// Chunks at both sides. Returns a compact list of regions. +func Diff(a, b *document.Document, start, length int64) (map[string]any, error) { + return DiffContext(context.Background(), a, b, start, length) +} + +func DiffContext(ctx context.Context, a, b *document.Document, start, length int64) (map[string]any, error) { + aSize := a.Size() + bSize := b.Size() + maxSize := aSize + if bSize > maxSize { + maxSize = bSize + } + if length < 0 { + length = maxSize - start + } + end := start + length + if end > maxSize { + end = maxSize + } + + const chunk = 1 << 20 + regions := make([]DiffRegion, 0, 64) + + var inRun bool + var runStart int64 + var runKind string + + flush := func(off int64) { + if inRun { + regions = append(regions, DiffRegion{Offset: runStart, Length: off - runStart, Kind: runKind}) + inRun = false + } + } + + for off := start; off < end; { + if err := ctx.Err(); err != nil { + return nil, err + } + winEnd := off + chunk + if winEnd > end { + winEnd = end + } + // Read both sides, tolerating short reads for the shorter file. + var ab, bb []byte + if off < aSize { + n := winEnd - off + if off+n > aSize { + n = aSize - off + } + var err error + ab, err = a.Range(off, n) + if err != nil { + return nil, err + } + } + if off < bSize { + n := winEnd - off + if off+n > bSize { + n = bSize - off + } + var err error + bb, err = b.Range(off, n) + if err != nil { + return nil, err + } + } + for i := int64(0); i < winEnd-off; i++ { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return nil, err + } + } + curOff := off + i + var av, bv byte + var av_ok, bv_ok bool + if curOff < aSize && int64(len(ab)) > i { + av = ab[i] + av_ok = true + } + if curOff < bSize && int64(len(bb)) > i { + bv = bb[i] + bv_ok = true + } + kind := "" + if av_ok && bv_ok { + if av != bv { + kind = "edit" + } + } else if av_ok && !bv_ok { + kind = "remove" + } else if !av_ok && bv_ok { + kind = "add" + } + if kind == "" { + flush(curOff) + continue + } + if !inRun { + inRun = true + runStart = curOff + runKind = kind + } else if kind != runKind { + flush(curOff) + inRun = true + runStart = curOff + runKind = kind + } + } + off = winEnd + } + flush(end) + + // Convert to JS-friendly output + summary + out := make([]map[string]any, len(regions)) + editBytes := int64(0) + addBytes := int64(0) + removeBytes := int64(0) + for i, r := range regions { + out[i] = map[string]any{ + "offset": r.Offset, + "length": r.Length, + "kind": r.Kind, + } + switch r.Kind { + case "edit": + editBytes += r.Length + case "add": + addBytes += r.Length + case "remove": + removeBytes += r.Length + } + } + const blockSize int64 = 4096 + blocks := diffBlocksFromRegions(regions, start, end, blockSize) + blockOut := make([]map[string]any, len(blocks)) + for i, b := range blocks { + blockOut[i] = map[string]any{ + "offset": b.Offset, + "length": b.Length, + "changedBytes": b.ChangedBytes, + "editBytes": b.EditBytes, + "addBytes": b.AddBytes, + "removeBytes": b.RemoveBytes, + "similarity": b.Similarity, + } + } + return map[string]any{ + "regions": out, + "regionCount": len(regions), + "blocks": blockOut, + "blockCount": len(blocks), + "blockSize": blockSize, + "editBytes": editBytes, + "addBytes": addBytes, + "removeBytes": removeBytes, + "aSize": aSize, + "bSize": bSize, + }, nil +} + +func diffBlocksFromRegions(regions []DiffRegion, start, end, blockSize int64) []DiffBlock { + if blockSize <= 0 || end <= start || len(regions) == 0 { + return nil + } + out := []DiffBlock{} + for alignedStart := start - start%blockSize; alignedStart < end; alignedStart += blockSize { + blockStart := alignedStart + blockEnd := alignedStart + blockSize + if blockEnd > end { + blockEnd = end + } + if blockEnd <= start { + continue + } + if blockStart < start { + blockStart = start + } + b := DiffBlock{Offset: blockStart, Length: blockEnd - blockStart} + for _, r := range regions { + rEnd := r.Offset + r.Length + if rEnd <= blockStart { + continue + } + if r.Offset >= blockEnd { + break + } + ovStart := r.Offset + if ovStart < blockStart { + ovStart = blockStart + } + ovEnd := rEnd + if ovEnd > blockEnd { + ovEnd = blockEnd + } + changed := ovEnd - ovStart + if changed <= 0 { + continue + } + b.ChangedBytes += changed + switch r.Kind { + case "edit": + b.EditBytes += changed + case "add": + b.AddBytes += changed + case "remove": + b.RemoveBytes += changed + } + } + if b.ChangedBytes == 0 { + continue + } + b.Similarity = 1 - float64(b.ChangedBytes)/float64(b.Length) + out = append(out, b) + } + return out +} diff --git a/internal/leviathan/analysis/entropy.go b/internal/leviathan/analysis/entropy.go new file mode 100644 index 0000000..590bd9a --- /dev/null +++ b/internal/leviathan/analysis/entropy.go @@ -0,0 +1,125 @@ +package analysis + +import ( + "context" + "math" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Entropy computes Shannon entropy per sliding window over the range. +// +// The returned map has: +// +// windowSize: int +// step: int +// samples: []float64 — entropy per window (0..8) +// offsets: []int64 — start offset of each window +// mean/min/max: summary stats +// highCount: number of windows with entropy >= 7.5 (candidate compressed/encrypted) +func Entropy(d *document.Document, start, length, windowSize, step int64) (map[string]any, error) { + return EntropyContext(context.Background(), d, start, length, windowSize, step) +} + +func EntropyContext(ctx context.Context, d *document.Document, start, length, windowSize, step int64) (map[string]any, error) { + if windowSize <= 0 { + windowSize = 4096 + } + if step <= 0 { + step = windowSize + } + if length < 0 { + length = d.Size() - start + } + if length <= 0 { + return map[string]any{ + "windowSize": windowSize, "step": step, + "samples": []float64{}, "offsets": []int64{}, + "mean": 0.0, "min": 0.0, "max": 0.0, "highCount": 0, + }, nil + } + + // We run in a streaming fashion: advance through chunks, refill + // a rolling window, and emit a sample whenever the cursor hits + // a step boundary. + samples := make([]float64, 0, length/step+1) + offsets := make([]int64, 0, length/step+1) + + // For bounded memory we read the document range chunk by chunk + // but we still need a window of size `windowSize`. Windows can be + // as small as 256 or as large as 1MB; we cap to length. + if windowSize > length { + windowSize = length + } + + // Simple approach: iterate in step-sized hops, reading window + // bytes at each hop. This scales O(n * (w/step)) but window and + // step are usually the same for UI. + mean := 0.0 + minE := math.Inf(1) + maxE := math.Inf(-1) + high := 0 + + for off := start; off < start+length; off += step { + if err := ctx.Err(); err != nil { + return nil, err + } + winLen := windowSize + if off+winLen > start+length { + winLen = start + length - off + } + buf, err := d.Range(off, winLen) + if err != nil { + return nil, err + } + e := shannon(buf) + samples = append(samples, e) + offsets = append(offsets, off) + mean += e + if e < minE { + minE = e + } + if e > maxE { + maxE = e + } + if e >= 7.5 { + high++ + } + } + if len(samples) > 0 { + mean /= float64(len(samples)) + } else { + minE, maxE = 0, 0 + } + + return map[string]any{ + "windowSize": windowSize, + "step": step, + "samples": samples, + "offsets": offsets, + "mean": mean, + "min": minE, + "max": maxE, + "highCount": high, + }, nil +} + +func shannon(buf []byte) float64 { + if len(buf) == 0 { + return 0 + } + var counts [256]int + for _, b := range buf { + counts[b]++ + } + n := float64(len(buf)) + e := 0.0 + for _, c := range counts { + if c == 0 { + continue + } + p := float64(c) / n + e -= p * math.Log2(p) + } + return e +} diff --git a/internal/leviathan/analysis/fuzzy.go b/internal/leviathan/analysis/fuzzy.go new file mode 100644 index 0000000..3f29eef --- /dev/null +++ b/internal/leviathan/analysis/fuzzy.go @@ -0,0 +1,224 @@ +package analysis + +import ( + "fmt" + "strconv" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// fuzzy.go implements context-triggered piecewise hashing (CTPH) compatible +// with ssdeep. CTPH produces a short signature of the form +// "blocksize:hash1:hash2" that stays similar when the input changes only a +// little, so investigators can cluster related samples or match a file +// against a known-bad ssdeep hash even after minor edits. +// +// The signature format and the rolling/sum hashes follow Andrew Tridgell's +// spamsum / Jesse Kornblum's ssdeep so the output interoperates with the +// standard tooling (e.g. an empty input hashes to "3::"). + +const ( + spamsumLength = 64 + rollingWindow = 7 + minBlocksize = 3 + hashInit = uint32(0x28021967) + hashPrime = uint32(0x01000193) +) + +const b64alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/" + +type rollState struct { + h1, h2, h3 uint32 + window [rollingWindow]byte + n int +} + +func (r *rollState) roll(c byte) uint32 { + r.h2 -= r.h1 + r.h2 += rollingWindow * uint32(c) + r.h1 += uint32(c) + r.h1 -= uint32(r.window[r.n%rollingWindow]) + r.window[r.n%rollingWindow] = c + r.n++ + r.h3 = r.h3 << 5 + r.h3 ^= uint32(c) + return r.h1 + r.h2 + r.h3 +} + +func sumHash(c byte, h uint32) uint32 { return (h * hashPrime) ^ uint32(c) } + +// FuzzyDigest computes the ssdeep-compatible signature for data. +func FuzzyDigest(data []byte) string { + bs := uint32(minBlocksize) + for uint64(bs)*spamsumLength < uint64(len(data)) { + bs *= 2 + } + for { + sig1, sig2 := ctphSigs(data, bs) + if bs > minBlocksize && len(sig1) < spamsumLength/2 { + bs /= 2 + continue + } + return fmt.Sprintf("%d:%s:%s", bs, sig1, sig2) + } +} + +func ctphSigs(data []byte, bs uint32) (string, string) { + var rs rollState + h1, h2 := hashInit, hashInit + var sig1, sig2 strings.Builder + var lastRoll uint32 + + for _, c := range data { + h1 = sumHash(c, h1) + h2 = sumHash(c, h2) + rh := rs.roll(c) + lastRoll = rh + + if rh%bs == bs-1 && sig1.Len() < spamsumLength-1 { + sig1.WriteByte(b64alphabet[h1%64]) + h1 = hashInit + } + if rh%(bs*2) == bs*2-1 && sig2.Len() < spamsumLength/2-1 { + sig2.WriteByte(b64alphabet[h2%64]) + h2 = hashInit + } + } + if lastRoll != 0 { + sig1.WriteByte(b64alphabet[h1%64]) + sig2.WriteByte(b64alphabet[h2%64]) + } + return sig1.String(), sig2.String() +} + +// FuzzyHash computes the CTPH signature over a document range. +func FuzzyHash(d *document.Document, start, length int64) (map[string]any, error) { + if length < 0 { + length = d.Size() - start + } + buf, err := d.Range(start, length) + if err != nil { + return nil, err + } + sig := FuzzyDigest(buf) + bs := strings.SplitN(sig, ":", 2)[0] + return map[string]any{ + "algo": "ssdeep", + "signature": sig, + "blocksize": bs, + "length": length, + }, nil +} + +// FuzzyCompare returns a 0-100 similarity score between two ssdeep +// signatures. 0 means unrelated (or incomparable block sizes); 100 means the +// signatures are identical. +func FuzzyCompare(a, b string) int { + bs1, a1, a2, ok1 := parseSig(a) + bs2, b1, b2, ok2 := parseSig(b) + if !ok1 || !ok2 { + return 0 + } + if bs1 != bs2 && bs1 != bs2*2 && bs2 != bs1*2 { + return 0 + } + switch { + case bs1 == bs2: + s1 := scoreStrings(a1, b1, bs1) + s2 := scoreStrings(a2, b2, bs1*2) + if s2 > s1 { + return s2 + } + return s1 + case bs1 == bs2*2: + return scoreStrings(a1, b2, bs1) + default: // bs2 == bs1*2 + return scoreStrings(a2, b1, bs2) + } +} + +func parseSig(s string) (uint32, string, string, bool) { + parts := strings.SplitN(s, ":", 3) + if len(parts) != 3 { + return 0, "", "", false + } + bs, err := strconv.ParseUint(parts[0], 10, 32) + if err != nil || bs == 0 { + return 0, "", "", false + } + return uint32(bs), parts[1], parts[2], true +} + +func scoreStrings(s1, s2 string, blocksize uint32) int { + if !hasCommonSubstring(s1, s2, rollingWindow) { + return 0 + } + if s1 == s2 { + return 100 + } + d := editDistance(s1, s2) + if len(s1)+len(s2) == 0 { + return 0 + } + score := (d * spamsumLength) / (len(s1) + len(s2)) + score = (100 * score) / spamsumLength + score = 100 - score + // Small block sizes describe little data, so cap the score they can claim. + cap := int(blocksize) / minBlocksize * minInt(len(s1), len(s2)) + if score > cap { + score = cap + } + if score > 100 { + score = 100 + } + if score < 0 { + score = 0 + } + return score +} + +func hasCommonSubstring(s1, s2 string, n int) bool { + if len(s1) < n || len(s2) < n { + return false + } + seen := make(map[string]struct{}, len(s1)) + for i := 0; i+n <= len(s1); i++ { + seen[s1[i:i+n]] = struct{}{} + } + for i := 0; i+n <= len(s2); i++ { + if _, ok := seen[s2[i:i+n]]; ok { + return true + } + } + return false +} + +// editDistance is classic Levenshtein with unit insert/delete/substitute cost. +func editDistance(s1, s2 string) int { + m, n := len(s1), len(s2) + prev := make([]int, n+1) + cur := make([]int, n+1) + for j := 0; j <= n; j++ { + prev[j] = j + } + for i := 1; i <= m; i++ { + cur[0] = i + for j := 1; j <= n; j++ { + cost := 1 + if s1[i-1] == s2[j-1] { + cost = 0 + } + cur[j] = minInt(minInt(cur[j-1]+1, prev[j]+1), prev[j-1]+cost) + } + prev, cur = cur, prev + } + return prev[n] +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} diff --git a/internal/leviathan/analysis/hash.go b/internal/leviathan/analysis/hash.go new file mode 100644 index 0000000..39b9638 --- /dev/null +++ b/internal/leviathan/analysis/hash.go @@ -0,0 +1,63 @@ +package analysis + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/sha512" + "encoding/hex" + "hash" + "hash/crc32" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// HashSet contains CRC32/MD5/SHA-1/SHA-256/SHA-512 for a range. +type HashSet struct { + Length int64 `json:"length"` + CRC32 string `json:"crc32"` + MD5 string `json:"md5"` + SHA1 string `json:"sha1"` + SHA256 string `json:"sha256"` + SHA512 string `json:"sha512"` +} + +// Hashes computes all supported hashes over the document range. +// The computation runs chunk-by-chunk so large files do not force a +// full-buffer duplication. +func Hashes(d *document.Document, start, length int64) (map[string]any, error) { + if length < 0 { + length = d.Size() - start + } + + crc := crc32.NewIEEE() + mh := md5.New() + s1 := sha1.New() + s256 := sha256.New() + s512 := sha512.New() + + writers := []hash.Hash{mh, s1, s256, s512} + + total := int64(0) + err := d.ForEachChunk(start, length, func(_ int64, chunk []byte) error { + crc.Write(chunk) + for _, w := range writers { + w.Write(chunk) + } + total += int64(len(chunk)) + return nil + }) + if err != nil { + return nil, err + } + return map[string]any{ + "length": total, + "crc32": toHex(crc.Sum(nil)), + "md5": toHex(mh.Sum(nil)), + "sha1": toHex(s1.Sum(nil)), + "sha256": toHex(s256.Sum(nil)), + "sha512": toHex(s512.Sum(nil)), + }, nil +} + +func toHex(b []byte) string { return hex.EncodeToString(b) } diff --git a/internal/leviathan/analysis/inspector.go b/internal/leviathan/analysis/inspector.go new file mode 100644 index 0000000..298aa83 --- /dev/null +++ b/internal/leviathan/analysis/inspector.go @@ -0,0 +1,162 @@ +package analysis + +import ( + "encoding/binary" + "fmt" + "math" + "time" + "unicode/utf16" + "unicode/utf8" +) + +// Inspect interprets bytes starting at `buf[0]` as many candidate +// types. It is resilient: types that don't have enough bytes simply +// return empty strings. +func Inspect(buf []byte, offset int64) map[string]any { + out := map[string]any{ + "offset": offset, + "length": len(buf), + } + + // 8-bit + if len(buf) >= 1 { + out["u8"] = int(buf[0]) + out["i8"] = int(int8(buf[0])) + out["char"] = printable(buf[0]) + out["bin8"] = fmt.Sprintf("%08b", buf[0]) + } + + // 16-bit + if len(buf) >= 2 { + le16 := binary.LittleEndian.Uint16(buf) + be16 := binary.BigEndian.Uint16(buf) + out["u16_le"] = int(le16) + out["u16_be"] = int(be16) + out["i16_le"] = int(int16(le16)) + out["i16_be"] = int(int16(be16)) + } + + // 32-bit + if len(buf) >= 4 { + le32 := binary.LittleEndian.Uint32(buf) + be32 := binary.BigEndian.Uint32(buf) + out["u32_le"] = uint(le32) + out["u32_be"] = uint(be32) + out["i32_le"] = int(int32(le32)) + out["i32_be"] = int(int32(be32)) + out["f32_le"] = float64(math.Float32frombits(le32)) + out["f32_be"] = float64(math.Float32frombits(be32)) + } + + // 64-bit + if len(buf) >= 8 { + le64 := binary.LittleEndian.Uint64(buf) + be64 := binary.BigEndian.Uint64(buf) + out["u64_le"] = float64(le64) + out["u64_be"] = float64(be64) + out["i64_le"] = float64(int64(le64)) + out["i64_be"] = float64(int64(be64)) + out["f64_le"] = math.Float64frombits(le64) + out["f64_be"] = math.Float64frombits(be64) + } + + // Timestamp guesses + if len(buf) >= 4 { + le := binary.LittleEndian.Uint32(buf) + out["unix_le"] = formatUnix(int64(le)) + } + if len(buf) >= 8 { + le := binary.LittleEndian.Uint64(buf) + out["filetime_le"] = formatFiletime(le) + out["webkit_le"] = formatWebkit(le) + out["oletime_le"] = formatOLEDate(math.Float64frombits(le)) + } + + // ASCII preview (up to 32 bytes) + n := len(buf) + if n > 32 { + n = 32 + } + ascii := make([]byte, n) + for i := 0; i < n; i++ { + if buf[i] >= 0x20 && buf[i] <= 0x7e { + ascii[i] = buf[i] + } else { + ascii[i] = '.' + } + } + out["ascii"] = string(ascii) + + // UTF-8 attempt + if utf8.Valid(buf[:n]) { + out["utf8"] = string(buf[:n]) + } + + // UTF-16 LE/BE preview + if len(buf) >= 2 { + lu := n / 2 + if lu > 0 { + u := make([]uint16, lu) + for i := 0; i < lu; i++ { + u[i] = uint16(buf[i*2]) | uint16(buf[i*2+1])<<8 + } + out["utf16le"] = string(utf16.Decode(u)) + for i := 0; i < lu; i++ { + u[i] = uint16(buf[i*2])<<8 | uint16(buf[i*2+1]) + } + out["utf16be"] = string(utf16.Decode(u)) + } + } + + return out +} + +func printable(b byte) string { + if b >= 0x20 && b <= 0x7e { + return string([]byte{b}) + } + return "." +} + +func formatUnix(sec int64) string { + if sec < 0 || sec > 4102444800 { // past 2100 + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +// Windows FILETIME epoch 1601-01-01 in 100ns ticks. +const filetimeEpochDelta = 11644473600 + +func formatFiletime(v uint64) string { + if v == 0 { + return "" + } + sec := int64(v/10_000_000) - filetimeEpochDelta + if sec < 0 || sec > 4102444800 { + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +// WebKit/Chrome epoch 1601-01-01 in microseconds. +func formatWebkit(v uint64) string { + if v == 0 { + return "" + } + sec := int64(v/1_000_000) - filetimeEpochDelta + if sec < 0 || sec > 4102444800 { + return "" + } + return time.Unix(sec, 0).UTC().Format(time.RFC3339) +} + +// OLE Automation date: days since 1899-12-30, as float64. +func formatOLEDate(days float64) string { + if days < 2 || days > 75000 { + return "" + } + base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC) + sec := int64(days * 86400) + return base.Add(time.Duration(sec) * time.Second).Format(time.RFC3339) +} diff --git a/internal/leviathan/analysis/ioc.go b/internal/leviathan/analysis/ioc.go new file mode 100644 index 0000000..d1d01ea --- /dev/null +++ b/internal/leviathan/analysis/ioc.go @@ -0,0 +1,89 @@ +package analysis + +import ( + "context" + "regexp" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ioc regexes. They run against printable ASCII strings extracted +// from the document, which is the standard approach for IOC hunting +// and avoids matching noise in raw binary data. +var iocRegexes = map[string]*regexp.Regexp{ + "url": regexp.MustCompile(`(?i)\b(?:https?|ftp)://[A-Za-z0-9\-._~:/?#\[\]@!$&'()*+,;=%]+`), + "domain": regexp.MustCompile(`(?i)\b(?:[a-z0-9](?:[a-z0-9\-]{0,61}[a-z0-9])?\.)+(?:com|net|org|io|co|uk|de|fr|ru|cn|info|xyz|app|dev|sh|me|gov|mil|edu|biz)\b`), + "email": regexp.MustCompile(`(?i)\b[A-Z0-9._%+\-]+@[A-Z0-9.\-]+\.[A-Z]{2,}\b`), + "ipv4": regexp.MustCompile(`\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b`), + "ipv6": regexp.MustCompile(`\b(?:[0-9a-fA-F]{1,4}:){7}[0-9a-fA-F]{1,4}\b`), + "guid": regexp.MustCompile(`\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b`), + "md5": regexp.MustCompile(`\b[a-fA-F0-9]{32}\b`), + "sha1": regexp.MustCompile(`\b[a-fA-F0-9]{40}\b`), + "sha256": regexp.MustCompile(`\b[a-fA-F0-9]{64}\b`), + "winpath": regexp.MustCompile(`[A-Za-z]:\\(?:[^\\/:*?"<>|\r\n]+\\)*[^\\/:*?"<>|\r\n]*`), + "unc": regexp.MustCompile(`\\\\[A-Za-z0-9._\-]+(?:\\[^\\/:*?"<>|\r\n]+)+`), + "unixpath": regexp.MustCompile(`(?:/(?:etc|var|usr|opt|home|tmp|root|srv|bin|sbin|lib|lib64)(?:/[A-Za-z0-9._\-]+)+)`), + "regkey": regexp.MustCompile(`(?i)\b(?:HKLM|HKCU|HKCR|HKU|HKCC|HKEY_[A-Z_]+)\\[^\s"<>]+`), + "onion": regexp.MustCompile(`(?i)\b[a-z2-7]{16,56}\.onion\b`), + "btc": regexp.MustCompile(`\b(?:[13][a-km-zA-HJ-NP-Z1-9]{25,34}|bc1[a-z0-9]{20,90})\b`), +} + +// ExtractIOCs runs the IOC regexes over ASCII strings first and then +// UTF-16 LE/BE strings. Returns a map of category -> slice of hits +// with offsets into the original file. +func ExtractIOCs(d *document.Document, start, length int64, minLen int) (map[string]any, error) { + return ExtractIOCsContext(context.Background(), d, start, length, minLen) +} + +func ExtractIOCsContext(ctx context.Context, d *document.Document, start, length int64, minLen int) (map[string]any, error) { + if length < 0 { + length = d.Size() - start + } + if minLen <= 0 { + minLen = 5 + } + + strings, err := ExtractStringsContext(ctx, d, start, length, minLen, []string{"ascii", "utf16le", "utf16be"}, 200_000) + if err != nil { + return nil, err + } + + result := map[string]any{} + totals := map[string]int{} + for category, re := range iocRegexes { + if err := ctx.Err(); err != nil { + return nil, err + } + hits := make([]map[string]any, 0) + for _, s := range strings { + if err := ctx.Err(); err != nil { + return nil, err + } + str, _ := s["value"].(string) + base, _ := s["offset"].(int64) + for _, loc := range re.FindAllStringIndex(str, -1) { + hits = append(hits, map[string]any{ + "category": category, + "offset": base + int64(loc[0]), + "length": loc[1] - loc[0], + "value": str[loc[0]:loc[1]], + }) + } + } + result[category] = hits + totals[category] = len(hits) + } + // flat list for UI convenience + all := []map[string]any{} + for _, v := range result { + if s, ok := v.([]map[string]any); ok { + all = append(all, s...) + } + } + return map[string]any{ + "byCategory": result, + "all": all, + "totals": totals, + "totalStrings": len(strings), + }, nil +} diff --git a/internal/leviathan/analysis/keys.go b/internal/leviathan/analysis/keys.go new file mode 100644 index 0000000..fa9c426 --- /dev/null +++ b/internal/leviathan/analysis/keys.go @@ -0,0 +1,367 @@ +package analysis + +import ( + "bytes" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "fmt" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// KeyArtefact is a single key-ish object we've found in the raw file. +// Length is the span we recognised; Note carries any decoding detail. +type KeyArtefact struct { + Kind string `json:"kind"` + Format string `json:"format"` // "pem" | "der" | "ssh" | "p12" + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Summary string `json:"summary"` + Note string `json:"note,omitempty"` +} + +// ScanKeys walks the whole document looking for cryptographic artefacts: +// - PEM blocks with common key-related labels +// - OpenSSH keys (authorized_keys lines and the "openssh-key-v1" private-key envelope) +// - PKCS#12 magic (1.2.840.113549.1.12.1 appears in headers) +// - Loose DER SEQUENCEs that parse as RSA/EC/PKCS#8 keys +// +// We cap the scan at 64 MiB. Users with larger blobs can select a range +// and call this against that subset via a future "keys.scanRange" bridge. +func ScanKeys(d *document.Document) ([]KeyArtefact, error) { + size := d.Size() + if size > 64*1024*1024 { + return nil, fmt.Errorf("keys.scan: file > 64 MiB; use selection") + } + buf, err := d.Range(0, size) + if err != nil { + return nil, err + } + out := []KeyArtefact{} + out = append(out, scanPEMKeys(buf)...) + out = append(out, scanOpenSSHPubKeys(buf)...) + out = append(out, scanOpenSSHPrivKeys(buf)...) + out = append(out, scanPKCS12(buf)...) + // Loose DER pass — for raw PKCS#8 / PKCS#1 blobs that aren't wrapped in PEM. + out = append(out, scanLooseDERKeys(buf)...) + return out, nil +} + +// Known PEM labels that indicate key-ish content. +var pemKeyLabels = map[string]string{ + "RSA PRIVATE KEY": "rsa-private", + "RSA PUBLIC KEY": "rsa-public", + "EC PRIVATE KEY": "ec-private", + "EC PUBLIC KEY": "ec-public", + "DSA PRIVATE KEY": "dsa-private", + "PRIVATE KEY": "pkcs8-private", + "ENCRYPTED PRIVATE KEY": "pkcs8-encrypted", + "PUBLIC KEY": "pkix-public", + "OPENSSH PRIVATE KEY": "openssh-private", + "SSH2 PUBLIC KEY": "ssh2-public", + "PGP PRIVATE KEY BLOCK": "pgp-private", + "PGP PUBLIC KEY BLOCK": "pgp-public", + "CERTIFICATE REQUEST": "csr", + "NEW CERTIFICATE REQUEST": "csr", + "DH PARAMETERS": "dh-params", + "PARAMETERS": "params", +} + +func scanPEMKeys(buf []byte) []KeyArtefact { + out := []KeyArtefact{} + rest := buf + origLen := int64(len(buf)) + for { + block, next := pem.Decode(rest) + if block == nil { + break + } + beginIdx := bytes.Index(rest, []byte("-----BEGIN")) + if beginIdx < 0 { + break + } + start := origLen - int64(len(rest)) + int64(beginIdx) + end := origLen - int64(len(next)) + kind, match := pemKeyLabels[block.Type] + if !match { + rest = next + continue + } + art := KeyArtefact{ + Kind: kind, Format: "pem", + Offset: start, Length: end - start, + } + art.Summary = describePEMBlock(block) + out = append(out, art) + rest = next + } + return out +} + +func describePEMBlock(b *pem.Block) string { + name := b.Type + switch b.Type { + case "RSA PRIVATE KEY": + if k, err := x509.ParsePKCS1PrivateKey(b.Bytes); err == nil { + return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen()) + } + case "RSA PUBLIC KEY": + if k, err := x509.ParsePKCS1PublicKey(b.Bytes); err == nil { + return fmt.Sprintf("%s — RSA %d-bit", name, k.N.BitLen()) + } + case "EC PRIVATE KEY": + if k, err := x509.ParseECPrivateKey(b.Bytes); err == nil { + return fmt.Sprintf("%s — curve %s", name, k.Curve.Params().Name) + } + case "PRIVATE KEY": + if k, err := x509.ParsePKCS8PrivateKey(b.Bytes); err == nil { + return fmt.Sprintf("%s — %s", name, describePrivateKey(k)) + } + case "PUBLIC KEY": + if k, err := x509.ParsePKIXPublicKey(b.Bytes); err == nil { + return fmt.Sprintf("%s — %s", name, describePublicKey(k)) + } + case "OPENSSH PRIVATE KEY": + return fmt.Sprintf("%s (%d B payload)", name, len(b.Bytes)) + case "ENCRYPTED PRIVATE KEY": + return fmt.Sprintf("%s — PKCS#8 encrypted (%d B)", name, len(b.Bytes)) + } + return fmt.Sprintf("%s (%d B)", name, len(b.Bytes)) +} + +func describePrivateKey(k any) string { + switch key := k.(type) { + case *rsa.PrivateKey: + return fmt.Sprintf("RSA %d-bit", key.N.BitLen()) + case *ecdsa.PrivateKey: + return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name) + case ed25519.PrivateKey: + return "Ed25519" + } + return fmt.Sprintf("%T", k) +} + +func describePublicKey(k any) string { + switch key := k.(type) { + case *rsa.PublicKey: + return fmt.Sprintf("RSA %d-bit", key.N.BitLen()) + case *ecdsa.PublicKey: + return fmt.Sprintf("ECDSA %s", key.Curve.Params().Name) + case ed25519.PublicKey: + return "Ed25519" + } + return fmt.Sprintf("%T", k) +} + +// OpenSSH authorized_keys / id_*.pub format is one line per key: +// " [comment]\n". We search for well-known algos and +// delimit by the next newline. False positives on text logs are possible, +// so we validate by base64-decoding. +var sshAlgos = []string{ + "ssh-rsa", "ssh-dss", "ssh-ed25519", + "ecdsa-sha2-nistp256", "ecdsa-sha2-nistp384", "ecdsa-sha2-nistp521", + "sk-ssh-ed25519@openssh.com", "sk-ecdsa-sha2-nistp256@openssh.com", +} + +func scanOpenSSHPubKeys(buf []byte) []KeyArtefact { + out := []KeyArtefact{} + for _, algo := range sshAlgos { + needle := []byte(algo + " ") + off := 0 + for { + idx := bytes.Index(buf[off:], needle) + if idx < 0 { + break + } + absStart := off + idx + // Must be at start of line or after whitespace. + if absStart > 0 { + prev := buf[absStart-1] + if prev != '\n' && prev != '\r' && prev != ' ' && prev != '\t' { + off = absStart + 1 + continue + } + } + // End of line. + endOfLine := absStart + len(needle) + for endOfLine < len(buf) && buf[endOfLine] != '\n' && buf[endOfLine] != '\r' { + endOfLine++ + } + line := buf[absStart:endOfLine] + // Expect at least one space-separated payload that decodes. + parts := bytes.SplitN(line, []byte(" "), 3) + if len(parts) < 2 || len(parts[1]) < 16 || !isBase64Like(parts[1]) { + off = absStart + 1 + continue + } + art := KeyArtefact{ + Kind: "openssh-public", Format: "ssh", + Offset: int64(absStart), Length: int64(len(line)), + Summary: fmt.Sprintf("%s (%d-byte payload b64)", algo, len(parts[1])), + } + if len(parts) == 3 && len(parts[2]) > 0 { + art.Note = "comment: " + string(parts[2]) + } + out = append(out, art) + off = endOfLine + } + } + return out +} + +func scanOpenSSHPrivKeys(buf []byte) []KeyArtefact { + // Already caught by PEM scan (label "OPENSSH PRIVATE KEY"). We also + // look for the magic string inside the PEM body — "openssh-key-v1\0". + out := []KeyArtefact{} + needle := []byte("openssh-key-v1\x00") + off := 0 + for { + idx := bytes.Index(buf[off:], needle) + if idx < 0 { + break + } + absStart := off + idx + out = append(out, KeyArtefact{ + Kind: "openssh-private-magic", Format: "ssh", + Offset: int64(absStart), Length: int64(len(needle)), + Summary: "OpenSSH v1 private-key magic", + Note: "magic only; no envelope length inferred", + }) + off = absStart + len(needle) + } + return out +} + +// PKCS#12 (.p12 / .pfx) files begin with an ASN.1 SEQUENCE; the second +// element is an OID for the bag type. Its encoded form is reasonably +// distinctive: "\x06\x0b\x2a\x86\x48\x86\xf7\x0d\x01\x07\x01" (data). +// The simplest reliable magic for our purposes is the OID for +// pkcs-12 (1.2.840.113549.1.12.1): "\x06\x0a\x2a\x86\x48\x86\xf7\x0d\x01\x0c\x01". +func scanPKCS12(buf []byte) []KeyArtefact { + out := []KeyArtefact{} + needle := []byte{0x06, 0x0a, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x01, 0x0c, 0x01} + off := 0 + for { + idx := bytes.Index(buf[off:], needle) + if idx < 0 { + break + } + absStart := off + idx + out = append(out, KeyArtefact{ + Kind: "pkcs12-oid", Format: "p12", + Offset: int64(absStart), Length: int64(len(needle)), + Summary: "pkcs-12 OID (1.2.840.113549.1.12.1)", + Note: "bag-type marker; may indicate a .p12 or .pfx container", + }) + off = absStart + len(needle) + } + return out +} + +// scanLooseDERKeys walks DER SEQUENCE headers and tries to parse each +// candidate as a known key format. False negatives are fine; we only +// emit an artefact on a positive parse. +func scanLooseDERKeys(buf []byte) []KeyArtefact { + out := []KeyArtefact{} + // Use the same SEQUENCE 0x30 0x82 xx yy (long-form, 2 length bytes) + // pattern as the cert scanner. Minimum useful key length ~200 bytes. + // + // Each candidate runs up to six x509 parse attempts over a blob up to + // 64 KiB, so an unfiltered scan of a buffer dense with "30 82" pairs is + // O(n) expensive parses and could freeze the UI. A cheap structural + // pre-filter (a DER key body begins with a SEQUENCE 0x30 or an INTEGER + // 0x02) rejects almost all false candidates without parsing, and a hard + // cap bounds adversarial input that defeats the pre-filter. + const maxDERAttempts = 1 << 16 + derAttempts := 0 + for i := 0; i+4 < len(buf); i++ { + if buf[i] != 0x30 || buf[i+1] != 0x82 { + continue + } + if c := buf[i+4]; c != 0x30 && c != 0x02 { + continue + } + total := int(buf[i+2])<<8 | int(buf[i+3]) + if total < 80 || i+4+total > len(buf) { + continue + } + if derAttempts++; derAttempts > maxDERAttempts { + break + } + blob := buf[i : i+4+total] + // Skip anything that looks like a certificate (already caught by certs panel). + if _, err := x509.ParseCertificate(blob); err == nil { + i += 4 + total - 1 + continue + } + if _, err := x509.ParsePKCS1PrivateKey(blob); err == nil { + out = append(out, KeyArtefact{ + Kind: "rsa-private", Format: "der", + Offset: int64(i), Length: int64(4 + total), + Summary: "RSA private key (PKCS#1 DER)", + }) + i += 4 + total - 1 + continue + } + if _, err := x509.ParsePKCS1PublicKey(blob); err == nil { + out = append(out, KeyArtefact{ + Kind: "rsa-public", Format: "der", + Offset: int64(i), Length: int64(4 + total), + Summary: "RSA public key (PKCS#1 DER)", + }) + i += 4 + total - 1 + continue + } + if k, err := x509.ParseECPrivateKey(blob); err == nil { + out = append(out, KeyArtefact{ + Kind: "ec-private", Format: "der", + Offset: int64(i), Length: int64(4 + total), + Summary: "EC private key (SEC1 DER) curve=" + k.Curve.Params().Name, + }) + i += 4 + total - 1 + continue + } + if k, err := x509.ParsePKCS8PrivateKey(blob); err == nil { + out = append(out, KeyArtefact{ + Kind: "pkcs8-private", Format: "der", + Offset: int64(i), Length: int64(4 + total), + Summary: "PKCS#8 private key — " + describePrivateKey(k), + }) + i += 4 + total - 1 + continue + } + if k, err := x509.ParsePKIXPublicKey(blob); err == nil { + out = append(out, KeyArtefact{ + Kind: "pkix-public", Format: "der", + Offset: int64(i), Length: int64(4 + total), + Summary: "PKIX public key — " + describePublicKey(k), + }) + i += 4 + total - 1 + continue + } + } + return out +} + +func isBase64Like(b []byte) bool { + if len(b) == 0 { + return false + } + for _, c := range b { + if (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || + c == '+' || c == '/' || c == '=' { + continue + } + return false + } + return true +} + +// Stringer helper used by the bridge; avoids leaking internal types to JS. +func (a KeyArtefact) String() string { + return strings.TrimSpace(fmt.Sprintf("%s %s @0x%x %s", a.Kind, a.Format, a.Offset, a.Summary)) +} diff --git a/internal/leviathan/analysis/multisearch.go b/internal/leviathan/analysis/multisearch.go new file mode 100644 index 0000000..2aa01e1 --- /dev/null +++ b/internal/leviathan/analysis/multisearch.go @@ -0,0 +1,113 @@ +package analysis + +import ( + "encoding/hex" + "errors" + "fmt" + "regexp" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +type MultiSearchQuery struct { + Mode string + Query string +} + +func ParseMultiSearchSpec(spec string) ([]MultiSearchQuery, error) { + spec = strings.ReplaceAll(spec, ";", "\n") + out := []MultiSearchQuery{} + for _, line := range strings.Split(spec, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + parts := strings.Fields(line) + if len(parts) < 2 { + return nil, errors.New("multi-search lines require: text|hex|regex ") + } + mode := strings.ToLower(parts[0]) + query := strings.TrimSpace(strings.TrimPrefix(line, parts[0])) + switch mode { + case "text", "hex", "regex": + default: + return nil, fmt.Errorf("unknown multi-search mode %q", mode) + } + if query == "" { + return nil, errors.New("empty multi-search query") + } + out = append(out, MultiSearchQuery{Mode: mode, Query: query}) + } + if len(out) == 0 { + return nil, errors.New("no multi-search queries") + } + return out, nil +} + +func MultiSearch(d *document.Document, start, length int64, spec string, maxPerQuery int) (map[string]any, error) { + queries, err := ParseMultiSearchSpec(spec) + if err != nil { + return nil, err + } + if maxPerQuery <= 0 { + maxPerQuery = 1000 + } + results := make([]map[string]any, 0, len(queries)) + all := []map[string]any{} + for i, q := range queries { + var hits []SearchHit + switch q.Mode { + case "text": + hits, err = SearchBytes(d, start, length, []byte(q.Query), maxPerQuery) + case "hex": + var needle []byte + needle, err = parseMultiSearchHex(q.Query) + if err == nil { + hits, err = SearchBytes(d, start, length, needle, maxPerQuery) + } + case "regex": + if _, err = regexp.Compile(q.Query); err == nil { + hits, err = SearchRegex(d, start, length, q.Query, maxPerQuery) + } + } + if err != nil { + return nil, fmt.Errorf("query %d: %w", i+1, err) + } + maps := HitsToMaps(hits) + name := fmt.Sprintf("%s:%s", q.Mode, q.Query) + for _, h := range maps { + h["mode"] = q.Mode + h["query"] = q.Query + h["name"] = name + all = append(all, h) + } + results = append(results, map[string]any{ + "mode": q.Mode, + "query": q.Query, + "count": len(hits), + "hits": maps, + }) + } + return map[string]any{ + "queries": results, + "all": all, + "total": len(all), + }, nil +} + +func parseMultiSearchHex(s string) ([]byte, error) { + s = strings.Map(func(r rune) rune { + switch r { + case ' ', '\t', '\n', '\r', ':', '-': + return -1 + default: + return r + } + }, strings.TrimSpace(s)) + s = strings.TrimPrefix(strings.ToLower(s), "0x") + if len(s)%2 != 0 { + s = "0" + s + } + return hex.DecodeString(s) +} diff --git a/internal/leviathan/analysis/overview.go b/internal/leviathan/analysis/overview.go new file mode 100644 index 0000000..7461714 --- /dev/null +++ b/internal/leviathan/analysis/overview.go @@ -0,0 +1,88 @@ +package analysis + +import ( + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Overview returns a compact first-pass report intended for both UI +// surfaces. It favours fast, bounded summaries over exhaustive output. +func Overview(d *document.Document) (map[string]any, error) { + size := d.Size() + hashes, err := Hashes(d, 0, -1) + if err != nil { + return nil, err + } + entropy, err := Entropy(d, 0, min64(size, 8<<20), 4096, 0) + if err != nil { + return nil, err + } + stringsOut, err := ExtractStrings(d, 0, min64(size, 16<<20), 4, []string{"ascii", "utf16le", "utf16be"}, 5000) + if err != nil { + return nil, err + } + iocs, err := ExtractIOCs(d, 0, min64(size, 16<<20), 5) + if err != nil { + return nil, err + } + carves, err := Carve(d, 0, min64(size, 64<<20), 128) + if err != nil { + return nil, err + } + return map[string]any{ + "name": d.Name(), + "size": size, + "hashes": hashes, + "entropy": overviewEntropy(entropy), + "strings": overviewStrings(stringsOut), + "iocs": iocs["totals"], + "carve": overviewCarve(carves), + "limits": map[string]any{ + "entropyBytes": min64(size, 8<<20), + "stringsBytes": min64(size, 16<<20), + "iocBytes": min64(size, 16<<20), + "carveBytes": min64(size, 64<<20), + }, + }, nil +} + +func overviewEntropy(in map[string]any) map[string]any { + return map[string]any{ + "windowSize": in["windowSize"], + "step": in["step"], + "mean": in["mean"], + "min": in["min"], + "max": in["max"], + "highCount": in["highCount"], + } +} + +func overviewStrings(items []map[string]any) map[string]any { + counts := map[string]int{} + for _, item := range items { + if enc, ok := item["encoding"].(string); ok { + counts[enc]++ + } + } + return map[string]any{ + "total": len(items), + "byEncoding": counts, + } +} + +func overviewCarve(items []map[string]any) map[string]any { + counts := map[string]int{} + valid := 0 + for _, item := range items { + if kind, ok := item["kind"].(string); ok { + counts[kind]++ + } + if status, ok := item["status"].(string); ok && status == "valid" { + valid++ + } + } + return map[string]any{ + "total": len(items), + "valid": valid, + "byKind": counts, + } +} diff --git a/internal/leviathan/analysis/rules.go b/internal/leviathan/analysis/rules.go new file mode 100644 index 0000000..6bbb5f3 --- /dev/null +++ b/internal/leviathan/analysis/rules.go @@ -0,0 +1,648 @@ +package analysis + +import ( + "context" + "encoding/hex" + "errors" + "regexp" + "strconv" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Rule represents a lightweight YARA-like rule. A rule matches when +// Condition evaluates to true for the document. Phase 1 supports: +// - "all of them" (every string pattern must appear) +// - "any of them" (at least one pattern must appear) +// - " of them" (at least N patterns) +// - named identifiers referencing specific strings with $x or !$x +type Rule struct { + Name string + Tags []string + Strings []RuleString + Condition string +} + +// RuleString is one string directive inside a rule. +type RuleString struct { + ID string + Kind string // "ascii", "hex", "regex" + Value string // original source text + // compiled: + raw []byte + regex *regexp.Regexp + nocase bool + ascii bool + wide bool + fullword bool +} + +// ParseRules parses a simplified YARA-like source: +// +// rule Name : tag1 tag2 { +// strings: +// $a = "hello" nocase +// $b = { 4d 5a ?? ?? } +// $c = /foo[0-9]+/ +// condition: +// any of them +// } +// +// Hex patterns support `??` wildcards by being converted to a regex. +func ParseRules(src string) ([]Rule, error) { + rules := []Rule{} + s := strings.TrimSpace(src) + if s == "" { + return rules, nil + } + // Split on "rule " keywords from start of line. + entries := splitRules(s) + for _, e := range entries { + r, err := parseRule(e) + if err != nil { + return rules, err + } + rules = append(rules, r) + } + return rules, nil +} + +func splitRules(src string) []string { + out := []string{} + i := 0 + for i < len(src) { + idx := strings.Index(src[i:], "rule ") + if idx < 0 { + break + } + j := i + idx + // find matching close brace (naïve but workable for phase 1) + depth := 0 + start := j + k := j + for k < len(src) { + c := src[k] + if c == '{' { + depth++ + } else if c == '}' { + depth-- + if depth == 0 { + out = append(out, src[start:k+1]) + k++ + break + } + } + k++ + } + i = k + } + return out +} + +var ruleHeaderRe = regexp.MustCompile(`(?s)^\s*rule\s+([A-Za-z_][A-Za-z0-9_]*)(?:\s*:\s*([^{]*))?\s*\{(.*)\}\s*$`) + +func parseRule(src string) (Rule, error) { + m := ruleHeaderRe.FindStringSubmatch(src) + if m == nil { + return Rule{}, errors.New("malformed rule") + } + r := Rule{Name: m[1]} + if tags := strings.TrimSpace(m[2]); tags != "" { + r.Tags = strings.Fields(tags) + } + body := m[3] + if i := strings.Index(body, "strings:"); i >= 0 { + sEnd := strings.Index(body, "condition:") + sect := body[i+len("strings:") : sEnd] + for _, line := range strings.Split(sect, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "//") { + continue + } + rs, err := parseRuleString(line) + if err != nil { + return r, err + } + r.Strings = append(r.Strings, rs) + } + } + if i := strings.Index(body, "condition:"); i >= 0 { + r.Condition = strings.TrimSpace(body[i+len("condition:"):]) + } else { + return r, errors.New("rule missing condition") + } + return r, nil +} + +var ruleStringRe = regexp.MustCompile(`^\s*(\$[A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.+?)\s*$`) + +func parseRuleString(line string) (RuleString, error) { + m := ruleStringRe.FindStringSubmatch(line) + if m == nil { + return RuleString{}, errors.New("bad string line: " + line) + } + id := m[1] + rest := strings.TrimSpace(m[2]) + value, modifiers, err := splitRuleValueModifiers(rest) + if err != nil { + return RuleString{}, err + } + rs := RuleString{ID: id, Value: value} + for _, mod := range modifiers { + switch strings.ToLower(mod) { + case "nocase": + rs.nocase = true + case "ascii": + rs.ascii = true + case "wide": + rs.wide = true + case "fullword": + rs.fullword = true + default: + return rs, errors.New("unknown string modifier: " + mod) + } + } + switch { + case strings.HasPrefix(value, `"`) && strings.HasSuffix(value, `"`) && len(value) >= 2: + rs.Kind = "ascii" + rs.raw = []byte(value[1 : len(value)-1]) + if !rs.ascii && !rs.wide { + rs.ascii = true + } + case strings.HasPrefix(value, "{") && strings.HasSuffix(value, "}"): + rs.Kind = "hex" + pattern := regexFromHexPattern(value[1 : len(value)-1]) + re, err := regexp.Compile("(?s)" + pattern) + if err != nil { + return rs, err + } + rs.regex = re + case strings.HasPrefix(value, "/") && strings.HasSuffix(value, "/"): + rs.Kind = "regex" + pat := value[1 : len(value)-1] + if rs.nocase { + pat = "(?i)" + pat + } + re, err := regexp.Compile(pat) + if err != nil { + return rs, err + } + rs.regex = re + default: + return rs, errors.New("unknown string form: " + value) + } + return rs, nil +} + +func splitRuleValueModifiers(rest string) (string, []string, error) { + rest = strings.TrimSpace(rest) + if rest == "" { + return "", nil, errors.New("empty string value") + } + var end int + switch rest[0] { + case '"': + escaped := false + end = -1 + for i := 1; i < len(rest); i++ { + c := rest[i] + if escaped { + escaped = false + continue + } + if c == '\\' { + escaped = true + continue + } + if c == '"' { + end = i + 1 + break + } + } + if end < 0 { + return "", nil, errors.New("unterminated string literal") + } + case '{': + idx := strings.IndexByte(rest, '}') + if idx < 0 { + return "", nil, errors.New("unterminated hex string") + } + end = idx + 1 + case '/': + escaped := false + end = -1 + for i := 1; i < len(rest); i++ { + c := rest[i] + if escaped { + escaped = false + continue + } + if c == '\\' { + escaped = true + continue + } + if c == '/' { + end = i + 1 + break + } + } + if end < 0 { + return "", nil, errors.New("unterminated regex") + } + default: + return "", nil, errors.New("unknown string form: " + rest) + } + return strings.TrimSpace(rest[:end]), strings.Fields(strings.TrimSpace(rest[end:])), nil +} + +func regexFromHexPattern(hexPat string) string { + var sb strings.Builder + hexPat = strings.ReplaceAll(hexPat, " ", "") + for i := 0; i < len(hexPat); { + if i+1 >= len(hexPat) { + break + } + a, b := hexPat[i], hexPat[i+1] + if a == '?' && b == '?' { + sb.WriteString(`[\x00-\xff]`) + } else { + decoded, err := hex.DecodeString(string([]byte{a, b})) + if err != nil { + // bail to literal to avoid panic + sb.WriteString(regexp.QuoteMeta(string([]byte{a, b}))) + } else { + sb.WriteString(regexp.QuoteMeta(string(decoded))) + } + } + i += 2 + } + return sb.String() +} + +// RuleMatch is a per-rule result. +type RuleMatch struct { + Name string + Tags []string + Matched bool + Hits map[string][]SearchHit +} + +// RunRules scans rules against the document range and returns matches. +func RunRules(d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) { + return RunRulesContext(context.Background(), d, start, length, rules, maxPerString) +} + +func RunRulesContext(ctx context.Context, d *document.Document, start, length int64, rules []Rule, maxPerString int) ([]map[string]any, error) { + if maxPerString <= 0 { + maxPerString = 1000 + } + out := make([]map[string]any, 0, len(rules)) + for _, r := range rules { + if err := ctx.Err(); err != nil { + return nil, err + } + hits := map[string][]SearchHit{} + for _, s := range r.Strings { + if err := ctx.Err(); err != nil { + return nil, err + } + switch s.Kind { + case "ascii": + found, err := searchRuleTextContext(ctx, d, start, length, s, maxPerString) + if err != nil { + return nil, err + } + hits[s.ID] = found + case "hex", "regex": + found, err := SearchRegexContext(ctx, d, start, length, s.regex.String(), maxPerString) + if err != nil { + return nil, err + } + if s.fullword { + found = filterFullWordHits(d, found, false) + } + hits[s.ID] = found + } + } + matched := evalCondition(r, hits) + var hitFlat []map[string]any + for id, hs := range hits { + for _, h := range hs { + hitFlat = append(hitFlat, map[string]any{ + "id": id, + "offset": h.Offset, + "length": h.Length, + "sample": h.Sample, + }) + } + } + out = append(out, map[string]any{ + "name": r.Name, + "tags": toAnySlice(r.Tags), + "matched": matched, + "hits": hitFlat, + }) + } + return out, nil +} + +func searchRuleText(d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) { + return searchRuleTextContext(context.Background(), d, start, length, s, max) +} + +func searchRuleTextContext(ctx context.Context, d *document.Document, start, length int64, s RuleString, max int) ([]SearchHit, error) { + out := []SearchHit{} + if s.ascii { + var ( + found []SearchHit + err error + ) + if s.nocase { + found, err = SearchRegexContext(ctx, d, start, length, "(?i)"+regexp.QuoteMeta(string(s.raw)), max) + } else { + found, err = SearchBytesContext(ctx, d, start, length, s.raw, max) + } + if err != nil { + return nil, err + } + if s.fullword { + found = filterFullWordHits(d, found, false) + } + out = append(out, found...) + } + if s.wide { + wide := utf16LEBytes(s.raw) + var ( + found []SearchHit + err error + ) + if s.nocase { + pat := wideNoCasePattern(s.raw) + found, err = SearchRegexContext(ctx, d, start, length, pat, max) + } else { + found, err = SearchBytesContext(ctx, d, start, length, wide, max) + } + if err != nil { + return nil, err + } + if s.fullword { + found = filterFullWordHits(d, found, true) + } + out = append(out, found...) + } + if len(out) > max { + out = out[:max] + } + return out, nil +} + +func utf16LEBytes(in []byte) []byte { + out := make([]byte, 0, len(in)*2) + for _, b := range in { + out = append(out, b, 0) + } + return out +} + +func wideNoCasePattern(in []byte) string { + var sb strings.Builder + sb.WriteString("(?i)") + for _, b := range in { + sb.WriteString(regexp.QuoteMeta(string([]byte{b, 0}))) + } + return sb.String() +} + +func filterFullWordHits(d *document.Document, hits []SearchHit, wide bool) []SearchHit { + out := hits[:0] + for _, h := range hits { + if isFullWordHit(d, h, wide) { + out = append(out, h) + } + } + return out +} + +func isFullWordHit(d *document.Document, h SearchHit, wide bool) bool { + if wide { + beforeOff := h.Offset - 2 + afterOff := h.Offset + int64(h.Length) + beforeWord := false + afterWord := false + if beforeOff >= 0 { + if b, err := d.Range(beforeOff, 2); err == nil && len(b) == 2 && b[1] == 0 { + beforeWord = isWordByte(b[0]) + } + } + if afterOff+1 < d.Size() { + if b, err := d.Range(afterOff, 2); err == nil && len(b) == 2 && b[1] == 0 { + afterWord = isWordByte(b[0]) + } + } + return !beforeWord && !afterWord + } + beforeWord := false + afterWord := false + if h.Offset > 0 { + if b, err := d.At(h.Offset - 1); err == nil { + beforeWord = isWordByte(b) + } + } + if h.Offset+int64(h.Length) < d.Size() { + if b, err := d.At(h.Offset + int64(h.Length)); err == nil { + afterWord = isWordByte(b) + } + } + return !beforeWord && !afterWord +} + +func isWordByte(b byte) bool { + return (b >= 'A' && b <= 'Z') || (b >= 'a' && b <= 'z') || (b >= '0' && b <= '9') || b == '_' +} + +func toAnySlice(in []string) []any { + out := make([]any, len(in)) + for i, s := range in { + out[i] = s + } + return out +} + +func evalCondition(r Rule, hits map[string][]SearchHit) bool { + cond := strings.TrimSpace(r.Condition) + condLower := strings.ToLower(cond) + present := 0 + for _, s := range r.Strings { + if len(hits[s.ID]) > 0 { + present++ + } + } + switch { + case condLower == "any of them": + return present > 0 + case condLower == "all of them": + return present == len(r.Strings) + case strings.HasSuffix(condLower, "of them"): + // " of them" + parts := strings.Fields(condLower) + if len(parts) >= 3 { + n, ok := parseRuleCount(parts[0]) + return ok && present >= n + } + } + if ok, matched := evalRuleSetCondition(cond, hits); ok { + return matched + } + p := conditionParser{tokens: tokenizeCondition(cond), hits: hits} + return p.parse() +} + +func evalRuleSetCondition(cond string, hits map[string][]SearchHit) (bool, bool) { + tokens := tokenizeCondition(cond) + if len(tokens) < 4 || strings.ToLower(tokens[1]) != "of" || tokens[2] != "(" || tokens[len(tokens)-1] != ")" { + return false, false + } + ids := []string{} + for _, tok := range tokens[3 : len(tokens)-1] { + if tok == "," { + continue + } + if strings.HasPrefix(tok, "$") { + ids = append(ids, tok) + } + } + if len(ids) == 0 { + return true, false + } + count := 0 + for _, id := range ids { + if len(hits[id]) > 0 { + count++ + } + } + switch strings.ToLower(tokens[0]) { + case "any": + return true, count > 0 + case "all": + return true, count == len(ids) + default: + n, ok := parseRuleCount(strings.ToLower(tokens[0])) + return true, ok && count >= n + } +} + +func parseRuleCount(s string) (int, bool) { + switch s { + case "1", "one": + return 1, true + case "2", "two": + return 2, true + case "3", "three": + return 3, true + default: + n, err := strconv.Atoi(s) + return n, err == nil + } +} + +type conditionParser struct { + tokens []string + pos int + hits map[string][]SearchHit +} + +func (p *conditionParser) parse() bool { + v := p.parseOr() + return v +} + +func (p *conditionParser) parseOr() bool { + v := p.parseAnd() + for strings.EqualFold(p.peek(), "or") { + p.next() + v = p.parseAnd() || v + } + return v +} + +func (p *conditionParser) parseAnd() bool { + v := p.parseNot() + for strings.EqualFold(p.peek(), "and") { + p.next() + v = p.parseNot() && v + } + return v +} + +func (p *conditionParser) parseNot() bool { + if strings.EqualFold(p.peek(), "not") || p.peek() == "!" { + p.next() + return !p.parseNot() + } + return p.parsePrimary() +} + +func (p *conditionParser) parsePrimary() bool { + tok := p.next() + switch { + case tok == "": + return false + case tok == "(": + v := p.parseOr() + if p.peek() == ")" { + p.next() + } + return v + case strings.EqualFold(tok, "true"): + return true + case strings.EqualFold(tok, "false"): + return false + case strings.HasPrefix(tok, "$"): + return len(p.hits[tok]) > 0 + default: + return false + } +} + +func (p *conditionParser) peek() string { + if p.pos >= len(p.tokens) { + return "" + } + return p.tokens[p.pos] +} + +func (p *conditionParser) next() string { + if p.pos >= len(p.tokens) { + return "" + } + tok := p.tokens[p.pos] + p.pos++ + return tok +} + +func tokenizeCondition(cond string) []string { + out := []string{} + for i := 0; i < len(cond); { + c := cond[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' { + i++ + continue + } + if strings.ContainsRune("(),!", rune(c)) { + out = append(out, string(c)) + i++ + continue + } + start := i + for i < len(cond) { + c = cond[i] + if c == ' ' || c == '\t' || c == '\n' || c == '\r' || strings.ContainsRune("(),!", rune(c)) { + break + } + i++ + } + out = append(out, cond[start:i]) + } + return out +} diff --git a/internal/leviathan/analysis/search.go b/internal/leviathan/analysis/search.go new file mode 100644 index 0000000..5e07558 --- /dev/null +++ b/internal/leviathan/analysis/search.go @@ -0,0 +1,189 @@ +package analysis + +import ( + "bytes" + "context" + "errors" + "regexp" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// SearchHit describes one search result. +type SearchHit struct { + Offset int64 `json:"offset"` + Length int `json:"length"` + Sample string `json:"sample"` // short hex preview +} + +// SearchBytes finds `needle` in the document range and returns hit +// offsets. Scans chunk by chunk with an overlap equal to len(needle)-1 +// so cross-chunk matches are not missed. +func SearchBytes(d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) { + return SearchBytesContext(context.Background(), d, start, length, needle, max) +} + +func SearchBytesContext(ctx context.Context, d *document.Document, start, length int64, needle []byte, max int) ([]SearchHit, error) { + if len(needle) == 0 { + return nil, errors.New("empty needle") + } + if max <= 0 { + max = 10_000 + } + if length < 0 { + length = d.Size() - start + } + + hits := make([]SearchHit, 0, 64) + overlap := int64(len(needle) - 1) + + // We chunk in windows of chunkSize + overlap. + const chunkSize = 1 << 20 // 1MB + off := start + endExclusive := start + length + + for off < endExclusive && len(hits) < max { + if err := ctx.Err(); err != nil { + return nil, err + } + winEnd := off + chunkSize + if winEnd > endExclusive { + winEnd = endExclusive + } + readEnd := winEnd + overlap + if readEnd > endExclusive { + readEnd = endExclusive + } + buf, err := d.Range(off, readEnd-off) + if err != nil { + return nil, err + } + // Limit search to bytes whose start lies in [0, winEnd-off) + searchLimit := winEnd - off + scan := 0 + for { + if err := ctx.Err(); err != nil { + return nil, err + } + idx := bytes.Index(buf[scan:], needle) + if idx < 0 { + break + } + abs := int64(scan+idx) + off + relStart := abs - off + if relStart >= searchLimit { + break + } + hits = append(hits, SearchHit{ + Offset: abs, + Length: len(needle), + Sample: hexPreview(buf[scan+idx:], len(needle)), + }) + if len(hits) >= max { + break + } + scan = scan + idx + 1 + if scan >= len(buf) { + break + } + } + off = winEnd + } + return hits, nil +} + +// SearchRegex runs a Go regexp across the document range, matching +// byte-wise. It decodes matches chunk by chunk with overlap, bounded +// by a generous max hit count. +func SearchRegex(d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) { + return SearchRegexContext(context.Background(), d, start, length, pattern, max) +} + +func SearchRegexContext(ctx context.Context, d *document.Document, start, length int64, pattern string, max int) ([]SearchHit, error) { + re, err := regexp.Compile(pattern) + if err != nil { + return nil, err + } + if max <= 0 { + max = 10_000 + } + if length < 0 { + length = d.Size() - start + } + + // For regex, a conservative overlap is 4 KiB so matches spanning + // chunk boundaries are still caught. Very long matches may still + // be missed in extreme cases. + const chunkSize = 1 << 20 + const overlap = 4096 + + hits := make([]SearchHit, 0, 64) + off := start + endExclusive := start + length + + for off < endExclusive && len(hits) < max { + if err := ctx.Err(); err != nil { + return nil, err + } + winEnd := off + chunkSize + if winEnd > endExclusive { + winEnd = endExclusive + } + readEnd := winEnd + overlap + if readEnd > endExclusive { + readEnd = endExclusive + } + buf, err := d.Range(off, readEnd-off) + if err != nil { + return nil, err + } + searchLimit := winEnd - off + locs := re.FindAllIndex(buf, max-len(hits)) + for _, loc := range locs { + if err := ctx.Err(); err != nil { + return nil, err + } + relStart := int64(loc[0]) + if relStart >= searchLimit { + continue + } + abs := relStart + off + hits = append(hits, SearchHit{ + Offset: abs, + Length: loc[1] - loc[0], + Sample: hexPreview(buf[loc[0]:loc[1]], loc[1]-loc[0]), + }) + } + off = winEnd + } + return hits, nil +} + +func hexPreview(b []byte, n int) string { + if n > 32 { + n = 32 + } + if n > len(b) { + n = len(b) + } + const hexchars = "0123456789abcdef" + out := make([]byte, n*2) + for i := 0; i < n; i++ { + out[i*2] = hexchars[b[i]>>4] + out[i*2+1] = hexchars[b[i]&0x0f] + } + return string(out) +} + +// HitsToMaps converts hits for bridge return. +func HitsToMaps(hits []SearchHit) []map[string]any { + out := make([]map[string]any, len(hits)) + for i, h := range hits { + out[i] = map[string]any{ + "offset": h.Offset, + "length": h.Length, + "sample": h.Sample, + } + } + return out +} diff --git a/internal/leviathan/analysis/strings.go b/internal/leviathan/analysis/strings.go new file mode 100644 index 0000000..c149e47 --- /dev/null +++ b/internal/leviathan/analysis/strings.go @@ -0,0 +1,198 @@ +package analysis + +import ( + "context" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// StringHit represents one extracted string. +type StringHit struct { + Offset int64 `json:"offset"` + Length int `json:"length"` + Encoding string `json:"encoding"` // ascii | utf16le | utf16be + Value string `json:"value"` +} + +// ExtractStrings finds printable runs in the specified range. +// Supports ASCII and UTF-16 LE/BE. minLen is in characters. +func ExtractStrings(d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) { + return ExtractStringsContext(context.Background(), d, start, length, minLen, encodings, max) +} + +func ExtractStringsContext(ctx context.Context, d *document.Document, start, length int64, minLen int, encodings []string, max int) ([]map[string]any, error) { + if minLen <= 0 { + minLen = 4 + } + if max <= 0 { + max = 50_000 + } + if length < 0 { + length = d.Size() - start + } + + doASCII, doLE, doBE := false, false, false + if len(encodings) == 0 { + doASCII, doLE, doBE = true, true, true + } else { + for _, e := range encodings { + switch e { + case "ascii": + doASCII = true + case "utf16le": + doLE = true + case "utf16be": + doBE = true + } + } + } + + out := make([]map[string]any, 0, 256) + + // ASCII + if doASCII { + if err := appendASCII(ctx, d, start, length, minLen, &out, max); err != nil { + return nil, err + } + } + // UTF-16 + if len(out) < max && doLE { + if err := appendUTF16(ctx, d, start, length, minLen, true, &out, max); err != nil { + return nil, err + } + } + if len(out) < max && doBE { + if err := appendUTF16(ctx, d, start, length, minLen, false, &out, max); err != nil { + return nil, err + } + } + return out, nil +} + +func appendASCII(ctx context.Context, d *document.Document, start, length int64, minLen int, out *[]map[string]any, max int) error { + // Chunked scan with a carry for runs crossing chunk boundaries. + var runStart int64 = -1 + var runBuf []byte + emit := func() { + if runStart >= 0 && len(runBuf) >= minLen { + *out = append(*out, map[string]any{ + "offset": runStart, + "length": len(runBuf), + "encoding": "ascii", + "value": string(runBuf), + }) + } + runStart = -1 + runBuf = runBuf[:0] + } + + if err := d.ForEachChunk(start, length, func(off int64, chunk []byte) error { + if err := ctx.Err(); err != nil { + return err + } + for i, b := range chunk { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + if isPrintableASCII(b) { + if runStart < 0 { + runStart = off + int64(i) + runBuf = make([]byte, 0, 32) + } + runBuf = append(runBuf, b) + } else { + if len(*out) >= max { + return nil + } + emit() + } + } + return nil + }); err != nil { + return err + } + emit() + return nil +} + +func appendUTF16(ctx context.Context, d *document.Document, start, length int64, minLen int, le bool, out *[]map[string]any, max int) error { + // Step by 2. Produce one string for each contiguous run of printable code units. + if length < 2 { + return nil + } + // Read chunks aligned to 2 bytes so pairs don't split. + const chunkSize = 1 << 20 + end := start + length + var runStart int64 = -1 + var runBuf []rune + + enc := "utf16le" + if !le { + enc = "utf16be" + } + + emit := func() { + if runStart >= 0 && len(runBuf) >= minLen { + *out = append(*out, map[string]any{ + "offset": runStart, + "length": len(runBuf) * 2, + "encoding": enc, + "value": string(runBuf), + }) + } + runStart = -1 + runBuf = runBuf[:0] + } + + for off := start; off+1 < end; { + if err := ctx.Err(); err != nil { + return err + } + winEnd := off + chunkSize + if winEnd > end { + winEnd = end + } + if (winEnd-off)%2 != 0 { + winEnd-- // keep pair alignment + } + buf, err := d.Range(off, winEnd-off) + if err != nil { + return err + } + for i := 0; i+1 < len(buf); i += 2 { + if i%4096 == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + var cu uint16 + if le { + cu = uint16(buf[i]) | uint16(buf[i+1])<<8 + } else { + cu = uint16(buf[i])<<8 | uint16(buf[i+1]) + } + if cu >= 0x20 && cu <= 0x7e { + if runStart < 0 { + runStart = off + int64(i) + runBuf = make([]rune, 0, 32) + } + runBuf = append(runBuf, rune(cu)) + } else { + if len(*out) >= max { + emit() + return nil + } + emit() + } + } + off = winEnd + } + emit() + return nil +} + +func isPrintableASCII(b byte) bool { + return b == '\t' || (b >= 0x20 && b <= 0x7e) +} diff --git a/internal/leviathan/analysis/timestamps.go b/internal/leviathan/analysis/timestamps.go new file mode 100644 index 0000000..a3d7815 --- /dev/null +++ b/internal/leviathan/analysis/timestamps.go @@ -0,0 +1,106 @@ +package analysis + +import ( + "encoding/binary" + "math" + "time" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ScanTimestamps walks the range looking for plausible timestamps in +// common encodings. This is a heuristic: it reports any 4/8-byte +// window whose decoded time falls in [1990-01-01, 2050-12-31]. +func ScanTimestamps(d *document.Document, start, length int64, maxHits int) (map[string]any, error) { + if length < 0 { + length = d.Size() - start + } + if maxHits <= 0 { + maxHits = 20_000 + } + + minT := time.Date(1990, 1, 1, 0, 0, 0, 0, time.UTC).Unix() + maxT := time.Date(2050, 12, 31, 0, 0, 0, 0, time.UTC).Unix() + ok := func(sec int64) bool { return sec >= minT && sec <= maxT } + + hits := make([]map[string]any, 0, 256) + counts := map[string]int{} + + const chunk = 1 << 20 + end := start + length + overlap := int64(7) // enough for 8-byte types + for off := start; off < end && len(hits) < maxHits; { + winEnd := off + chunk + if winEnd > end { + winEnd = end + } + readEnd := winEnd + overlap + if readEnd > end { + readEnd = end + } + buf, err := d.Range(off, readEnd-off) + if err != nil { + return nil, err + } + limit := int(winEnd - off) + for i := 0; i < limit && len(hits) < maxHits; i += 4 { + if i+4 <= len(buf) { + vLE := int64(int32(binary.LittleEndian.Uint32(buf[i:]))) + if ok(vLE) { + hits = append(hits, hit("unix_le", off+int64(i), 4, time.Unix(vLE, 0).UTC())) + counts["unix_le"]++ + continue + } + vBE := int64(int32(binary.BigEndian.Uint32(buf[i:]))) + if ok(vBE) { + hits = append(hits, hit("unix_be", off+int64(i), 4, time.Unix(vBE, 0).UTC())) + counts["unix_be"]++ + continue + } + } + if i+8 <= len(buf) { + v := binary.LittleEndian.Uint64(buf[i:]) + // Windows FILETIME + sec := int64(v/10_000_000) - filetimeEpochDelta + if ok(sec) { + hits = append(hits, hit("filetime_le", off+int64(i), 8, time.Unix(sec, 0).UTC())) + counts["filetime_le"]++ + continue + } + // WebKit (microseconds since 1601) + sec = int64(v/1_000_000) - filetimeEpochDelta + if ok(sec) { + hits = append(hits, hit("webkit_le", off+int64(i), 8, time.Unix(sec, 0).UTC())) + counts["webkit_le"]++ + continue + } + // OLE automation date + fdays := math.Float64frombits(v) + if fdays > 2 && fdays < 75000 { + base := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC) + t := base.Add(time.Duration(fdays*86400) * time.Second) + if t.Unix() >= minT && t.Unix() <= maxT { + hits = append(hits, hit("ole_le", off+int64(i), 8, t)) + counts["ole_le"]++ + continue + } + } + } + } + off = winEnd + } + return map[string]any{ + "hits": hits, + "counts": counts, + }, nil +} + +func hit(kind string, off int64, size int, t time.Time) map[string]any { + return map[string]any{ + "kind": kind, + "offset": off, + "length": size, + "iso": t.Format(time.RFC3339), + "unix": t.Unix(), + } +} diff --git a/internal/leviathan/binsnoop/README.md b/internal/leviathan/binsnoop/README.md new file mode 100644 index 0000000..c37c8e3 --- /dev/null +++ b/internal/leviathan/binsnoop/README.md @@ -0,0 +1,277 @@ +# binsnoop + +A pure-Go binary analysis library for detecting and extracting embedded files, firmware images, compressed data, filesystems, and more from binary blobs. Ported from the [binwalk](https://github.com/ReFirmLabs/binwalk) Rust codebase with like-for-like scanning logic. + +**Zero external dependencies** — everything is self-contained, including the Aho-Corasick multi-pattern matcher, CRC32 validation, and binary structure parsing. + +## Features + +- **111 built-in signatures** covering archives, compression, filesystems, images, executables, firmware headers, cryptographic constructs, and more +- **Aho-Corasick** multi-pattern scanning with overlapping match support +- **Signature validation** — every magic byte match is verified by a dedicated parser that checks headers, checksums, and structural integrity +- **Confidence scoring** — results are ranked Low / Medium / High so callers can filter noise +- **Extraction support** — carve identified objects out of a binary, or invoke external tools (7z, unsquashfs, etc.) to fully extract them +- **Filtering** — include/exclude signatures by name, restrict short signatures to offset 0, or add custom signatures +- **JSON-friendly** — all result types carry `json` struct tags for easy serialisation + +## Installation + +```bash +go get binsnoop +``` + +Or add to your `go.mod`: + +``` +require binsnoop v0.0.0 +``` + +## Quick Start + +```go +package main + +import ( + "fmt" + "binsnoop" +) + +func main() { + scanner := binsnoop.New() + + // Scan a file + results, err := scanner.ScanFile("firmware.bin") + if err != nil { + panic(err) + } + + for _, r := range results { + fmt.Printf("[0x%08X] %s (confidence: %d, size: %d)\n", + r.Offset, r.Description, r.Confidence, r.Size) + } +} +``` + +## API Reference + +### Creating a Scanner + +```go +// Default scanner with all 111 signatures +scanner := binsnoop.New() + +// Scanner with options +scanner, err := binsnoop.NewWithOptions(binsnoop.Options{ + Include: []string{"gzip", "squashfs", "uimage"}, // only these signatures + Exclude: []string{"copyright"}, // remove these signatures + FullSearch: true, // match short sigs at any offset + AdditionalSignatures: []binsnoop.Signature{...}, // add custom signatures +}) +``` + +### Scanning + +```go +// Scan raw bytes +results := scanner.ScanBytes(data) + +// Scan a file on disk +results, err := scanner.ScanFile("/path/to/firmware.bin") +``` + +Both return `[]SignatureResult` sorted by offset. + +### Analysis (Scan + Extraction) + +```go +// Analyse raw bytes (with optional extraction) +analysis := scanner.AnalyzeBytes(data, "firmware.bin", true) + +// Analyse a file on disk +analysis, err := scanner.AnalyzeFile("/path/to/firmware.bin", true) +``` + +Returns `*AnalysisResults` containing the file map and extraction outcomes. + +### Carving + +Extract a single identified object from raw data: + +```go +err := binsnoop.CarveResult(data, results[0], "/tmp/output") +``` + +This writes the bytes at the result's offset/size to a file in the output directory. + +### Listing Signatures + +```go +names := binsnoop.ListSignatures() // []string of all registered signature names +``` + +### Scanner Info + +```go +scanner.SignatureCount() // number of registered signatures (111) +scanner.PatternCount() // total magic byte patterns across all signatures +``` + +## Types + +### `SignatureResult` + +```go +type SignatureResult struct { + Offset int `json:"offset"` // byte offset in the input data + ID string `json:"id"` // unique identifier for this match + Size int `json:"size"` // size in bytes (0 = unknown) + Name string `json:"name"` // signature name (e.g. "gzip") + Confidence uint8 `json:"confidence"` // 0=Low, 128=Medium, 250=High + Description string `json:"description"` // human-readable description + AlwaysDisplay bool `json:"always_display"` // show even if low confidence + ExtractionDeclined bool `json:"extraction_declined"` // parser advises against extraction + PreferredExtractor *Extractor `json:"-"` // extractor to use for this result +} +``` + +### `Options` + +```go +type Options struct { + Include []string // restrict to these signature names + Exclude []string // remove these signature names + FullSearch bool // short sigs match at any offset + AdditionalSignatures []Signature // user-defined signatures +} +``` + +### `AnalysisResults` + +```go +type AnalysisResults struct { + FilePath string `json:"file_path"` + FileMap []SignatureResult `json:"file_map"` + Extractions map[string]*ExtractionResult `json:"extractions"` +} +``` + +### Confidence Levels + +| Constant | Value | Meaning | +|----------|-------|---------| +| `ConfidenceLow` | 0 | Magic matched but weak validation | +| `ConfidenceMedium` | 128 | Header parsed and mostly valid | +| `ConfidenceHigh` | 250 | Full structural validation passed | + +## Custom Signatures + +You can add your own signatures at scanner creation: + +```go +mySignature := binsnoop.Signature{ + Name: "my_format", + Magic: [][]byte{[]byte("MYMAGIC")}, + Parser: func(data []byte, offset int) (*binsnoop.SignatureResult, error) { + // Validate the match and return a result + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data") + } + return &binsnoop.SignatureResult{ + Offset: offset, + Size: 1024, + Description: "My custom format, size: 1024 bytes", + Confidence: binsnoop.ConfidenceHigh, + }, nil + }, + Description: "My custom file format", +} + +scanner, _ := binsnoop.NewWithOptions(binsnoop.Options{ + AdditionalSignatures: []binsnoop.Signature{mySignature}, +}) +``` + +## Supported Signatures + +### Archives (12) +gzip, zip, tarball, 7zip, rar, cab, deb, cpio, arj, iso9660, dahua\_zip + +### Compression (10) +lzma, bzip2, xz, zstd, lz4, lzop, lzfse, zlib, compress'd + +### Filesystems (13) +squashfs, cramfs, jffs2, yaffs, romfs, ext, fat, ntfs, apfs, btrfs, logfs, ubi, ubifs + +### Images & Media (7) +png, jpeg, bmp, gif, svg, riff, pdf + +### Executables (2) +elf, pe + +### Firmware (32) +uimage, uboot, trx, tplink, tplink\_rtos, dlob, arcadyan, chk, seama, cfe, jboot\_arm, jboot\_stag, jboot\_sch2, packimg, binhdr, autel, android\_sparse, android\_bootimg, dtb, dlink\_tlv, dlke, shrs, dkbs, dms, encfw, encrpted\_img, wince, mh01, csman, dxbc, matter\_ota, rtk + +### Cryptographic (17) +aes\_sbox, aes\_forward\_table, aes\_reverse\_table, aes\_rcon, aes\_acceleration\_table, rsa, openssl, gpg\_signed, pem\_certificate, pem\_public\_key, pem\_private\_key, pkcs\_der\_hash, luks, crc32, sha256, md5 + +### Miscellaneous (18) +copyright, wind\_kernel, vxworks\_symtab, ecos, linux\_kernel, linux\_boot\_image, linux\_arm\_zimage, linux\_arm64\_boot\_image, efigpt, mbr, uefi\_pi\_volume, uefi\_capsule, pchrom, pcapng, pjl, srecord, dmg, qnx\_ifs, dpapi, qcow + +## Extraction + +Many signatures have associated extractors that invoke external command-line tools. When you call `AnalyzeFile` or `AnalyzeBytes` with `extract: true`, binsnoop will: + +1. Carve the identified data to a temporary file +2. Run the configured external tool (e.g. `7z x`, `unsquashfs`, `jefferson`) +3. Return the extraction results including output directory and success/failure status + +**Common external tools used:** + +| Tool | Signatures | +|------|-----------| +| `7z` | zip, cpio, arj, iso9660, xz, lzma, squashfs, dmg, efigpt | +| `gzip` | gzip | +| `bzip2` | bzip2 | +| `zstd` | zstd | +| `lz4` | lz4 | +| `unsquashfs` | squashfs | +| `jefferson` | jffs2 | +| `unyaffs` | yaffs | +| `tsk_recover` | ext, fat, ntfs | +| `uefi-firmware-parser` | uefi\_pi\_volume, uefi\_capsule, pchrom | +| `simg2img` | android\_sparse | +| `dtc` | dtb | + +## How It Works + +### Scanning Algorithm + +1. **Short-signature pass** — signatures marked `Short` (e.g. zlib, MBR) are checked only at offset 0 (unless `FullSearch` is enabled) +2. **Aho-Corasick scan** — all magic byte patterns are compiled into an Aho-Corasick automaton for efficient multi-pattern matching. When a high-confidence sized match is found, scanning restarts from the end of that match +3. **Validation** — every magic match is passed to its signature parser which validates headers, checksums, and structure +4. **Post-processing** — results are sorted by offset, overlapping/duplicate matches are resolved (higher confidence wins), and unknown sizes are estimated based on the next significant match or EOF + +### Architecture + +``` +binsnoop.go Public API (New, ScanBytes, ScanFile, AnalyzeFile, CarveResult, etc.) +scanner.go Core scan engine (Aho-Corasick integration, post-processing) +match.go Self-contained Aho-Corasick trie with overlapping match support +parse.go Generic binary structure parser (u8/u16/u24/u32/u64, big/little endian) +types.go Core types (SignatureResult, Signature, Extractor, Options, etc.) +common.go Utility functions (CRC32, endian helpers, string helpers) +extract.go Carving and external tool execution +magic.go Signature registry — wires all 111 signatures together +sig_archives.go Archive signature parsers +sig_compression.go Compression signature parsers +sig_filesystems.go Filesystem signature parsers +sig_images.go Image/media signature parsers +sig_executables.go Executable signature parsers +sig_firmware.go Firmware header signature parsers +sig_crypto.go Cryptographic signature parsers +sig_misc.go Miscellaneous signature parsers +``` + +## License + +This project is a Go port of [binwalk](https://github.com/ReFirmLabs/binwalk) by ReFirmLabs. See the original project for licensing details. diff --git a/internal/leviathan/binsnoop/binsnoop.go b/internal/leviathan/binsnoop/binsnoop.go new file mode 100644 index 0000000..7f79085 --- /dev/null +++ b/internal/leviathan/binsnoop/binsnoop.go @@ -0,0 +1,177 @@ +// Package binsnoop identifies and optionally extracts files embedded inside binary data. +// +// It is a Go port of the binwalk scanning engine, implementing all magic byte signatures, +// structure parsers, and the Aho-Corasick based multi-pattern matching algorithm. +// +// # Quick Start +// +// scanner := binsnoop.New() +// data, _ := os.ReadFile("firmware.bin") +// results := scanner.Scan(data) +// for _, r := range results { +// fmt.Printf("0x%08X %s\n", r.Offset, r.Description) +// } +// +// # Extraction +// +// // Extract a specific result +// err := binsnoop.CarveResult(data, results[0], "/tmp/output") +// +// # Filtering +// +// // Only scan for specific signatures +// scanner := binsnoop.NewWithOptions(Options{Include: []string{"gzip", "squashfs"}}) +package binsnoop + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// Options configures the Scanner. +type Options struct { + // Include restricts scanning to only these signature names. + Include []string + // Exclude removes these signature names from scanning. + Exclude []string + // FullSearch makes short signatures match at any offset, not just offset 0. + FullSearch bool + // AdditionalSignatures adds user-defined signatures to the scanner. + AdditionalSignatures []Signature +} + +// New creates a Scanner with all default signatures enabled. +func New() *Scanner { + s, _ := NewWithOptions(Options{}) + return s +} + +// NewWithOptions creates a Scanner with the specified options. +func NewWithOptions(opts Options) (*Scanner, error) { + s := &Scanner{ + patternSigTable: make(map[int]Signature), + extractorLookup: make(map[string]*Extractor), + } + + allSigs := AllSignatures() + if opts.AdditionalSignatures != nil { + allSigs = append(allSigs, opts.AdditionalSignatures...) + } + + for _, sig := range allSigs { + if !includeSig(&sig, opts.Include, opts.Exclude) { + continue + } + + s.signatureCount++ + s.patternCount += len(sig.Magic) + + // Register extractor + var ext *Extractor + if sig.Extractor != nil { + extCopy := *sig.Extractor + ext = &extCopy + } + s.extractorLookup[sig.Name] = ext + + for _, pattern := range sig.Magic { + if sig.Short && !opts.FullSearch { + s.shortSignatures = append(s.shortSignatures, sig) + break + } + s.patternSigTable[len(s.patterns)] = sig + s.patterns = append(s.patterns, pattern) + } + } + + s.compile() + return s, nil +} + +// ScanBytes scans raw bytes for signatures and returns all matches. +func (s *Scanner) ScanBytes(data []byte) []SignatureResult { + return s.Scan(data) +} + +// ScanFile reads a file from disk and scans it for signatures. +func (s *Scanner) ScanFile(path string) ([]SignatureResult, error) { + data, err := ReadFile(path) + if err != nil { + return nil, fmt.Errorf("binsnoop: read file: %w", err) + } + return s.Scan(data), nil +} + +// AnalyzeBytes scans and optionally extracts from raw bytes. +func (s *Scanner) AnalyzeBytes(data []byte, filePath string, extract bool) *AnalysisResults { + results := &AnalysisResults{ + FilePath: filePath, + Extractions: make(map[string]*ExtractionResult), + } + + results.FileMap = s.Scan(data) + + if extract && len(results.FileMap) > 0 { + results.Extractions = s.Extract(data, filePath, results.FileMap) + } + + return results +} + +// AnalyzeFile reads a file from disk, scans, and optionally extracts. +func (s *Scanner) AnalyzeFile(path string, extract bool) (*AnalysisResults, error) { + data, err := ReadFile(path) + if err != nil { + return nil, fmt.Errorf("binsnoop: read file: %w", err) + } + return s.AnalyzeBytes(data, path, extract), nil +} + +// CarveResult extracts raw bytes for a specific signature result to a file. +func CarveResult(data []byte, result SignatureResult, outputDir string) error { + if result.Offset < 0 || result.Offset+result.Size > len(data) { + return fmt.Errorf("binsnoop: result offset/size exceeds data bounds") + } + if err := os.MkdirAll(outputDir, 0755); err != nil { + return fmt.Errorf("binsnoop: mkdir: %w", err) + } + outPath := filepath.Join(outputDir, fmt.Sprintf("%x_%s.bin", result.Offset, result.Name)) + return os.WriteFile(outPath, data[result.Offset:result.Offset+result.Size], 0644) +} + +// SignatureCount returns the number of loaded signatures. +func (s *Scanner) SignatureCount() int { return s.signatureCount } + +// PatternCount returns the total number of magic byte patterns. +func (s *Scanner) PatternCount() int { return s.patternCount } + +// ListSignatures returns the names of all available signatures. +func ListSignatures() []string { + sigs := AllSignatures() + names := make([]string, len(sigs)) + for i, s := range sigs { + names[i] = s.Name + } + return names +} + +func includeSig(sig *Signature, include, exclude []string) bool { + if len(include) > 0 { + for _, inc := range include { + if strings.EqualFold(sig.Name, inc) { + return true + } + } + return false + } + if len(exclude) > 0 { + for _, exc := range exclude { + if strings.EqualFold(sig.Name, exc) { + return false + } + } + } + return true +} diff --git a/internal/leviathan/binsnoop/common.go b/internal/leviathan/binsnoop/common.go new file mode 100644 index 0000000..8e33b12 --- /dev/null +++ b/internal/leviathan/binsnoop/common.go @@ -0,0 +1,127 @@ +package binsnoop + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "os" + "time" +) + +const MaxFileReadBytes int64 = 64 << 20 + +// CRC32 calculates the CRC32 checksum using the IEEE polynomial. +func CRC32(data []byte) uint32 { + return crc32.ChecksumIEEE(data) +} + +// EpochToString converts a Unix timestamp to a formatted time string. +func EpochToString(epoch uint32) string { + t := time.Unix(int64(epoch), 0).UTC() + return t.Format("2006-01-02 15:04:05") +} + +// GetCString extracts a NULL-terminated C string from raw bytes. +func GetCString(data []byte) string { + for i, b := range data { + if b == 0 { + s := string(data[:i]) + // Validate UTF-8 printability + for _, c := range s { + if c == 0 { + return "" + } + } + return s + } + } + return string(data) +} + +// IsASCIINumber returns true if the byte is an ASCII digit (0-9). +func IsASCIINumber(b byte) bool { + return b >= '0' && b <= '9' +} + +// IsPrintableASCII returns true if the byte is a printable ASCII character. +func IsPrintableASCII(b byte) bool { + return b >= 0x0A && b <= 0x7E +} + +// IsOffsetSafe validates that the next offset is within bounds and progressing forward. +func IsOffsetSafe(availableData, nextOffset int, lastOffset *int) bool { + if lastOffset != nil { + if *lastOffset >= nextOffset { + return false + } + } + return nextOffset < availableData +} + +// ReadFile reads a bounded file into memory for scanner APIs that require a +// byte slice. Larger callers should use a streaming document path. +func ReadFile(path string) ([]byte, error) { + return ReadFileLimited(path, MaxFileReadBytes) +} + +func ReadFileLimited(path string, maxBytes int64) ([]byte, error) { + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.IsDir() { + return nil, fmt.Errorf("binsnoop: path is a directory") + } + if maxBytes > 0 && info.Size() > maxBytes { + return nil, fmt.Errorf("binsnoop: file is too large: %d bytes exceeds %d bytes", info.Size(), maxBytes) + } + return os.ReadFile(path) +} + +// LE16 reads a little-endian uint16 from data at the given offset. +func LE16(data []byte, offset int) uint16 { + if offset+2 > len(data) { + return 0 + } + return binary.LittleEndian.Uint16(data[offset:]) +} + +// BE16 reads a big-endian uint16 from data at the given offset. +func BE16(data []byte, offset int) uint16 { + if offset+2 > len(data) { + return 0 + } + return binary.BigEndian.Uint16(data[offset:]) +} + +// LE32 reads a little-endian uint32 from data at the given offset. +func LE32(data []byte, offset int) uint32 { + if offset+4 > len(data) { + return 0 + } + return binary.LittleEndian.Uint32(data[offset:]) +} + +// BE32 reads a big-endian uint32 from data at the given offset. +func BE32(data []byte, offset int) uint32 { + if offset+4 > len(data) { + return 0 + } + return binary.BigEndian.Uint32(data[offset:]) +} + +// LE64 reads a little-endian uint64 from data at the given offset. +func LE64(data []byte, offset int) uint64 { + if offset+8 > len(data) { + return 0 + } + return binary.LittleEndian.Uint64(data[offset:]) +} + +// BE64 reads a big-endian uint64 from data at the given offset. +func BE64(data []byte, offset int) uint64 { + if offset+8 > len(data) { + return 0 + } + return binary.BigEndian.Uint64(data[offset:]) +} diff --git a/internal/leviathan/binsnoop/common_test.go b/internal/leviathan/binsnoop/common_test.go new file mode 100644 index 0000000..e3a933c --- /dev/null +++ b/internal/leviathan/binsnoop/common_test.go @@ -0,0 +1,22 @@ +package binsnoop + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestReadFileLimitedRejectsOversizedFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "firmware.bin") + if err := os.WriteFile(path, []byte("abcd"), 0600); err != nil { + t.Fatalf("WriteFile() failed: %v", err) + } + _, err := ReadFileLimited(path, 3) + if err == nil { + t.Fatal("expected oversized file to be rejected") + } + if !strings.Contains(err.Error(), "file is too large") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/leviathan/binsnoop/extract.go b/internal/leviathan/binsnoop/extract.go new file mode 100644 index 0000000..5891729 --- /dev/null +++ b/internal/leviathan/binsnoop/extract.go @@ -0,0 +1,124 @@ +package binsnoop + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +// Carve writes a slice of data to a file on disk. +func Carve(outputPath string, data []byte, offset, size int) error { + if offset < 0 || offset+size > len(data) { + return fmt.Errorf("carve: offset %d + size %d exceeds data length %d", offset, size, len(data)) + } + dir := filepath.Dir(outputPath) + if err := os.MkdirAll(dir, 0755); err != nil { + return fmt.Errorf("carve: mkdir: %w", err) + } + return os.WriteFile(outputPath, data[offset:offset+size], 0644) +} + +// executeExternal runs an external extraction tool. +func executeExternal(ext *Extractor, carvedFilePath, outputDir string) *ExtractionResult { + result := &ExtractionResult{ + ExtractorName: ext.Command, + OutputDirectory: outputDir, + } + + if err := os.MkdirAll(outputDir, 0755); err != nil { + return result + } + + // Build arguments, replacing the source file placeholder + args := make([]string, len(ext.Arguments)) + for i, arg := range ext.Arguments { + args[i] = strings.ReplaceAll(arg, SourceFilePlaceholder, carvedFilePath) + } + + cmd := exec.Command(ext.Command, args...) + cmd.Dir = outputDir + + output, err := cmd.CombinedOutput() + _ = output + + exitCode := 0 + if err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + exitCode = exitErr.ExitCode() + } else { + return result + } + } + + // Check if exit code is in the accepted list + for _, code := range ext.ExitCodes { + if exitCode == code { + result.Success = true + break + } + } + + result.DoNotRecurse = ext.DoNotRecurse + return result +} + +// executeInternal runs an internal extraction function. +func executeInternal(ext *Extractor, data []byte, offset int, outputDir string) *ExtractionResult { + if ext.Function == nil { + return &ExtractionResult{} + } + result := ext.Function(data, offset, outputDir) + if result == nil { + result = &ExtractionResult{} + } + result.DoNotRecurse = ext.DoNotRecurse + return result +} + +// ExecuteExtraction performs extraction for a signature result. +func ExecuteExtraction(data []byte, filePath string, sig *SignatureResult, ext *Extractor) *ExtractionResult { + if ext == nil { + return &ExtractionResult{} + } + + // Use preferred extractor if specified + actualExt := ext + if sig.PreferredExtractor != nil { + actualExt = sig.PreferredExtractor + } + + // Build output directory path + baseName := filepath.Base(filePath) + outputDir := filepath.Join(filepath.Dir(filePath), baseName+".extracted", + fmt.Sprintf("%x", sig.Offset)) + + switch actualExt.Type { + case ExtractorTypeInternal: + return executeInternal(actualExt, data, sig.Offset, outputDir) + + case ExtractorTypeExternal: + // Carve data to a temp file for external tools + if err := os.MkdirAll(outputDir, 0755); err != nil { + return &ExtractionResult{OutputDirectory: outputDir} + } + + carvedFile := filepath.Join(outputDir, fmt.Sprintf("carved.%s", actualExt.Extension)) + endOffset := sig.Offset + sig.Size + if endOffset > len(data) { + endOffset = len(data) + } + if err := os.WriteFile(carvedFile, data[sig.Offset:endOffset], 0644); err != nil { + return &ExtractionResult{OutputDirectory: outputDir} + } + + result := executeExternal(actualExt, carvedFile, outputDir) + result.OutputDirectory = outputDir + result.ExtractorName = actualExt.Command + return result + + default: + return &ExtractionResult{} + } +} diff --git a/internal/leviathan/binsnoop/magic.go b/internal/leviathan/binsnoop/magic.go new file mode 100644 index 0000000..50153d9 --- /dev/null +++ b/internal/leviathan/binsnoop/magic.go @@ -0,0 +1,268 @@ +package binsnoop + +// AllSignatures returns every supported signature definition. +// This is the equivalent of magic::patterns() in the Rust binwalk codebase. +func AllSignatures() []Signature { + return []Signature{ + // ===================== Archives ===================== + {Name: "gzip", Magic: gzipMagic(), Parser: gzipParser, Description: gzipDescription, Extractor: extGzip()}, + {Name: "deb", Magic: debMagic(), Parser: debParser, Description: debDescription}, + {Name: "7zip", Magic: sevenzipMagic(), Parser: sevenzipParser, Description: sevenzipDescription, Extractor: extSevenzip()}, + {Name: "xz", Magic: XZMagic(), Parser: XZParser, Description: XZDescription, Extractor: extLzma()}, + {Name: "tarball", Magic: tarballMagic(), Parser: tarballParser, Description: tarballDescription, Extractor: extTar()}, + {Name: "zip", Magic: zipMagic(), Parser: zipParser, Description: zipDescription, Extractor: extSevenzip()}, + {Name: "cpio", Magic: cpioMagic(), Parser: cpioParser, Description: cpioDescription, Extractor: extSevenzip()}, + {Name: "iso9660", Magic: iso9660Magic(), Parser: iso9660Parser, Description: iso9660Description, Extractor: extSevenzip()}, + {Name: "rar", Magic: rarMagic(), Parser: rarParser, Description: rarDescription, Extractor: extRar()}, + {Name: "cab", Magic: cabMagic(), Parser: cabParser, Description: cabDescription, Extractor: extCab()}, + {Name: "arj", Magic: arjMagic(), Parser: arjParser, Description: arjDescription, Extractor: extSevenzip()}, + {Name: "dahua_zip", Magic: dahuaZipMagic(), Parser: dahuaZipParser, Description: dahuaZipDescription, Extractor: extSevenzip()}, + + // ===================== Compression ===================== + {Name: "lzma", Magic: LZMAMagic(), Parser: LZMAParser, Description: LZMADescription, Extractor: extLzma()}, + {Name: "bzip2", Magic: Bzip2Magic(), Parser: Bzip2Parser, Description: Bzip2Description, Extractor: extBzip2()}, + {Name: "zstd", Magic: ZSTDMagic(), Parser: ZSTDParser, Description: ZSTDDescription, Extractor: extZstd()}, + {Name: "lz4", Magic: LZ4Magic(), Parser: LZ4Parser, Description: LZ4Description, Extractor: extLz4()}, + {Name: "lzop", Magic: LZOPMagic(), Parser: LZOPParser, Description: LZOPDescription, Extractor: extLzop()}, + {Name: "lzfse", Magic: LZFSEMagic(), Parser: LZFSEParser, Description: LZFSEDescription, Extractor: extLzfse()}, + {Name: "zlib", Short: true, Magic: ZlibMagic(), Parser: ZlibParser, Description: ZlibDescription, Extractor: extZlib()}, + {Name: "compressd", Short: true, Magic: CompressdMagic(), Parser: CompressdParser, Description: CompressdDescription, Extractor: extSevenzip()}, + + // ===================== Filesystems ===================== + {Name: "squashfs", Magic: squashfs_magic(), Parser: squashfs_parser, Description: SquashFSDescription, Extractor: extSquashfs()}, + {Name: "cramfs", Magic: cramfs_magic(), Parser: cramfs_parser, Description: CramFSDescription, Extractor: extSevenzip()}, + {Name: "jffs2", Magic: jffs2_magic(), Parser: jffs2_parser, Description: JFFS2Description, Extractor: extJffs2()}, + {Name: "yaffs", Magic: yaffs_magic(), Parser: yaffs_parser, Description: YAFFSDescription, Extractor: extYaffs2()}, + {Name: "romfs", Magic: romfs_magic(), Parser: romfs_parser, Description: RomFSDescription, Extractor: extRomfs()}, + {Name: "ext", Magic: ext_magic(), Parser: ext_parser, Description: EXTDescription, Extractor: extTsk()}, + {Name: "fat", Short: true, MagicOffset: FATMagicOffset, Magic: fat_magic(), Parser: fat_parser, Description: FATDescription, Extractor: extTsk()}, + {Name: "ntfs", Magic: ntfs_magic(), Parser: ntfs_parser, Description: NTFSDescription, Extractor: extTsk()}, + {Name: "apfs", Magic: apfs_magic(), Parser: apfs_parser, Description: APFSDescription, Extractor: extSevenzip()}, + {Name: "btrfs", AlwaysDisplay: true, Magic: btrfs_magic(), Parser: btrfs_parser, Description: BTRFSDescription}, + {Name: "logfs", Magic: logfs_magic(), Parser: logfs_parser, Description: LogFSDescription}, + {Name: "ubi", Magic: ubi_magic(), Parser: ubi_parser, Description: UBIImageDescription, Extractor: extUbi()}, + {Name: "ubifs", Magic: ubifs_magic(), Parser: ubifs_parser, Description: UBIFSDescription, Extractor: extUbifs()}, + + // ===================== Images & Media ===================== + {Name: "png", Magic: PNGMagic(), Parser: PNGParser, Description: PNGDescription, Extractor: extPng()}, + {Name: "jpeg", Magic: JPEGMagic(), Parser: JPEGParser, Description: JPEGDescription, Extractor: extJpeg()}, + {Name: "bmp", Magic: BMPMagic(), Parser: BMPParser, Description: BMPDescription, Extractor: extBmp()}, + {Name: "gif", Magic: GIFMagic(), Parser: GIFParser, Description: GIFDescription, Extractor: extGif()}, + {Name: "svg", Magic: SVGMagic(), Parser: SVGParser, Description: SVGDescription, Extractor: extSvg()}, + {Name: "riff", Magic: RIFFMagic(), Parser: RIFFParser, Description: RIFFDescription, Extractor: extRiff()}, + {Name: "pdf", Magic: PDFMagic(), Parser: PDFParser, Description: PDFDescription}, + + // ===================== Executables ===================== + {Name: "elf", Magic: ELFMagic(), Parser: ELFParser, Description: ELFDescription}, + {Name: "pe", Magic: PEMagic(), Parser: PEParser, Description: PEDescription}, + + // ===================== Firmware Headers ===================== + {Name: "uimage", Magic: uimageMagic(), Parser: uimageParser, Description: uimageDescription, Extractor: extUimage()}, + {Name: "packimg", Magic: packimgMagic(), Parser: packimgParser, Description: packimgDescription}, + {Name: "uboot", AlwaysDisplay: true, Magic: ubootMagic(), Parser: ubootParser, Description: ubootDescription}, + {Name: "trx", Magic: trxMagic(), Parser: trxParser, Description: trxDescription, Extractor: extTrx()}, + {Name: "tplink", Magic: tplinkMagic(), Parser: tplinkParser, Description: tplinkDescription}, + {Name: "tplink_rtos", Magic: tplinkRTOSMagic(), Parser: tplinkRTOSParser, Description: tplinkRTOSDescription}, + {Name: "dlob", Magic: dlobMagic(), Parser: dlobParser, Description: dlobDescription}, + {Name: "arcadyan", Magic: arcadyanMagic(), Parser: arcadyanParser, Description: arcadyanDescription}, + {Name: "chk", Magic: chkMagic(), Parser: chkParser, Description: chkDescription}, + {Name: "seama", AlwaysDisplay: true, Magic: seamaMagic(), Parser: seamaParser, Description: seamaDescription}, + {Name: "cfe", AlwaysDisplay: true, Magic: cfeMagic(), Parser: cfeParser, Description: cfeDescription}, + {Name: "jboot_arm", Magic: jbootARMMagic(), Parser: jbootARMParser, Description: jbootARMDescription}, + {Name: "jboot_stag", Magic: jbootSTAGMagic(), Parser: jbootSTAGParser, Description: jbootSTAGDescription}, + {Name: "jboot_sch2", Magic: jbootSCH2Magic(), Parser: jbootSCH2Parser, Description: jbootSCH2Description, Extractor: extJboot()}, + {Name: "binhdr", Magic: binhdrMagic(), Parser: binhdrParser, Description: binhdrDescription}, + {Name: "autel", Magic: autelMagic(), Parser: autelParser, Description: autelDescription, Extractor: extAutel()}, + {Name: "android_sparse", Magic: androidSparseMagic(), Parser: androidSparseParser, Description: androidSparseDescription, Extractor: extAndroidSparse()}, + {Name: "dtb", Magic: dtbMagic(), Parser: dtbParser, Description: dtbDescription, Extractor: extDtb()}, + {Name: "android_bootimg", Magic: androidBootimgMagic(), Parser: androidBootimgParser, Description: androidBootimgDescription}, + {Name: "dlink_tlv", Magic: dlinkTLVMagic(), Parser: dlinkTLVParser, Description: dlinkTLVDescription, Extractor: extEncfw()}, + {Name: "dlke", Magic: dlkeMagic(), Parser: dlkeParser, Description: dlkeDescription, Extractor: extEncfw()}, + {Name: "shrs", Magic: shrsMagic(), Parser: shrsParser, Description: shrsDescription, Extractor: extEncfw()}, + {Name: "dkbs", Magic: dkbsMagic(), Parser: dkbsParser, Description: dkbsDescription, Extractor: extEncfw()}, + {Name: "dms", Magic: dmsMagic(), Parser: dmsParser, Description: dmsDescription}, + {Name: "encfw", Short: true, AlwaysDisplay: true, Magic: encfwMagic(), Parser: encfwParser, Description: encfwDescription, Extractor: extEncfw()}, + {Name: "encrpted_img", Short: true, Magic: encrptedImgMagic(), Parser: encrptedImgParser, Description: encrptedImgDescription, Extractor: extEncfw()}, + {Name: "wince", Magic: winceMagic(), Parser: winceParser, Description: winceDescription}, + {Name: "mh01", Magic: mh01Magic(), Parser: mh01Parser, Description: mh01Description}, + {Name: "csman", Short: true, Magic: csmanMagic(), Parser: csmanParser, Description: csmanDescription}, + {Name: "dxbc", Magic: dxbcMagic(), Parser: dxbcParser, Description: dxbcDescription}, + {Name: "matter_ota", Short: true, Magic: matterOTAMagic(), Parser: matterOTAParser, Description: matterOTADescription}, + {Name: "rtk", Short: true, Magic: rtkMagic(), Parser: rtkParser, Description: rtkDescription}, + {Name: "dpapi", Short: true, AlwaysDisplay: true, Magic: dpapiMagic(), Parser: dpapiParser, Description: dpapiDescription}, + {Name: "qcow", Short: true, AlwaysDisplay: true, Magic: qcowMagic(), Parser: qcowParser, Description: qcowDescription}, + + // ===================== Crypto & Hashes ===================== + {Name: "aes_sbox", Magic: aesSBoxMagic(), Parser: aesSBoxParser, Description: DescriptionAESSBox}, + {Name: "aes_forward_table", Magic: aesForwardTableMagic(), Parser: aesForwardTableParser, Description: DescriptionAESFT}, + {Name: "aes_reverse_table", Magic: aesReverseTableMagic(), Parser: aesReverseTableParser, Description: DescriptionAESRT}, + {Name: "aes_rcon", Magic: aesRconMagic(), Parser: aesRconParser, Description: DescriptionAESRCON}, + {Name: "aes_acceleration_table", Magic: aesAccelerationTableMagic(), Parser: aesAccelerationTableParser, Description: DescriptionAESAcc}, + {Name: "rsa", AlwaysDisplay: true, Magic: rsaMagic(), Parser: rsaParser, Description: DescriptionRSA}, + {Name: "openssl", AlwaysDisplay: true, Magic: opensslMagic(), Parser: opensslParser, Description: DescriptionOpenSSL, Extractor: extEncfw()}, + {Name: "gpg_signed", Short: true, Magic: gpgSignedMagic(), Parser: gpgSignedParser, Description: DescriptionGPGSigned, Extractor: extGpg()}, + {Name: "pem_certificate", Magic: pemCertificateMagic(), Parser: pemParser, Description: DescriptionPEMCertificate, Extractor: extPemCert()}, + {Name: "pem_public_key", AlwaysDisplay: true, Magic: pemPublicKeyMagic(), Parser: pemParser, Description: DescriptionPEMPublicKey, Extractor: extPemKey()}, + {Name: "pem_private_key", AlwaysDisplay: true, Magic: pemPrivateKeyMagic(), Parser: pemParser, Description: DescriptionPEMPrivateKey, Extractor: extPemKey()}, + {Name: "pkcs_der_hash", Magic: derHashMagic(), Parser: derHashParser, Description: DescriptionDERHash}, + {Name: "luks", Magic: luksMagic(), Parser: luksParser, Description: DescriptionLUKS}, + {Name: "crc32", Magic: crc32ConstMagic(), Parser: crc32Parser, Description: DescriptionCRC32}, + {Name: "sha256", Magic: sha256ConstMagic(), Parser: sha256Parser, Description: DescriptionSHA256}, + {Name: "md5", Magic: md5ConstMagic(), Parser: md5Parser, Description: DescriptionMD5}, + + // ===================== Misc ===================== + {Name: "copyright", Magic: copyrightMagic(), Parser: copyrightParser, Description: DescriptionCopyright}, + {Name: "wind_kernel", AlwaysDisplay: true, Magic: windKernelMagic(), Parser: windKernelParser, Description: DescriptionWINDKernelVersion}, + {Name: "vxworks_symtab", AlwaysDisplay: true, Magic: vxworksSymtabMagic(), Parser: vxworksSymtabParser, Description: DescriptionVxWorksSymTab}, + {Name: "ecos", AlwaysDisplay: true, Magic: ecosExceptionHandlerMagic(), Parser: ecosExceptionHandlerParser, Description: DescriptionECosExceptionHandler}, + {Name: "linux_kernel", AlwaysDisplay: true, Magic: linuxKernelVersionMagic(), Parser: linuxKernelVersionParser, Description: DescriptionLinuxKernelVersion, Extractor: extLinuxKernel()}, + {Name: "linux_boot_image", Magic: linuxBootImageMagic(), Parser: linuxBootImageParser, Description: DescriptionLinuxBootImage}, + {Name: "linux_arm_zimage", Magic: linuxARMzImageMagic(), Parser: linuxARMzImageParser, Description: DescriptionLinuxARMzImage}, + {Name: "linux_arm64_boot_image", Magic: linuxARM64BootImageMagic(), Parser: linuxARM64BootImageParser, Description: DescriptionLinuxARM64BootImage}, + {Name: "efigpt", Magic: efigptMagic(), Parser: efigptParser, Description: DescriptionEFIGPT, Extractor: extSevenzip()}, + {Name: "mbr", Short: true, MagicOffset: MBRMagicOffset, AlwaysDisplay: true, Magic: mbrMagic(), Parser: mbrParser, Description: DescriptionMBR, Extractor: extMbr()}, + {Name: "uefi_pi_volume", Magic: uefiVolumeMagic(), Parser: uefiVolumeParser, Description: DescriptionUEFIVolume, Extractor: extUefi()}, + {Name: "uefi_capsule", Magic: uefiCapsuleMagic(), Parser: uefiCapsuleParser, Description: DescriptionUEFICapsule, Extractor: extUefi()}, + {Name: "pchrom", Magic: pchRomMagic(), Parser: pchRomParser, Description: DescriptionPCHRom, Extractor: extUefi()}, + {Name: "pcapng", Magic: pcapngMagic(), Parser: pcapngParser, Description: DescriptionPcapNG, Extractor: extPcap()}, + {Name: "pjl", Magic: pjlMagic(), Parser: pjlParser, Description: DescriptionPJL}, + {Name: "srecord", Magic: srecordMagic(), Parser: srecordParser, Description: DescriptionSRecord, Extractor: extSrec()}, + {Name: "srecord_generic", Short: true, Magic: srecordGenericMagic(), Parser: srecordParser, Description: DescriptionSRecordGeneric, Extractor: extSrec()}, + {Name: "dmg", Magic: dmgMagic(), Parser: dmgParser, Description: DescriptionDMG, Extractor: extDmg()}, + {Name: "qnx_ifs", Magic: qnxIFSMagic(), Parser: qnxIFSParser, Description: DescriptionQNXIFS, Extractor: extDumpifs()}, + } +} + +// ============================================================================ +// External extractor definitions (command-line tools) +// ============================================================================ + +func extSevenzip() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extGzip() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "gzip", Extension: "gz", Arguments: []string{"-d", "-k", "-f", "%e"}, ExitCodes: []int{0}} +} +func extLzma() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "xz", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extTar() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "tar", Extension: "tar", Arguments: []string{"xf", "%e"}, ExitCodes: []int{0}} +} +func extBzip2() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "bzip2", Extension: "bz2", Arguments: []string{"-d", "-k", "-f", "%e"}, ExitCodes: []int{0}} +} +func extZstd() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "zstd", Extension: "zst", Arguments: []string{"-d", "-f", "%e"}, ExitCodes: []int{0}} +} +func extLz4() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "lz4", Extension: "lz4", Arguments: []string{"-d", "-f", "%e", "decompressed.bin"}, ExitCodes: []int{0}} +} +func extLzop() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "lzop", Extension: "lzo", Arguments: []string{"-d", "-f", "%e"}, ExitCodes: []int{0}} +} +func extLzfse() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "lzfse", Extension: "lzfse", Arguments: []string{"-decode", "-i", "%e", "-o", "decompressed.bin"}, ExitCodes: []int{0}} +} +func extZlib() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extSquashfs() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "unsquashfs", Extension: "sqsh", Arguments: []string{"-f", "-d", "squashfs-root", "%e"}, ExitCodes: []int{0}} +} +func extJffs2() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "jefferson", Extension: "jffs2", Arguments: []string{"-d", "jffs2-root", "%e"}, ExitCodes: []int{0}} +} +func extYaffs2() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "unyaffs", Extension: "yaffs", Arguments: []string{"%e", "yaffs-root"}, ExitCodes: []int{0}} +} +func extRomfs() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "romfs", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extTsk() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "tsk_recover", Extension: "img", Arguments: []string{"-a", "-e", "%e", "tsk-root"}, ExitCodes: []int{0}} +} +func extCab() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cabextract", Extension: "cab", Arguments: []string{"%e"}, ExitCodes: []int{0}} +} +func extRar() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "unrar", Extension: "rar", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0}} +} +func extUbi() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "ubireader_extract_images", Extension: "ubi", Arguments: []string{"%e"}, ExitCodes: []int{0}} +} +func extUbifs() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "ubireader_extract_files", Extension: "ubifs", Arguments: []string{"%e"}, ExitCodes: []int{0}} +} +func extPng() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "png", Arguments: []string{"%e", "image.png"}, ExitCodes: []int{0}} +} +func extJpeg() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "jpg", Arguments: []string{"%e", "image.jpg"}, ExitCodes: []int{0}} +} +func extBmp() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "bmp", Arguments: []string{"%e", "image.bmp"}, ExitCodes: []int{0}} +} +func extGif() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "gif", Arguments: []string{"%e", "image.gif"}, ExitCodes: []int{0}} +} +func extSvg() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "svg", Arguments: []string{"%e", "image.svg"}, ExitCodes: []int{0}} +} +func extRiff() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "riff", Arguments: []string{"%e", "media.riff"}, ExitCodes: []int{0}} +} +func extUimage() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extTrx() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extJboot() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extAutel() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extAndroidSparse() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "simg2img", Extension: "simg", Arguments: []string{"%e", "output.img"}, ExitCodes: []int{0}} +} +func extDtb() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "dtc", Extension: "dtb", Arguments: []string{"-I", "dtb", "-O", "dts", "-o", "device_tree.dts", "%e"}, ExitCodes: []int{0}} +} +func extEncfw() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "bin", Arguments: []string{"%e", "encrypted_firmware.bin"}, ExitCodes: []int{0}} +} +func extGpg() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "gpg", Extension: "gpg", Arguments: []string{"--decrypt", "-o", "decrypted.bin", "%e"}, ExitCodes: []int{0}} +} +func extPemCert() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pem", Arguments: []string{"%e", "certificate.pem"}, ExitCodes: []int{0}} +} +func extPemKey() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pem", Arguments: []string{"%e", "key.pem"}, ExitCodes: []int{0}} +} +func extLinuxKernel() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extMbr() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "bin", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extUefi() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "uefi-firmware-parser", Extension: "bin", Arguments: []string{"-e", "-O", "%e"}, ExitCodes: []int{0}} +} +func extPcap() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "cp", Extension: "pcapng", Arguments: []string{"%e", "capture.pcapng"}, ExitCodes: []int{0}} +} +func extSrec() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "srec_cat", Extension: "srec", Arguments: []string{"%e", "-o", "output.bin", "-binary"}, ExitCodes: []int{0}} +} +func extDmg() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "dmg", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} +func extDumpifs() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "dumpifs", Extension: "ifs", Arguments: []string{"-x", "%e"}, ExitCodes: []int{0}} +} +func extIso9660() *Extractor { + return &Extractor{Type: ExtractorTypeExternal, Command: "7z", Extension: "iso", Arguments: []string{"x", "-y", "%e"}, ExitCodes: []int{0, 2}} +} diff --git a/internal/leviathan/binsnoop/match.go b/internal/leviathan/binsnoop/match.go new file mode 100644 index 0000000..264820a --- /dev/null +++ b/internal/leviathan/binsnoop/match.go @@ -0,0 +1,134 @@ +package binsnoop + +// ahoCorasick implements the Aho-Corasick multi-pattern byte matching algorithm. +type ahoCorasick struct { + gotoFn []map[byte]int + failFn []int + outputFn [][]int + patterns [][]byte +} + +// ahoMatch represents a single pattern match. +type ahoMatch struct { + PatternIndex int + Start int +} + +// newAhoCorasick builds an Aho-Corasick automaton from the given byte patterns. +func newAhoCorasick(patterns [][]byte) *ahoCorasick { + ac := &ahoCorasick{ + gotoFn: []map[byte]int{make(map[byte]int)}, + failFn: []int{0}, + outputFn: [][]int{nil}, + patterns: patterns, + } + + // Build the trie (goto function) + for i, pattern := range patterns { + state := 0 + for _, b := range pattern { + if next, ok := ac.gotoFn[state][b]; ok { + state = next + } else { + newState := len(ac.gotoFn) + ac.gotoFn[state][b] = newState + ac.gotoFn = append(ac.gotoFn, make(map[byte]int)) + ac.failFn = append(ac.failFn, 0) + ac.outputFn = append(ac.outputFn, nil) + state = newState + } + } + ac.outputFn[state] = append(ac.outputFn[state], i) + } + + // Build failure links (BFS) + queue := make([]int, 0) + for _, next := range ac.gotoFn[0] { + queue = append(queue, next) + } + + for len(queue) > 0 { + r := queue[0] + queue = queue[1:] + + for b, s := range ac.gotoFn[r] { + queue = append(queue, s) + state := ac.failFn[r] + for { + if _, ok := ac.gotoFn[state][b]; ok || state == 0 { + break + } + state = ac.failFn[state] + } + if next, ok := ac.gotoFn[state][b]; ok && next != s { + ac.failFn[s] = next + } else { + ac.failFn[s] = 0 + } + // Merge output from failure state + if ac.outputFn[ac.failFn[s]] != nil { + ac.outputFn[s] = append(ac.outputFn[s], ac.outputFn[ac.failFn[s]]...) + } + } + } + + return ac +} + +// findOverlapping searches data starting at startOffset and returns all overlapping matches. +func (ac *ahoCorasick) findOverlapping(data []byte, startOffset int) []ahoMatch { + var matches []ahoMatch + state := 0 + + for i := startOffset; i < len(data); i++ { + b := data[i] + for { + if _, ok := ac.gotoFn[state][b]; ok || state == 0 { + break + } + state = ac.failFn[state] + } + if next, ok := ac.gotoFn[state][b]; ok { + state = next + } + for _, patIdx := range ac.outputFn[state] { + matchStart := i - len(ac.patterns[patIdx]) + 1 + matches = append(matches, ahoMatch{ + PatternIndex: patIdx, + Start: matchStart, + }) + } + } + + return matches +} + +// findOverlappingLimit is the bounded variant used by deep scans. It stops +// collecting once limit matches have been found so adversarial input cannot +// force an unbounded match slice. +func (ac *ahoCorasick) findOverlappingLimit(data []byte, startOffset, limit int) ([]ahoMatch, bool) { + if limit <= 0 { + return nil, false + } + matches := make([]ahoMatch, 0, min(limit, 256)) + state := 0 + for i := startOffset; i < len(data); i++ { + b := data[i] + for { + if _, ok := ac.gotoFn[state][b]; ok || state == 0 { + break + } + state = ac.failFn[state] + } + if next, ok := ac.gotoFn[state][b]; ok { + state = next + } + for _, patIdx := range ac.outputFn[state] { + matches = append(matches, ahoMatch{PatternIndex: patIdx, Start: i - len(ac.patterns[patIdx]) + 1}) + if len(matches) >= limit { + return matches, true + } + } + } + return matches, false +} diff --git a/internal/leviathan/binsnoop/parse.go b/internal/leviathan/binsnoop/parse.go new file mode 100644 index 0000000..4057cd4 --- /dev/null +++ b/internal/leviathan/binsnoop/parse.go @@ -0,0 +1,107 @@ +package binsnoop + +import ( + "errors" + "fmt" +) + +// ErrParse is returned when structure parsing fails. +var ErrParse = errors.New("structure parse error") + +// StructField defines a single field in a binary structure. +type StructField struct { + Name string + Type string // "u8", "u16", "u24", "u32", "u64" +} + +// typeSize returns the byte size for a type string. +func typeSize(ctype string) (int, error) { + switch ctype { + case "u8": + return 1, nil + case "u16": + return 2, nil + case "u24": + return 3, nil + case "u32": + return 4, nil + case "u64": + return 8, nil + default: + return 0, fmt.Errorf("unknown type: %s", ctype) + } +} + +// StructSize returns the total byte size of a structure definition. +func StructSize(fields []StructField) int { + total := 0 + for _, f := range fields { + sz, err := typeSize(f.Type) + if err != nil { + continue + } + total += sz + } + return total +} + +// Parse parses a binary structure from data according to the field definitions. +// Endianness must be "big" or "little". All values are returned as int. +func Parse(data []byte, fields []StructField, endianness string) (map[string]int, error) { + structSize := StructSize(fields) + if len(data) < structSize { + return nil, ErrParse + } + + result := make(map[string]int, len(fields)) + offset := 0 + + for _, f := range fields { + sz, err := typeSize(f.Type) + if err != nil { + return nil, ErrParse + } + + var value int + + switch sz { + case 1: + value = int(data[offset]) + case 2: + if endianness == "big" { + value = int(data[offset])<<8 | int(data[offset+1]) + } else { + value = int(data[offset+1])<<8 | int(data[offset]) + } + case 3: + if endianness == "big" { + value = int(data[offset])<<16 | int(data[offset+1])<<8 | int(data[offset+2]) + } else { + value = int(data[offset+2])<<16 | int(data[offset+1])<<8 | int(data[offset]) + } + case 4: + if endianness == "big" { + value = int(data[offset])<<24 | int(data[offset+1])<<16 | int(data[offset+2])<<8 | int(data[offset+3]) + } else { + value = int(data[offset+3])<<24 | int(data[offset+2])<<16 | int(data[offset+1])<<8 | int(data[offset]) + } + case 8: + if endianness == "big" { + value = int(uint64(data[offset])<<56 | uint64(data[offset+1])<<48 | + uint64(data[offset+2])<<40 | uint64(data[offset+3])<<32 | + uint64(data[offset+4])<<24 | uint64(data[offset+5])<<16 | + uint64(data[offset+6])<<8 | uint64(data[offset+7])) + } else { + value = int(uint64(data[offset+7])<<56 | uint64(data[offset+6])<<48 | + uint64(data[offset+5])<<40 | uint64(data[offset+4])<<32 | + uint64(data[offset+3])<<24 | uint64(data[offset+2])<<16 | + uint64(data[offset+1])<<8 | uint64(data[offset])) + } + } + + offset += sz + result[f.Name] = value + } + + return result, nil +} diff --git a/internal/leviathan/binsnoop/scanner.go b/internal/leviathan/binsnoop/scanner.go new file mode 100644 index 0000000..4afb117 --- /dev/null +++ b/internal/leviathan/binsnoop/scanner.go @@ -0,0 +1,352 @@ +package binsnoop + +import "sort" + +// Scanner is the core scanning engine. It holds compiled signature patterns +// and performs Aho-Corasick based multi-pattern matching on binary data. +type Scanner struct { + signatureCount int + patternCount int + shortSignatures []Signature + patterns [][]byte + patternSigTable map[int]Signature + extractorLookup map[string]*Extractor + ac *ahoCorasick +} + +// compile builds the Aho-Corasick automaton from loaded patterns. +func (s *Scanner) compile() { + if len(s.patterns) > 0 { + s.ac = newAhoCorasick(s.patterns) + } +} + +// Scan searches data for all known file signatures and returns validated results. +// This implements the same algorithm as the Rust binwalk scanner: +// 1. Check short signatures at file offset 0 +// 2. Run Aho-Corasick scan with restart on high-confidence sized matches +// 3. Sort and de-conflict overlapping results +// 4. Estimate sizes for signatures with unknown size +func (s *Scanner) Scan(data []byte) []SignatureResult { + var fileMap []SignatureResult + nextValidOffset := 0 + availableData := len(data) + + // Phase 1: Check short signatures at offset 0 + for i := range s.shortSignatures { + sig := &s.shortSignatures[i] + for _, magic := range sig.Magic { + magicStart := sig.MagicOffset + magicEnd := magicStart + len(magic) + + if len(data) >= magicEnd && bytesEqual(data[magicStart:magicEnd], magic) { + result, err := sig.Parser(data, magicStart) + if err != nil { + continue + } + autoPopulate(result, sig) + fileMap = append(fileMap, *result) + + if result.Confidence >= ConfidenceHigh { + nextValidOffset = result.Offset + result.Size + } + break + } + } + } + + // Phase 2: Aho-Corasick scan. + // + // The automaton is run exactly once over the whole buffer; we then walk + // the matches in position order. When a high/medium-confidence sized + // signature is accepted, nextValidOffset jumps past it and any later + // matches that start inside that consumed region are skipped. This is the + // same result the binwalk scanner produces, but without re-running the + // search from every new offset — a previous implementation rescanned the + // tail of the file on each accepted match, which is O(n^2) on inputs that + // contain many closely-spaced signatures (e.g. real firmware images) and + // could appear to hang. + if s.ac == nil { + return s.postProcess(data, fileMap) + } + + matches := s.ac.findOverlapping(data, nextValidOffset) + for _, m := range matches { + magicOffset := m.Start + if magicOffset < nextValidOffset { + continue + } + + sig, ok := s.patternSigTable[m.PatternIndex] + if !ok { + continue + } + + result, err := sig.Parser(data, magicOffset) + if err != nil { + continue + } + + sigEnd := result.Offset + result.Size + if sigEnd > availableData { + continue + } + + autoPopulate(result, &sig) + fileMap = append(fileMap, *result) + + if result.Confidence >= ConfidenceMedium && result.Size > 0 { + nextValidOffset = sigEnd + } + } + + return s.postProcess(data, fileMap) +} + +// postProcess sorts results, removes conflicts, and estimates unknown sizes. +func (s *Scanner) postProcess(data []byte, fileMap []SignatureResult) []SignatureResult { + if len(fileMap) == 0 { + return fileMap + } + + // Sort by offset + sort.Slice(fileMap, func(i, j int) bool { + return fileMap[i].Offset < fileMap[j].Offset + }) + + // De-conflict overlapping signatures in a single pass, filtering in place. + // (The previous implementation removed elements with + // append(fileMap[:i], fileMap[i+1:]...) inside the loop; each removal is + // O(n), so a file with many overlapping hits was O(n^2) and could appear + // to hang.) Removed entries are dropped; on a duplicate offset the + // higher-confidence signature wins, matching the original behaviour. + out := fileMap[:0] + nextValidOffset := 0 + for _, thisSig := range fileMap { + // Duplicate offset: keep only the highest-confidence signature. + if len(out) > 0 && thisSig.Offset == out[len(out)-1].Offset { + if thisSig.Confidence > out[len(out)-1].Confidence { + out[len(out)-1] = thisSig + if thisSig.Confidence >= ConfidenceMedium { + nextValidOffset = thisSig.Offset + thisSig.Size + } + } + continue + } + + // Drop signatures that fall inside a previously accepted sized region. + if thisSig.Offset < nextValidOffset { + continue + } + + // Drop signatures whose declared size runs past EOF or wraps. + remainingSize := len(data) - thisSig.Offset + if thisSig.Size > remainingSize || (thisSig.Offset+thisSig.Size) < 0 { + continue + } + + out = append(out, thisSig) + if thisSig.Confidence >= ConfidenceMedium { + nextValidOffset = thisSig.Offset + thisSig.Size + } + } + fileMap = out + + // Estimate sizes for signatures with unknown size (0): extend each to the + // offset of the next medium-or-higher confidence signature, or to EOF. + // Computed in a single right-to-left pass to stay O(n). + nextMediumOffset := len(data) + for i := len(fileMap) - 1; i >= 0; i-- { + if fileMap[i].Size == 0 { + fileMap[i].Size = nextMediumOffset - fileMap[i].Offset + } + if fileMap[i].Confidence >= ConfidenceMedium { + nextMediumOffset = fileMap[i].Offset + } + } + + return fileMap +} + +// ScanDeep validates signatures without suppressing hits inside an already +// identified container. Results are bounded and annotated with containment. +func (s *Scanner) ScanDeep(data []byte, maxResults int) DeepScan { + if maxResults <= 0 || maxResults > 4096 { + maxResults = 4096 + } + result := DeepScan{} + if len(data) == 0 { + return result + } + + results := make([]SignatureResult, 0) + seen := make(map[struct { + offset int + name string + }]int) + add := func(parsed *SignatureResult, sig *Signature) { + if parsed == nil || parsed.Offset < 0 || parsed.Offset > len(data) { + return + } + if parsed.Size < 0 || parsed.Size > len(data)-parsed.Offset { + return + } + autoPopulate(parsed, sig) + key := struct { + offset int + name string + }{parsed.Offset, parsed.Name} + if idx, ok := seen[key]; ok { + if parsed.Confidence > results[idx].Confidence { + results[idx] = *parsed + } + return + } + if len(results) >= maxResults { + result.Truncated = true + return + } + seen[key] = len(results) + results = append(results, *parsed) + } + + for i := range s.shortSignatures { + sig := &s.shortSignatures[i] + for _, magic := range sig.Magic { + start := sig.MagicOffset + end := start + len(magic) + if start >= 0 && end <= len(data) && bytesEqual(data[start:end], magic) { + parsed, err := sig.Parser(data, start) + if err == nil { + add(parsed, sig) + } + break + } + } + } + + if s.ac != nil && len(results) < maxResults { + matchLimit := maxResults * 16 + matches, truncated := s.ac.findOverlappingLimit(data, 0, matchLimit) + result.Truncated = result.Truncated || truncated + for _, match := range matches { + sig, ok := s.patternSigTable[match.PatternIndex] + if !ok { + continue + } + parsed, err := sig.Parser(data, match.Start) + if err != nil { + continue + } + add(parsed, &sig) + if len(results) >= maxResults { + result.Truncated = true + break + } + } + } + + sort.Slice(results, func(i, j int) bool { + if results[i].Offset != results[j].Offset { + return results[i].Offset < results[j].Offset + } + if results[i].Confidence != results[j].Confidence { + return results[i].Confidence > results[j].Confidence + } + return results[i].Size > results[j].Size + }) + unique := results[:0] + for _, hit := range results { + if len(unique) > 0 && unique[len(unique)-1].Offset == hit.Offset { + continue + } + unique = append(unique, hit) + } + results = unique + + nextOffset := len(data) + for i := len(results) - 1; i >= 0; i-- { + if results[i].Size == 0 { + results[i].Size = nextOffset - results[i].Offset + } + nextOffset = results[i].Offset + } + + active := make([]int, 0) + for _, hit := range results { + end := hit.Offset + hit.Size + kept := active[:0] + for _, idx := range active { + if result.Results[idx].End > hit.Offset { + kept = append(kept, idx) + } + } + active = kept + parent := -1 + overlap := false + for i := len(active) - 1; i >= 0; i-- { + candidate := result.Results[active[i]] + if candidate.End >= end && candidate.Offset < hit.Offset { + parent = active[i] + break + } + overlap = true + } + deep := DeepScanResult{SignatureResult: hit, End: end, ParentOffset: -1, Overlap: overlap} + if parent >= 0 { + deep.ParentOffset = result.Results[parent].Offset + deep.Depth = result.Results[parent].Depth + 1 + } + result.Results = append(result.Results, deep) + active = append(active, len(result.Results)-1) + } + return result +} + +// Extract performs extraction on all extractable signatures in the file map. +func (s *Scanner) Extract(data []byte, filePath string, fileMap []SignatureResult) map[string]*ExtractionResult { + results := make(map[string]*ExtractionResult) + + for _, sig := range fileMap { + if sig.ExtractionDeclined { + continue + } + + ext, ok := s.extractorLookup[sig.Name] + if !ok || ext == nil { + continue + } + + result := ExecuteExtraction(data, filePath, &sig, ext) + if !result.Success && sig.Size < len(data)-sig.Offset { + // Retry with all available data + expandedSig := sig + expandedSig.Size = len(data) - sig.Offset + result = ExecuteExtraction(data, filePath, &expandedSig, ext) + } + + results[sig.ID] = result + } + + return results +} + +// bytesEqual compares two byte slices. +func bytesEqual(a, b []byte) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +// autoPopulate fills in auto-generated fields on a SignatureResult. +func autoPopulate(result *SignatureResult, sig *Signature) { + result.ID = newUUID() + result.Name = sig.Name + result.AlwaysDisplay = sig.AlwaysDisplay +} diff --git a/internal/leviathan/binsnoop/sig_archives.go b/internal/leviathan/binsnoop/sig_archives.go new file mode 100644 index 0000000..b010c5a --- /dev/null +++ b/internal/leviathan/binsnoop/sig_archives.go @@ -0,0 +1,1143 @@ +package binsnoop + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + "strconv" + "strings" +) + +// --------------------------------------------------------------------------- +// gzip +// --------------------------------------------------------------------------- + +const gzipDescription = "gzip compressed data" + +func gzipMagic() [][]byte { + return [][]byte{ + {0x1f, 0x8b, 0x08}, + } +} + +func gzipParser(data []byte, offset int) (*SignatureResult, error) { + const ( + flagCRC = 0b0000_0010 + flagExtra = 0b0000_0100 + flagName = 0b0000_1000 + flagComment = 0b0001_0000 + flagReserved = 0b1110_0000 + + deflateCompression = 8 + crcSize = 2 + nullByteSize = 1 + ) + + knownOSIDs := map[int]string{ + 0: "FAT filesystem (MS-DOS, OS/2, NT/Win32", + 1: "Amiga", + 2: "VMS (or OpenVMS)", + 3: "Unix", + 4: "VM/CMS", + 5: "Atari TOS", + 6: "HPFS filesystem (OS/2, NT)", + 7: "Macintosh", + 8: "Z-System", + 9: "CP/M", + 10: "TOPS-20", + 11: "NTFS filesystem (NT)", + 12: "QDOS", + 13: "Acorn RISCOS", + 255: "unknown", + } + + gzipHeaderFields := []StructField{ + {"magic", "u16"}, + {"compression_method", "u8"}, + {"flags", "u8"}, + {"timestamp", "u32"}, + {"extra_flags", "u8"}, + {"osid", "u8"}, + } + + gzipExtraHeaderFields := []StructField{ + {"id", "u16"}, + {"extra_data_len", "u16"}, + } + + if offset >= len(data) { + return nil, fmt.Errorf("gzip: offset out of bounds") + } + + headerData := data[offset:] + hdr, err := Parse(headerData, gzipHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("gzip: failed to parse header") + } + + if hdr["flags"]&flagReserved != 0 { + return nil, fmt.Errorf("gzip: reserved flags set") + } + if hdr["compression_method"] != deflateCompression { + return nil, fmt.Errorf("gzip: invalid compression method %d", hdr["compression_method"]) + } + osName, ok := knownOSIDs[hdr["osid"]] + if !ok { + return nil, fmt.Errorf("gzip: unknown OS ID %d", hdr["osid"]) + } + + headerSize := StructSize(gzipHeaderFields) + + // Handle optional extra data + if hdr["flags"]&flagExtra != 0 { + extraHeaderSize := StructSize(gzipExtraHeaderFields) + extraStart := headerSize + extraEnd := extraStart + extraHeaderSize + if extraEnd > len(headerData) { + return nil, fmt.Errorf("gzip: not enough data for extra header") + } + extraHdr, err := Parse(headerData[extraStart:], gzipExtraHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("gzip: failed to parse extra header") + } + headerSize += extraHeaderSize + extraHdr["extra_data_len"] + } + + // Handle optional original file name (NULL-terminated) + originalName := "" + if hdr["flags"]&flagName != 0 { + if headerSize >= len(headerData) { + return nil, fmt.Errorf("gzip: not enough data for original name") + } + originalName = GetCString(headerData[headerSize:]) + headerSize += len(originalName) + nullByteSize + } + + // Handle optional comment (NULL-terminated) + if hdr["flags"]&flagComment != 0 { + if headerSize >= len(headerData) { + return nil, fmt.Errorf("gzip: not enough data for comment") + } + comment := GetCString(headerData[headerSize:]) + headerSize += len(comment) + nullByteSize + } + + // Handle optional header CRC + if hdr["flags"]&flagCRC != 0 { + headerSize += crcSize + } + + if len(headerData) < headerSize { + return nil, fmt.Errorf("gzip: data shorter than header") + } + + originalFileNameText := "" + if originalName != "" { + originalFileNameText = fmt.Sprintf(" original file name: \"%s\",", originalName) + } + + description := fmt.Sprintf("%s,%s operating system: %s, timestamp: %s", + gzipDescription, + originalFileNameText, + osName, + EpochToString(uint32(hdr["timestamp"])), + ) + + return &SignatureResult{ + Offset: offset, + Size: 0, // size unknown without decompression + Description: description, + Confidence: ConfidenceMedium, + }, nil +} + +// --------------------------------------------------------------------------- +// zip +// --------------------------------------------------------------------------- + +const zipDescription = "ZIP archive" + +func zipMagic() [][]byte { + return [][]byte{ + {0x50, 0x4b, 0x03, 0x04}, // PK\x03\x04 + } +} + +func zipParser(data []byte, offset int) (*SignatureResult, error) { + return parseZipLike(data, offset, zipDescription) +} + +// parseZipLike is shared between zip and dahua_zip parsers. +func parseZipLike(data []byte, offset int, descPrefix string) (*SignatureResult, error) { + const ( + unusedFlagsMask = 0b11010111_10000000 + compressionEncrypted = 99 + ) + + zipLocalFileFields := []StructField{ + {"magic", "u32"}, + {"version", "u16"}, + {"flags", "u16"}, + {"compression", "u16"}, + {"modification_time", "u16"}, + {"modification_date", "u16"}, + {"crc", "u32"}, + {"compressed_size", "u32"}, + {"uncompressed_size", "u32"}, + {"file_name_len", "u16"}, + {"extra_field_len", "u16"}, + } + + allowedCompressionMethods := map[int]bool{ + 0: true, 1: true, 2: true, 3: true, 4: true, 5: true, 6: true, + 8: true, 9: true, 10: true, 12: true, 14: true, 18: true, + 19: true, 20: true, 93: true, 94: true, 95: true, 96: true, + 97: true, 98: true, compressionEncrypted: true, + } + + if offset >= len(data) { + return nil, fmt.Errorf("zip: offset out of bounds") + } + + zipData := data[offset:] + hdr, err := Parse(zipData, zipLocalFileFields, "little") + if err != nil { + return nil, fmt.Errorf("zip: failed to parse local file header") + } + + if hdr["flags"]&unusedFlagsMask != 0 { + return nil, fmt.Errorf("zip: unused flags set") + } + if !allowedCompressionMethods[hdr["compression"]] { + return nil, fmt.Errorf("zip: unsupported compression method %d", hdr["compression"]) + } + + versionMajor := hdr["version"] / 10 + versionMinor := hdr["version"] % 10 + + localHeaderSize := StructSize(zipLocalFileFields) + hdr["file_name_len"] + hdr["extra_field_len"] + dataSize := hdr["compressed_size"] + if dataSize == 0 { + dataSize = hdr["uncompressed_size"] + } + firstEntryTotalSize := localHeaderSize + dataSize + + // Try to find the end-of-central-directory header + eocdMagic := []byte("PK\x05\x06\x00\x00\x00\x00") + eocdFields := []StructField{ + {"magic", "u32"}, + {"disk_number", "u16"}, + {"central_directory_disk_number", "u16"}, + {"central_directory_disk_entries", "u16"}, + {"central_directory_total_entries", "u16"}, + {"central_directory_size", "u32"}, + {"central_directory_offset", "u32"}, + {"comment_length", "u16"}, + } + + totalSize := 0 + fileCount := 0 + missingEOCD := false + + // Search for EOCD in data after offset + searchData := data[offset:] + eocdIdx := -1 + for i := 0; i+len(eocdMagic) <= len(searchData); i++ { + if bytes.Equal(searchData[i:i+len(eocdMagic)], eocdMagic) { + eocdData := searchData[i:] + eocdHdr, err := Parse(eocdData, eocdFields, "little") + if err != nil { + continue + } + if eocdHdr["central_directory_disk_entries"] == eocdHdr["central_directory_total_entries"] && + eocdHdr["central_directory_total_entries"] > 0 { + eocdSize := StructSize(eocdFields) + eocdHdr["comment_length"] + eocdIdx = i + totalSize = eocdIdx + eocdSize + fileCount = eocdHdr["central_directory_total_entries"] + break + } + } + } + + if eocdIdx < 0 { + // No EOCD found — walk local file entries + missingEOCD = true + availableData := len(data) - offset + var previousOffset *int + nextOffset := offset + firstEntryTotalSize + + for IsOffsetSafe(availableData, nextOffset, previousOffset) { + entryHdr, err := Parse(data[nextOffset:], zipLocalFileFields, "little") + if err != nil { + totalSize = nextOffset - offset + break + } + if entryHdr["flags"]&unusedFlagsMask != 0 || !allowedCompressionMethods[entryHdr["compression"]] { + totalSize = nextOffset - offset + break + } + entryHeaderSize := StructSize(zipLocalFileFields) + entryHdr["file_name_len"] + entryHdr["extra_field_len"] + entryDataSize := entryHdr["compressed_size"] + if entryDataSize == 0 { + entryDataSize = entryHdr["uncompressed_size"] + } + prev := nextOffset + previousOffset = &prev + nextOffset += entryHeaderSize + entryDataSize + } + if totalSize == 0 { + totalSize = nextOffset - offset + } + } + + if totalSize <= 0 { + return nil, fmt.Errorf("zip: could not determine archive size") + } + + var description string + if missingEOCD { + description = fmt.Sprintf("%s, version: %d.%d, missing end-of-central-directory header, total size: %d bytes", + descPrefix, versionMajor, versionMinor, totalSize) + } else { + description = fmt.Sprintf("%s, version: %d.%d, file count: %d, total size: %d bytes", + descPrefix, versionMajor, versionMinor, fileCount, totalSize) + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: description, + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// tarball +// --------------------------------------------------------------------------- + +const tarballDescription = "POSIX tar archive" + +const ( + tarballBlockSize = 512 + tarballMagicOffset = 257 + tarballMagicSize = 5 + tarballSizeOffset = 124 + tarballSizeLen = 11 + tarballMinExpectedHdrs = 10 + tarballChecksumStart = 148 + tarballChecksumEnd = 156 +) + +var tarballUniversalMagic = []byte("ustar") + +func tarballMagic() [][]byte { + return [][]byte{ + {0x75, 0x73, 0x74, 0x61, 0x72, 0x00}, // ustar\x00 + {0x75, 0x73, 0x74, 0x61, 0x72, 0x20, 0x20, 0x00}, // ustar\x20\x20\x00 + } +} + +func tarballParser(data []byte, offset int) (*SignatureResult, error) { + // The magic bytes are at offset 257 within a tar header block + tarballStartOffset := offset - tarballMagicOffset + + if tarballStartOffset < 0 || tarballStartOffset >= len(data) { + return nil, fmt.Errorf("tarball: invalid start offset") + } + + var tarballTotalSize int + var validHeaderCount int + nextHeaderStart := tarballStartOffset + var previousHeaderStart *int + availableData := len(data) + + for IsOffsetSafe(availableData, nextHeaderStart, previousHeaderStart) { + nextHeaderEnd := nextHeaderStart + tarballBlockSize + if nextHeaderEnd > len(data) { + break + } + headerBlock := data[nextHeaderStart:nextHeaderEnd] + + if !tarballHeaderChecksumValid(headerBlock) { + break + } + + validHeaderCount++ + + entrySize, err := tarballEntrySize(headerBlock) + if err != nil { + break + } + + tarballTotalSize += entrySize + prev := nextHeaderStart + previousHeaderStart = &prev + nextHeaderStart += entrySize + } + + if tarballTotalSize < tarballBlockSize { + return nil, fmt.Errorf("tarball: archive too small") + } + + confidence := ConfidenceMedium + if validHeaderCount >= tarballMinExpectedHdrs { + confidence = ConfidenceHigh + } + + return &SignatureResult{ + Offset: tarballStartOffset, + Size: tarballTotalSize, + Description: fmt.Sprintf("%s, file count: %d", tarballDescription, validHeaderCount), + Confidence: confidence, + }, nil +} + +func tarballHeaderChecksumValid(headerBlock []byte) bool { + checksumValueBytes := headerBlock[tarballChecksumStart:tarballChecksumEnd] + reportedChecksum := tarballOctal(checksumValueBytes) + var sum int + for i, b := range headerBlock { + if i >= tarballChecksumStart && i < tarballChecksumEnd { + sum += 0x20 + } else { + sum += int(b) + } + } + return sum == reportedChecksum +} + +func tarballEntrySize(entryData []byte) (int, error) { + entryMagic := entryData[tarballMagicOffset : tarballMagicOffset+tarballMagicSize] + if !bytes.Equal(entryMagic, tarballUniversalMagic) { + return 0, fmt.Errorf("tarball: invalid entry magic") + } + + entrySizeBytes := entryData[tarballSizeOffset : tarballSizeOffset+tarballSizeLen] + reportedEntrySize := tarballOctal(entrySizeBytes) + + blockCount := 1 + int(math.Ceil(float64(reportedEntrySize)/float64(tarballBlockSize))) + return blockCount * tarballBlockSize, nil +} + +func tarballOctal(octalStr []byte) int { + var num int + for _, c := range octalStr { + if c < 0x30 || c > 0x39 { + break + } + num = num*8 + int(c) - 0x30 + } + return num +} + +// --------------------------------------------------------------------------- +// 7-zip (sevenzip) +// --------------------------------------------------------------------------- + +const sevenzipDescription = "7-zip archive data" + +func sevenzipMagic() [][]byte { + return [][]byte{ + {0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c}, // 7z\xbc\xaf\x27\x1c + } +} + +func sevenzipParser(data []byte, offset int) (*SignatureResult, error) { + const ( + sevenzipCRCStart = 12 + sevenzipHeaderSize = 32 + ) + + sevenzipFields := []StructField{ + {"magic_p1", "u16"}, + {"magic_p2", "u32"}, + {"major_version", "u8"}, + {"minor_version", "u8"}, + {"header_crc", "u32"}, + {"next_header_offset", "u64"}, + {"next_header_size", "u64"}, + {"next_header_crc", "u32"}, + } + + if offset >= len(data) { + return nil, fmt.Errorf("7zip: offset out of bounds") + } + + szData := data[offset:] + hdr, err := Parse(szData, sevenzipFields, "little") + if err != nil { + return nil, fmt.Errorf("7zip: failed to parse header") + } + + // Validate the header CRC (over bytes 12..32) + if sevenzipHeaderSize > len(szData) { + return nil, fmt.Errorf("7zip: not enough data for header CRC") + } + crcData := szData[sevenzipCRCStart:sevenzipHeaderSize] + calculatedHeaderCRC := CRC32(crcData) + if calculatedHeaderCRC != uint32(hdr["header_crc"]) { + return nil, fmt.Errorf("7zip: header CRC mismatch") + } + + // Validate the next_header CRC + nextCRCStart := offset + sevenzipHeaderSize + hdr["next_header_offset"] + nextCRCEnd := nextCRCStart + hdr["next_header_size"] + if nextCRCEnd > len(data) { + return nil, fmt.Errorf("7zip: not enough data for next header CRC") + } + nextCRCData := data[nextCRCStart:nextCRCEnd] + calculatedNextCRC := CRC32(nextCRCData) + if int(calculatedNextCRC) != hdr["next_header_crc"] { + return nil, fmt.Errorf("7zip: next header CRC mismatch") + } + + totalSize := sevenzipHeaderSize + hdr["next_header_offset"] + hdr["next_header_size"] + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, version %d.%d, total size: %d bytes", sevenzipDescription, hdr["major_version"], hdr["minor_version"], totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// RAR +// --------------------------------------------------------------------------- + +const rarDescription = "RAR archive" + +func rarMagic() [][]byte { + return [][]byte{ + {0x52, 0x61, 0x72, 0x21, 0x1a, 0x07}, // Rar!\x1A\x07 + } +} + +func rarParser(data []byte, offset int) (*SignatureResult, error) { + archiveHeaderFields := []StructField{ + {"magic_p1", "u32"}, + {"magic_p2", "u16"}, + {"version", "u8"}, + } + + // version field 0 => RARv4, 1 => RARv5 + versionMap := map[int]int{0: 4, 1: 5} + + if offset >= len(data) { + return nil, fmt.Errorf("rar: offset out of bounds") + } + + hdr, err := Parse(data[offset:], archiveHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("rar: failed to parse header") + } + + version, ok := versionMap[hdr["version"]] + if !ok { + return nil, fmt.Errorf("rar: unknown version %d", hdr["version"]) + } + + // EOF markers for RARv4 and RARv5 + eofMarkers := map[int][]byte{ + 4: {0xc4, 0x3d, 0x7b, 0x00, 0x40, 0x07, 0x00}, + 5: {0x1d, 0x77, 0x56, 0x51, 0x03, 0x05, 0x04, 0x00}, + } + + totalSize := 0 + extraDesc := "" + + if marker, ok := eofMarkers[version]; ok { + searchData := data[offset:] + idx := bytes.Index(searchData, marker) + if idx >= 0 { + totalSize = idx + len(marker) + } else { + extraDesc = " (failed to locate RAR EOF)" + } + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, version: %d, total size: %d bytes%s", rarDescription, version, totalSize, extraDesc), + Confidence: ConfidenceMedium, + }, nil +} + +// --------------------------------------------------------------------------- +// CAB (Microsoft Cabinet) +// --------------------------------------------------------------------------- + +const cabDescription = "Microsoft Cabinet archive" + +func cabMagic() [][]byte { + return [][]byte{ + {0x4d, 0x53, 0x43, 0x46, 0x00, 0x00, 0x00, 0x00}, // MSCF\x00\x00\x00\x00 + } +} + +func cabParser(data []byte, offset int) (*SignatureResult, error) { + const ( + majorVersion = 1 + minorVersion = 3 + cabStructSize = 40 + cabExtraStructSize = 20 + flagExtraDataPresent = 4 + ) + + cabHeaderFields := []StructField{ + {"magic", "u32"}, + {"reserved1", "u32"}, + {"size", "u32"}, + {"reserved2", "u32"}, + {"first_file_offset", "u32"}, + {"reserved3", "u32"}, + {"minor_version", "u8"}, + {"major_version", "u8"}, + {"folder_count", "u16"}, + {"file_count", "u16"}, + {"flags", "u16"}, + {"id", "u16"}, + {"set_number", "u16"}, + {"extra_header_size", "u16"}, + {"cbCFFolder", "u8"}, + {"cbCFData", "u8"}, + } + + cabExtraHeaderFields := []StructField{ + {"unknown1", "u32"}, + {"data_offset", "u32"}, + {"data_size", "u32"}, + {"unknown2", "u32"}, + {"unknown3", "u32"}, + } + + if offset >= len(data) { + return nil, fmt.Errorf("cab: offset out of bounds") + } + + hdr, err := Parse(data[offset:], cabHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("cab: failed to parse header") + } + + // All reserved fields must be 0 + if hdr["reserved1"] != 0 || hdr["reserved2"] != 0 || hdr["reserved3"] != 0 { + return nil, fmt.Errorf("cab: reserved fields not zero") + } + + // Version must be 1.3 + if hdr["major_version"] != majorVersion || hdr["minor_version"] != minorVersion { + return nil, fmt.Errorf("cab: invalid version %d.%d", hdr["major_version"], hdr["minor_version"]) + } + + totalSize := hdr["size"] + headerSize := cabStructSize + + // Handle extra data flag + if hdr["flags"]&flagExtraDataPresent != 0 && hdr["extra_header_size"] == cabExtraStructSize { + extraStart := cabStructSize + extraEnd := extraStart + cabExtraStructSize + if offset+extraEnd > len(data) { + return nil, fmt.Errorf("cab: not enough data for extra header") + } + extraHdr, err := Parse(data[offset+extraStart:], cabExtraHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("cab: failed to parse extra header") + } + if extraHdr["data_offset"] != hdr["size"] { + return nil, fmt.Errorf("cab: extra data offset mismatch") + } + totalSize += extraHdr["data_size"] + _ = headerSize + } else if hdr["flags"]&flagExtraDataPresent != 0 { + return nil, fmt.Errorf("cab: extra data flag set but wrong extra header size") + } + + availableData := len(data) - offset + if totalSize > availableData { + return nil, fmt.Errorf("cab: reported size exceeds available data") + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, file count: %d, folder count: %d, header size: %d, total size: %d bytes", cabDescription, hdr["file_count"], hdr["folder_count"], cabStructSize, totalSize), + Confidence: ConfidenceMedium, + }, nil +} + +// --------------------------------------------------------------------------- +// Debian package (deb) +// --------------------------------------------------------------------------- + +const debDescription = "Debian package file" + +func debMagic() [][]byte { + return [][]byte{ + // "!\ndebian-binary " + {0x21, 0x3c, 0x61, 0x72, 0x63, 0x68, 0x3e, 0x0a, + 0x64, 0x65, 0x62, 0x69, 0x61, 0x6e, 0x2d, 0x62, + 0x69, 0x6e, 0x61, 0x72, 0x79, 0x20, 0x20, 0x20}, + } +} + +func debParser(data []byte, offset int) (*SignatureResult, error) { + const ( + endMarkerSize = 2 + dataFileSizeLen = 10 + dataFileSizeOffset = 48 + controlFileSizeEnd = 130 + controlFileSizeStart = 120 + ) + + if offset >= len(data) { + return nil, fmt.Errorf("deb: offset out of bounds") + } + + debData := data[offset:] + + if len(debData) < controlFileSizeEnd { + return nil, fmt.Errorf("deb: not enough data for control file size") + } + + // Read control file size as ASCII decimal + controlFileSizeBytes := debData[controlFileSizeStart:controlFileSizeEnd] + controlFileSizeStr := strings.TrimSpace(string(controlFileSizeBytes)) + controlFileSize, err := strconv.Atoi(controlFileSizeStr) + if err != nil { + return nil, fmt.Errorf("deb: invalid control file size: %s", controlFileSizeStr) + } + + // Calculate data file size location + dataFileSizeStart := controlFileSizeEnd + endMarkerSize + controlFileSize + dataFileSizeOffset + dataFileSizeEnd := dataFileSizeStart + dataFileSizeLen + + if dataFileSizeEnd > len(debData) { + return nil, fmt.Errorf("deb: not enough data for data file size") + } + + dataFileSizeBytes := debData[dataFileSizeStart:dataFileSizeEnd] + dataFileSizeStr := strings.TrimSpace(string(dataFileSizeBytes)) + dataFileSize, err := strconv.Atoi(dataFileSizeStr) + if err != nil { + return nil, fmt.Errorf("deb: invalid data file size: %s", dataFileSizeStr) + } + + totalSize := dataFileSizeEnd + endMarkerSize + dataFileSize + + if totalSize > len(debData) { + return nil, fmt.Errorf("deb: reported size exceeds available data") + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, total size: %d bytes", debDescription, totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// CPIO +// --------------------------------------------------------------------------- + +const cpioDescription = "CPIO ASCII archive" + +const cpioHeaderSize = 110 + +func cpioMagic() [][]byte { + return [][]byte{ + {0x30, 0x37, 0x30, 0x37, 0x30, 0x31}, // 070701 + {0x30, 0x37, 0x30, 0x37, 0x30, 0x32}, // 070702 + } +} + +func cpioParser(data []byte, offset int) (*SignatureResult, error) { + const eofMarker = "TRAILER!!!" + + if offset >= len(data) { + return nil, fmt.Errorf("cpio: offset out of bounds") + } + + validMagics := cpioMagic() + + var headerCount int + var totalSize int + nextHeaderOffset := offset + var previousHeaderOffset *int + availableData := len(data) + + for IsOffsetSafe(availableData, nextHeaderOffset, previousHeaderOffset) { + if nextHeaderOffset >= len(data) { + break + } + cpioEntryData := data[nextHeaderOffset:] + + entryHeader, err := parseCPIOEntryHeader(cpioEntryData) + if err != nil { + break + } + + // Validate magic + magicValid := false + for _, m := range validMagics { + if bytes.Equal(entryHeader.magic, m) { + magicValid = true + break + } + } + if !magicValid { + break + } + + headerCount++ + totalSize += entryHeader.headerSize + entryHeader.dataSize + + if entryHeader.fileName == eofMarker { + if headerCount > 1 { + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, file count: %d", cpioDescription, headerCount-1), + Confidence: ConfidenceHigh, + }, nil + } + break + } + + prev := nextHeaderOffset + previousHeaderOffset = &prev + nextHeaderOffset = offset + totalSize + } + + return nil, fmt.Errorf("cpio: no valid archive found") +} + +type cpioEntry struct { + magic []byte + fileName string + headerSize int + dataSize int +} + +func parseCPIOEntryHeader(cpioData []byte) (*cpioEntry, error) { + const ( + nullByteSize = 1 + cpioMagicStart = 0 + cpioMagicEnd = 6 + fileSizeStart = 54 + fileSizeEnd = 62 + fileNameSizeStart = 94 + fileNameSizeEnd = 102 + ) + + if len(cpioData) <= cpioHeaderSize { + return nil, fmt.Errorf("cpio: not enough data") + } + + headerMagic := make([]byte, cpioMagicEnd-cpioMagicStart) + copy(headerMagic, cpioData[cpioMagicStart:cpioMagicEnd]) + + // Get file data size (ASCII hex) + fileSizeStr := string(cpioData[fileSizeStart:fileSizeEnd]) + fileDataSize, err := strconv.ParseInt(fileSizeStr, 16, 64) + if err != nil { + return nil, fmt.Errorf("cpio: invalid file data size") + } + + // Get file name size (ASCII hex) + fileNameSizeStr := string(cpioData[fileNameSizeStart:fileNameSizeEnd]) + fileNameSize, err := strconv.ParseInt(fileNameSizeStr, 16, 64) + if err != nil { + return nil, fmt.Errorf("cpio: invalid file name size") + } + + fileNameStart := cpioHeaderSize + fileNameEnd := fileNameStart + int(fileNameSize) - nullByteSize + + if fileNameEnd > len(cpioData) || fileNameEnd < fileNameStart { + return nil, fmt.Errorf("cpio: file name out of bounds") + } + + fileName := string(cpioData[fileNameStart:fileNameEnd]) + + headerTotalSize := cpioHeaderSize + int(fileNameSize) + + return &cpioEntry{ + magic: headerMagic, + fileName: fileName, + headerSize: headerTotalSize + cpioBytePadding(headerTotalSize), + dataSize: int(fileDataSize) + cpioBytePadding(int(fileDataSize)), + }, nil +} + +func cpioBytePadding(n int) int { + mod := n % 4 + if mod == 0 { + return 0 + } + return 4 - mod +} + +// --------------------------------------------------------------------------- +// ARJ +// --------------------------------------------------------------------------- + +const arjDescription = "ARJ archive data" + +func arjMagic() [][]byte { + return [][]byte{ + {0x60, 0xea}, + } +} + +func arjParser(data []byte, offset int) (*SignatureResult, error) { + arjHeaderFields := []StructField{ + {"magic", "u16"}, + {"basic_header_size", "u16"}, + {"extra_header_size", "u8"}, + {"archiver_version", "u8"}, + {"min_version", "u8"}, + {"host_os", "u8"}, + {"internal_flags", "u8"}, + {"compression_method", "u8"}, + {"file_type", "u8"}, + {"reserved1", "u8"}, + {"datetime_file", "u32"}, + {"compressed_filesize", "u32"}, + {"original_filesize", "u32"}, + } + + if offset >= len(data) { + return nil, fmt.Errorf("arj: offset out of bounds") + } + + hdr, err := Parse(data[offset:], arjHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("arj: failed to parse header") + } + + // Version validation + if hdr["archiver_version"] < 1 || hdr["archiver_version"] > 16 || + hdr["min_version"] < 1 || hdr["min_version"] > 16 || + hdr["archiver_version"] < hdr["min_version"] { + return nil, fmt.Errorf("arj: invalid version info") + } + + // Flags + flags := "no password" + if hdr["internal_flags"]&0x01 != 0 { + flags = "password" + } + if hdr["internal_flags"]&0x04 != 0 { + flags += "|multi-volume" + } + if hdr["internal_flags"]&0x10 != 0 { + flags += "|slash-switched" + } + if hdr["internal_flags"]&0x20 != 0 { + flags += "|backup" + } + + // Host OS + hostOSMap := map[int]string{ + 0: "MS-DOS", 1: "PRIMOS", 2: "UNIX", 3: "AMIGA", 4: "MAX-OS", + 5: "OS/2", 6: "APPLE GS", 7: "ATARI ST", 8: "NeXT", 9: "VAX VMS", + } + hostOS, ok := hostOSMap[hdr["host_os"]] + if !ok { + return nil, fmt.Errorf("arj: unknown host OS %d", hdr["host_os"]) + } + + // Compression method + compressionMethodMap := map[int]string{ + 0: "stored", 1: "compressed most", 2: "compressed", + 3: "compressed faster", 4: "compressed fastest", + } + compressionMethod, ok := compressionMethodMap[hdr["compression_method"]] + if !ok { + return nil, fmt.Errorf("arj: unknown compression method %d", hdr["compression_method"]) + } + + // File type + fileTypeMap := map[int]string{ + 0: "binary", 1: "7-bit text", 2: "comment header", + 3: "directory", 4: "volume label", + } + fileType, ok := fileTypeMap[hdr["file_type"]] + if !ok { + return nil, fmt.Errorf("arj: unknown file type %d", hdr["file_type"]) + } + + // Compressed/uncompressed sizes — check for negative as signed i32 + compressedFileSize := int32(hdr["compressed_filesize"]) + if compressedFileSize < 0 { + return nil, fmt.Errorf("arj: negative compressed file size") + } + uncompressedFileSize := int32(hdr["original_filesize"]) + if uncompressedFileSize < 0 { + return nil, fmt.Errorf("arj: negative uncompressed file size") + } + + // Original file name + headerSize := hdr["extra_header_size"] + originalName := "" + if headerSize+4 < len(data[offset:]) { + originalName = GetCString(data[offset+headerSize+4:]) + } + + originalFileDate := EpochToString(uint32(hdr["datetime_file"])) + + availableData := len(data) - offset + if headerSize > availableData { + return nil, fmt.Errorf("arj: header size exceeds available data") + } + + extractionDeclined := fileType != "comment header" + + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf( + "%s, header size: %d, version %d, minimum version to extract: %d, flags: %s, compression method: %s, file type: %s, original name: %s, original file date: %s, compressed file size: %d, uncompressed file size: %d, os: %s", + arjDescription, + headerSize, + hdr["archiver_version"], + hdr["min_version"], + flags, + compressionMethod, + fileType, + originalName, + originalFileDate, + compressedFileSize, + uncompressedFileSize, + hostOS, + ), + Confidence: ConfidenceMedium, + ExtractionDeclined: extractionDeclined, + }, nil +} + +// --------------------------------------------------------------------------- +// ISO 9660 +// --------------------------------------------------------------------------- + +const iso9660Description = "ISO9660 primary volume" + +func iso9660Magic() [][]byte { + return [][]byte{ + {0x01, 0x43, 0x44, 0x30, 0x30, 0x31, 0x01, 0x00}, // \x01CD001\x01\x00 + } +} + +func iso9660Parser(data []byte, offset int) (*SignatureResult, error) { + const ( + isoMagicOffset = 32768 + isoStructStart = 32840 + ) + + isoFields := []StructField{ + {"unused1", "u64"}, + {"volume_size_lsb", "u32"}, + {"volume_size_msb", "u32"}, + {"unused2", "u64"}, + {"unused3", "u64"}, + {"unused4", "u64"}, + {"unused5", "u64"}, + {"set_size_lsb", "u16"}, + {"set_size_msb", "u16"}, + {"sequence_number_lsb", "u16"}, + {"sequence_number_msb", "u16"}, + {"block_size_lsb", "u16"}, + {"block_size_msb", "u16"}, + {"path_table_size_lsb", "u32"}, + {"path_table_size_msb", "u32"}, + } + + if offset < isoMagicOffset { + return nil, fmt.Errorf("iso9660: not enough data before magic offset") + } + + isoStart := offset - isoMagicOffset + + if isoStart+isoStructStart >= len(data) { + return nil, fmt.Errorf("iso9660: not enough data for header") + } + + isoHeaderData := data[isoStart+isoStructStart:] + hdr, err := Parse(isoHeaderData, isoFields, "little") + if err != nil { + return nil, fmt.Errorf("iso9660: failed to parse header") + } + + // Unused fields must be 0 + if hdr["unused1"] != 0 || hdr["unused2"] != 0 || hdr["unused3"] != 0 || + hdr["unused4"] != 0 || hdr["unused5"] != 0 { + return nil, fmt.Errorf("iso9660: unused fields not zero") + } + + // Validate byte-swapped field pairs + if hdr["set_size_lsb"] != int(swapU16(uint16(hdr["set_size_msb"]))) { + return nil, fmt.Errorf("iso9660: set_size mismatch") + } + if hdr["block_size_lsb"] != int(swapU16(uint16(hdr["block_size_msb"]))) { + return nil, fmt.Errorf("iso9660: block_size mismatch") + } + if hdr["volume_size_lsb"] != int(swapU32(uint32(hdr["volume_size_msb"]))) { + return nil, fmt.Errorf("iso9660: volume_size mismatch") + } + if hdr["sequence_number_lsb"] != int(swapU16(uint16(hdr["sequence_number_msb"]))) { + return nil, fmt.Errorf("iso9660: sequence_number mismatch") + } + if hdr["path_table_size_lsb"] != int(swapU32(uint32(hdr["path_table_size_msb"]))) { + return nil, fmt.Errorf("iso9660: path_table_size mismatch") + } + + imageSize := hdr["volume_size_lsb"] * hdr["block_size_lsb"] + + return &SignatureResult{ + Offset: isoStart, + Size: imageSize, + Description: fmt.Sprintf("%s, total size: %d bytes", iso9660Description, imageSize), + Confidence: ConfidenceHigh, + }, nil +} + +// swapU16 swaps bytes of a uint16 (big-endian to native / vice versa) +func swapU16(v uint16) uint16 { + b := make([]byte, 2) + binary.BigEndian.PutUint16(b, v) + return binary.LittleEndian.Uint16(b) +} + +// swapU32 swaps bytes of a uint32 +func swapU32(v uint32) uint32 { + b := make([]byte, 4) + binary.BigEndian.PutUint32(b, v) + return binary.LittleEndian.Uint32(b) +} + +// --------------------------------------------------------------------------- +// Dahua ZIP +// --------------------------------------------------------------------------- + +const dahuaZipDescription = "Dahua ZIP archive" + +func dahuaZipMagic() [][]byte { + return [][]byte{ + {0x44, 0x48, 0x03, 0x04}, // DH\x03\x04 + } +} + +func dahuaZipParser(data []byte, offset int) (*SignatureResult, error) { + // Parse as a normal ZIP file, then replace the description prefix + result, err := parseZipLike(data, offset, zipDescription) + if err != nil { + return nil, fmt.Errorf("dahua_zip: %v", err) + } + result.Description = strings.Replace(result.Description, zipDescription, dahuaZipDescription, 1) + return result, nil +} diff --git a/internal/leviathan/binsnoop/sig_compression.go b/internal/leviathan/binsnoop/sig_compression.go new file mode 100644 index 0000000..eccfd7a --- /dev/null +++ b/internal/leviathan/binsnoop/sig_compression.go @@ -0,0 +1,1081 @@ +package binsnoop + +import ( + "encoding/binary" + "fmt" +) + +// --------------------------------------------------------------------------- +// LZMA +// --------------------------------------------------------------------------- + +const LZMADescription = "LZMA compressed data" + +func LZMAMagic() [][]byte { + // Common LZMA properties + properties := []byte{0x5D, 0x6E, 0x6D, 0x6C} + + dictionarySizes := []uint32{ + 0x10000000, + 0x20000000, + 0x01000000, + 0x02000000, + 0x04000000, + 0x00800000, + 0x00400000, + 0x00200000, + 0x00100000, + 0x00080000, + 0x00020000, + 0x00010000, + } + + // Build magic signatures from property+dictionary_size combinations (little-endian) + var sigs [][]byte + for _, prop := range properties { + for _, dict := range dictionarySizes { + magic := make([]byte, 5) + magic[0] = prop + binary.LittleEndian.PutUint32(magic[1:], dict) + sigs = append(sigs, magic) + } + } + return sigs +} + +func LZMAParser(data []byte, offset int) (*SignatureResult, error) { + // Streamed data has a reported size of -1 (0xFFFFFFFFFFFFFFFF) + const lzmaStreamSize = -1 + + const minSupportedDecompressedSize = 256 + const maxSupportedDecompressedSize = 0xFFFFFFFF + + lzmaStructure := []StructField{ + {"properties", "u8"}, + {"dictionary_size", "u32"}, + {"decompressed_size", "u64"}, + {"null_byte", "u8"}, + } + + lzmaData := data[offset:] + if len(lzmaData) < StructSize(lzmaStructure) { + return nil, fmt.Errorf("LZMA: not enough data for header") + } + + hdr, err := Parse(lzmaData, lzmaStructure, "little") + if err != nil { + return nil, fmt.Errorf("LZMA: header parse error: %w", err) + } + + // Make sure the expected NULL byte is NULL + if hdr["null_byte"] != 0 { + return nil, fmt.Errorf("LZMA: null byte is not zero") + } + + // Sanity check the reported decompressed size. + // In Go int (signed 64-bit), 0xFFFFFFFFFFFFFFFF becomes -1. + decompressedSize := hdr["decompressed_size"] + validSize := decompressedSize == lzmaStreamSize || + (decompressedSize >= minSupportedDecompressedSize && + decompressedSize <= maxSupportedDecompressedSize) + if !validSize { + return nil, fmt.Errorf("LZMA: invalid decompressed size: %d", decompressedSize) + } + + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf( + "%s, properties: 0x%02X, dictionary size: %d bytes, uncompressed size: %d bytes", + LZMADescription, + hdr["properties"], + hdr["dictionary_size"], + decompressedSize, + ), + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// bzip2 +// --------------------------------------------------------------------------- + +const Bzip2Description = "bzip2 compressed data" + +func Bzip2Magic() [][]byte { + // Magic bytes include version, block size, and compressed magic signature + return [][]byte{ + []byte("BZh91AY&SY"), + []byte("BZh81AY&SY"), + []byte("BZh71AY&SY"), + []byte("BZh61AY&SY"), + []byte("BZh51AY&SY"), + []byte("BZh41AY&SY"), + []byte("BZh31AY&SY"), + []byte("BZh21AY&SY"), + []byte("BZh11AY&SY"), + } +} + +func Bzip2Parser(data []byte, offset int) (*SignatureResult, error) { + // Validate we have enough data for at least the magic + some compressed data + if offset+10 >= len(data) { + return nil, fmt.Errorf("bzip2: not enough data") + } + + return &SignatureResult{ + Offset: offset, + Description: Bzip2Description, + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// XZ +// --------------------------------------------------------------------------- + +const XZDescription = "XZ compressed data" + +func XZMagic() [][]byte { + return [][]byte{ + {0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}, + } +} + +func XZParser(data []byte, offset int) (*SignatureResult, error) { + const xzCRCStart = 6 + const xzCRCEnd = 8 + const xzHeaderSize = 12 + + xzStructure := []StructField{ + {"magic_p1", "u32"}, + {"magic_p2", "u16"}, + {"flags", "u16"}, + {"header_crc", "u32"}, + } + + xzData := data[offset:] + if len(xzData) < StructSize(xzStructure) { + return nil, fmt.Errorf("XZ: not enough data for header") + } + + hdr, err := Parse(xzData, xzStructure, "little") + if err != nil { + return nil, fmt.Errorf("XZ: header parse error: %w", err) + } + + // Validate the header CRC over the flags bytes (bytes 6..8) + if xzCRCEnd > len(xzData) { + return nil, fmt.Errorf("XZ: not enough data for CRC") + } + + crcData := xzData[xzCRCStart:xzCRCEnd] + calculatedCRC := CRC32(crcData) + if calculatedCRC != uint32(hdr["header_crc"]) { + return nil, fmt.Errorf("XZ: header CRC mismatch: expected 0x%08X, got 0x%08X", + hdr["header_crc"], calculatedCRC) + } + + return &SignatureResult{ + Offset: offset, + Description: XZDescription, + Confidence: ConfidenceHigh, + Size: xzHeaderSize, + }, nil +} + +// --------------------------------------------------------------------------- +// ZSTD +// --------------------------------------------------------------------------- + +const ZSTDDescription = "ZSTD compressed data" + +func ZSTDMagic() [][]byte { + return [][]byte{ + {0x28, 0xB5, 0x2F, 0xFD}, + } +} + +func ZSTDParser(data []byte, offset int) (*SignatureResult, error) { + const eofChecksumSize = 4 + const minBlockCount = 2 + + availableData := len(data) + + // Parse the ZSTD frame header + zstdHeader, err := parseZSTDHeader(data[offset:]) + if err != nil { + return nil, fmt.Errorf("ZSTD: %w", err) + } + + // Calculate first block header offset accounting for optional fields + nextBlockStart := offset + zstdHeader.fixedHeaderSize + + // If single segment flag is not set, window descriptor byte is present + if !zstdHeader.singleSegmentFlag { + nextBlockStart++ + } + + // Dictionary ID field size + if zstdHeader.dictionaryIDFlag == 1 { + nextBlockStart++ + } else if zstdHeader.dictionaryIDFlag == 2 { + nextBlockStart += 2 + } else if zstdHeader.dictionaryIDFlag == 3 { + nextBlockStart += 4 + } + + // Frame content header field size + if zstdHeader.frameContentFlag == 0 && zstdHeader.singleSegmentFlag { + nextBlockStart++ + } else if zstdHeader.frameContentFlag == 1 { + nextBlockStart += 2 + } else if zstdHeader.frameContentFlag == 2 { + nextBlockStart += 4 + } else if zstdHeader.frameContentFlag == 3 { + nextBlockStart += 8 + } + + var previousBlockStart *int + blockCount := 0 + totalSize := 0 + + // Walk through blocks to determine end of ZSTD data + for IsOffsetSafe(availableData, nextBlockStart, previousBlockStart) { + blockHeader, err := parseZSTDBlockHeader(data[nextBlockStart:]) + if err != nil { + break + } + + blockCount++ + prev := nextBlockStart + previousBlockStart = &prev + nextBlockStart += blockHeader.headerSize + blockHeader.blockSize + + if blockHeader.lastBlock { + totalSize = nextBlockStart - offset + if zstdHeader.contentChecksumPresent { + totalSize += eofChecksumSize + } + if blockCount >= minBlockCount { + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", ZSTDDescription, totalSize), + Confidence: ConfidenceHigh, + Size: totalSize, + }, nil + } + break + } + } + + return nil, fmt.Errorf("ZSTD: failed to validate block structure") +} + +type zstdHeaderInfo struct { + fixedHeaderSize int + dictionaryIDFlag int + contentChecksumPresent bool + singleSegmentFlag bool + frameContentFlag int +} + +func parseZSTDHeader(zstdData []byte) (*zstdHeaderInfo, error) { + const frameUnusedBitsMask = 0b00011000 + const dictionaryIDMask = 0b11 + const contentChecksumMask = 0b100 + const singleSegmentMask = 0b100000 + const frameContentMask = 0b11000000 + const frameContentShift = 6 + + structure := []StructField{ + {"magic", "u32"}, + {"frame_header_descriptor", "u8"}, + } + + hdr, err := Parse(zstdData, structure, "little") + if err != nil { + return nil, fmt.Errorf("header parse error: %w", err) + } + + // Unused bits should be zero + if (hdr["frame_header_descriptor"] & frameUnusedBitsMask) != 0 { + return nil, fmt.Errorf("unused bits are set in frame header descriptor") + } + + return &zstdHeaderInfo{ + fixedHeaderSize: StructSize(structure), + dictionaryIDFlag: hdr["frame_header_descriptor"] & dictionaryIDMask, + contentChecksumPresent: (hdr["frame_header_descriptor"] & contentChecksumMask) != 0, + singleSegmentFlag: (hdr["frame_header_descriptor"] & singleSegmentMask) != 0, + frameContentFlag: (hdr["frame_header_descriptor"] & frameContentMask) >> frameContentShift, + }, nil +} + +type zstdBlockHeaderInfo struct { + headerSize int + blockType int + blockSize int + lastBlock bool +} + +func parseZSTDBlockHeader(blockData []byte) (*zstdBlockHeaderInfo, error) { + const blockTypeMask = 0b110 + const blockTypeShift = 1 + const rleBlockType = 1 + const reservedBlockType = 3 + const lastBlockMask = 0b1 + const blockSizeMask = 0b1111_1111_1111_1111_1111_1000 + const blockSizeShift = 3 + + structure := []StructField{{"info_bits", "u24"}} + + hdr, err := Parse(blockData, structure, "little") + if err != nil { + return nil, fmt.Errorf("block header parse error: %w", err) + } + + info := &zstdBlockHeaderInfo{ + headerSize: StructSize(structure), + lastBlock: (hdr["info_bits"] & lastBlockMask) != 0, + blockType: (hdr["info_bits"] & blockTypeMask) >> blockTypeShift, + blockSize: (hdr["info_bits"] & blockSizeMask) >> blockSizeShift, + } + + // RLE block: single byte of raw data + if info.blockType == rleBlockType { + info.blockSize = 1 + } + + // Reserved block type is invalid + if info.blockType == reservedBlockType { + return nil, fmt.Errorf("reserved block type encountered") + } + + return info, nil +} + +// --------------------------------------------------------------------------- +// LZ4 +// --------------------------------------------------------------------------- + +const LZ4Description = "LZ4 compressed data" + +func LZ4Magic() [][]byte { + return [][]byte{ + {0x04, 0x22, 0x4D, 0x18}, + } +} + +func LZ4Parser(data []byte, offset int) (*SignatureResult, error) { + const contentChecksumLen = 4 + + lz4Header, err := parseLZ4FileHeader(data[offset:]) + if err != nil { + return nil, fmt.Errorf("LZ4: %w", err) + } + + if offset+lz4Header.headerSize >= len(data) { + return nil, fmt.Errorf("LZ4: not enough data after header") + } + + lz4Data := data[offset+lz4Header.headerSize:] + dataSize, err := getLZ4DataSize(lz4Data, lz4Header.blockChecksumPresent) + if err != nil { + return nil, fmt.Errorf("LZ4: %w", err) + } + + totalSize := lz4Header.headerSize + dataSize + if lz4Header.contentChecksumPresent { + totalSize += contentChecksumLen + } + + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", LZ4Description, totalSize), + Confidence: ConfidenceMedium, + Size: totalSize, + }, nil +} + +type lz4FileHeaderInfo struct { + headerSize int + blockChecksumPresent bool + contentChecksumPresent bool +} + +func parseLZ4FileHeader(lz4Data []byte) (*lz4FileHeaderInfo, error) { + const magicSize = 4 + const lz4StructSize = 6 + + const bdReservedMask = 0b10001111 + const flagsReservedMask = 0b00000010 + + const flagDictionaryPresent = 0b00000001 + const flagContentSizePresent = 0b00001000 + const flagBlockChecksumPresent = 0b00010000 + const flagContentChecksumPresent = 0b00000100 + + const dictionaryLen = 4 + const contentSizeLen = 8 + + structure := []StructField{ + {"magic", "u32"}, + {"flags", "u8"}, + {"bd", "u8"}, + } + + hdr, err := Parse(lz4Data, structure, "little") + if err != nil { + return nil, fmt.Errorf("file header parse error: %w", err) + } + + // Reserved bits must not be set + if (hdr["flags"] & flagsReservedMask) != 0 { + return nil, fmt.Errorf("flags reserved bits are set") + } + if (hdr["bd"] & bdReservedMask) != 0 { + return nil, fmt.Errorf("bd reserved bits are set") + } + + // Calculate CRC data boundaries (CRC covers descriptor excluding magic) + crcDataStart := magicSize + crcDataEnd := crcDataStart + (lz4StructSize - magicSize) + + if (hdr["flags"] & flagContentSizePresent) != 0 { + crcDataEnd += contentSizeLen + } + if (hdr["flags"] & flagDictionaryPresent) != 0 { + crcDataEnd += dictionaryLen + } + + if crcDataEnd >= len(lz4Data) { + return nil, fmt.Errorf("not enough data for header CRC") + } + + crcData := lz4Data[crcDataStart:crcDataEnd] + actualCRC := lz4Data[crcDataEnd] + + // LZ4 header CRC is the second byte of xxh32 hash over the descriptor + calculatedCRC := byte((xxh32(crcData, 0) >> 8) & 0xFF) + if actualCRC != calculatedCRC { + return nil, fmt.Errorf("header CRC mismatch: expected 0x%02X, got 0x%02X", + actualCRC, calculatedCRC) + } + + return &lz4FileHeaderInfo{ + headerSize: crcDataEnd + 1, + blockChecksumPresent: (hdr["flags"] & flagBlockChecksumPresent) != 0, + contentChecksumPresent: (hdr["flags"] & flagContentChecksumPresent) != 0, + }, nil +} + +func getLZ4DataSize(lz4Data []byte, checksumPresent bool) (int, error) { + dataSize := 0 + var lastOffset *int + available := len(lz4Data) + + for IsOffsetSafe(available, dataSize, lastOffset) { + if dataSize >= len(lz4Data) { + break + } + + blockHeader, err := parseLZ4BlockHeader(lz4Data[dataSize:], checksumPresent) + if err != nil { + break + } + + prev := dataSize + lastOffset = &prev + dataSize += blockHeader.headerSize + blockHeader.dataSize + blockHeader.checksumSize + + if blockHeader.lastBlock { + return dataSize, nil + } + } + + return 0, fmt.Errorf("failed to find LZ4 end marker") +} + +type lz4BlockHeaderInfo struct { + dataSize int + headerSize int + checksumSize int + lastBlock bool +} + +func parseLZ4BlockHeader(blockData []byte, checksumPresent bool) (*lz4BlockHeaderInfo, error) { + const sizeMask uint32 = 0x7FFFFFFF + const endMarker = 0 + const checksumSize = 4 + const blockStructSize = 4 + + structure := []StructField{{"block_size", "u32"}} + + hdr, err := Parse(blockData, structure, "little") + if err != nil { + return nil, fmt.Errorf("block header parse error: %w", err) + } + + info := &lz4BlockHeaderInfo{ + headerSize: blockStructSize, + lastBlock: hdr["block_size"] == endMarker, + } + + if checksumPresent { + info.checksumSize = checksumSize + } + + // High bit is not part of the actual block size + info.dataSize = int(uint32(hdr["block_size"]) & sizeMask) + + return info, nil +} + +// xxh32 implements the XXHash32 algorithm used by LZ4 for header checksums. +func xxh32(input []byte, seed uint32) uint32 { + const ( + prime1 uint32 = 0x9E3779B1 + prime2 uint32 = 0x85EBCA77 + prime3 uint32 = 0xC2B2AE3D + prime4 uint32 = 0x27D4EB2F + prime5 uint32 = 0x165667B1 + ) + + n := len(input) + p := 0 + var h32 uint32 + + if n >= 16 { + v1 := seed + prime1 + prime2 + v2 := seed + prime2 + v3 := seed + v4 := seed - prime1 + for p+16 <= n { + v1 = xxh32Round(v1, binary.LittleEndian.Uint32(input[p:]), prime1, prime2) + v2 = xxh32Round(v2, binary.LittleEndian.Uint32(input[p+4:]), prime1, prime2) + v3 = xxh32Round(v3, binary.LittleEndian.Uint32(input[p+8:]), prime1, prime2) + v4 = xxh32Round(v4, binary.LittleEndian.Uint32(input[p+12:]), prime1, prime2) + p += 16 + } + h32 = ((v1 << 1) | (v1 >> 31)) + + ((v2 << 7) | (v2 >> 25)) + + ((v3 << 12) | (v3 >> 20)) + + ((v4 << 18) | (v4 >> 14)) + } else { + h32 = seed + prime5 + } + + h32 += uint32(n) + + for p+4 <= n { + h32 += binary.LittleEndian.Uint32(input[p:]) * prime3 + h32 = ((h32 << 17) | (h32 >> 15)) * prime4 + p += 4 + } + + for p < n { + h32 += uint32(input[p]) * prime5 + h32 = ((h32 << 11) | (h32 >> 21)) * prime1 + p++ + } + + h32 ^= h32 >> 15 + h32 *= prime2 + h32 ^= h32 >> 13 + h32 *= prime3 + h32 ^= h32 >> 16 + + return h32 +} + +func xxh32Round(acc, input, prime1, prime2 uint32) uint32 { + acc += input * prime2 + acc = (acc << 13) | (acc >> 19) + acc *= prime1 + return acc +} + +// --------------------------------------------------------------------------- +// LZOP +// --------------------------------------------------------------------------- + +const LZOPDescription = "LZO compressed data" + +func LZOPMagic() [][]byte { + return [][]byte{ + {0x89, 0x4C, 0x5A, 0x4F, 0x00, 0x0D, 0x0A, 0x1A, 0x0A}, + } +} + +func LZOPParser(data []byte, offset int) (*SignatureResult, error) { + lzopHeader, err := parseLZOPFileHeader(data[offset:]) + if err != nil { + return nil, fmt.Errorf("LZOP: %w", err) + } + + if offset+lzopHeader.headerSize >= len(data) { + return nil, fmt.Errorf("LZOP: not enough data after header") + } + + lzopData := data[offset+lzopHeader.headerSize:] + dataSize, err := getLZODataSize(lzopData, lzopHeader.blockChecksumPresent) + if err != nil { + return nil, fmt.Errorf("LZOP: %w", err) + } + + totalSize := lzopHeader.headerSize + dataSize + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", LZOPDescription, totalSize), + Confidence: ConfidenceHigh, + Size: totalSize, + }, nil +} + +type lzopFileHeaderInfo struct { + headerSize int + blockChecksumPresent bool +} + +func parseLZOPFileHeader(lzopData []byte) (*lzopFileHeaderInfo, error) { + const lzoMaxVersion = 0x1040 + const lzoHeaderSizeP1 = 21 + const lzoHeaderSizeP2 = 13 + const filterSize = 4 + const lzoChecksumSize = 4 + + const flagFilter = 0x0000_0800 + const flagCRC32C = 0x0000_0200 + const flagAdler32C = 0x0000_0002 + + structureP1 := []StructField{ + {"magic_p1", "u8"}, + {"magic_p2", "u64"}, + {"version", "u16"}, + {"lib_version", "u16"}, + {"version_needed", "u16"}, + {"method", "u8"}, + {"level", "u8"}, + {"flags", "u32"}, + } + + structureP2 := []StructField{ + {"mode", "u32"}, + {"mtime", "u32"}, + {"gmt_diff", "u32"}, + {"file_name_length", "u8"}, + } + + allowedMethods := []int{1, 2, 3} + + hdrP1, err := Parse(lzopData, structureP1, "big") + if err != nil { + return nil, fmt.Errorf("header part 1 parse error: %w", err) + } + + // Sanity check methods + methodValid := false + for _, m := range allowedMethods { + if hdrP1["method"] == m { + methodValid = true + break + } + } + if !methodValid { + return nil, fmt.Errorf("invalid method: %d", hdrP1["method"]) + } + + // Sanity check version numbers + if hdrP1["version"] > lzoMaxVersion { + return nil, fmt.Errorf("version too high: 0x%04X", hdrP1["version"]) + } + if hdrP1["version"] < hdrP1["version_needed"] { + return nil, fmt.Errorf("version %d < version_needed %d", hdrP1["version"], hdrP1["version_needed"]) + } + + headerP2Start := lzoHeaderSizeP1 + if (hdrP1["flags"] & flagFilter) != 0 { + headerP2Start += filterSize + } + + headerP2End := headerP2Start + lzoHeaderSizeP2 + if headerP2End > len(lzopData) { + return nil, fmt.Errorf("not enough data for header part 2") + } + + hdrP2, err := Parse(lzopData[headerP2Start:headerP2End], structureP2, "big") + if err != nil { + return nil, fmt.Errorf("header part 2 parse error: %w", err) + } + + headerSize := headerP2End + hdrP2["file_name_length"] + lzoChecksumSize + + // Faithfully ported: in the Rust source this is & (AND) which always yields false + blockChecksumPresent := (hdrP1["flags"] & flagAdler32C & flagCRC32C) != 0 + + if headerSize > len(lzopData) { + return nil, fmt.Errorf("calculated header size exceeds available data") + } + + return &lzopFileHeaderInfo{ + headerSize: headerSize, + blockChecksumPresent: blockChecksumPresent, + }, nil +} + +type lzopBlockHeaderInfo struct { + headerSize int + compressedSize int + uncompressedSize int + checksumSize int +} + +func parseLZOPBlockHeader(lzoData []byte, compressedChecksumPresent bool) (*lzopBlockHeaderInfo, error) { + const blockHeaderSize = 12 + const maxUncompressedBlockSize = 64 * 1024 * 1024 + const lzoChecksumSize = 4 + + structure := []StructField{ + {"uncompressed_size", "u32"}, + {"compressed_size", "u32"}, + {"uncompressed_checksum", "u32"}, + } + + hdr, err := Parse(lzoData, structure, "big") + if err != nil { + return nil, fmt.Errorf("block header parse error: %w", err) + } + + if hdr["compressed_size"] == 0 || + hdr["uncompressed_size"] == 0 || + hdr["uncompressed_checksum"] == 0 || + hdr["uncompressed_size"] > maxUncompressedBlockSize { + return nil, fmt.Errorf("invalid block header values") + } + + info := &lzopBlockHeaderInfo{ + headerSize: blockHeaderSize, + compressedSize: hdr["compressed_size"], + uncompressedSize: hdr["uncompressed_size"], + } + + if compressedChecksumPresent { + info.checksumSize = lzoChecksumSize + } + + return info, nil +} + +func parseLZOPEOFMarker(eofData []byte) (int, error) { + const eofMarkerValue = 0 + const eofMarkerSize = 4 + + structure := []StructField{{"marker", "u32"}} + + hdr, err := Parse(eofData, structure, "big") + if err != nil { + return 0, fmt.Errorf("EOF marker parse error: %w", err) + } + + if hdr["marker"] != eofMarkerValue { + return 0, fmt.Errorf("invalid EOF marker: %d", hdr["marker"]) + } + + return eofMarkerSize, nil +} + +func getLZODataSize(lzoData []byte, compressedChecksumPresent bool) (int, error) { + const minBlockCount = 2 + + available := len(lzoData) + var lastOffset *int + dataSize := 0 + blockCount := 0 + + for IsOffsetSafe(available, dataSize, lastOffset) { + blockHeader, err := parseLZOPBlockHeader(lzoData[dataSize:], compressedChecksumPresent) + if err != nil { + break + } + + blockCount++ + prev := dataSize + lastOffset = &prev + dataSize += blockHeader.headerSize + blockHeader.compressedSize + blockHeader.checksumSize + } + + if blockCount >= minBlockCount { + if dataSize < len(lzoData) { + eofSize, err := parseLZOPEOFMarker(lzoData[dataSize:]) + if err != nil { + return 0, fmt.Errorf("EOF marker error: %w", err) + } + dataSize += eofSize + return dataSize, nil + } + } + + return 0, fmt.Errorf("failed to validate LZO block structure") +} + +// --------------------------------------------------------------------------- +// LZFSE +// --------------------------------------------------------------------------- + +const LZFSEDescription = "LZFSE compressed data" + +func LZFSEMagic() [][]byte { + return [][]byte{ + []byte("bvx-"), + []byte("bvx1"), + []byte("bvx2"), + []byte("bvxn"), + } +} + +func LZFSEParser(data []byte, offset int) (*SignatureResult, error) { + available := len(data) + nextBlockOffset := offset + var previousBlockOffset *int + + for IsOffsetSafe(available, nextBlockOffset, previousBlockOffset) { + prev := nextBlockOffset + previousBlockOffset = &prev + + block, err := parseLZFSEBlockHeader(data[nextBlockOffset:]) + if err != nil { + break + } + + nextBlockOffset += block.headerSize + block.dataSize + + if block.eof { + totalSize := nextBlockOffset - offset + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", LZFSEDescription, totalSize), + Confidence: ConfidenceHigh, + Size: totalSize, + }, nil + } + } + + return nil, fmt.Errorf("LZFSE: failed to find end-of-stream block") +} + +type lzfseBlockInfo struct { + eof bool + dataSize int + headerSize int +} + +func parseLZFSEBlockHeader(lzfseData []byte) (*lzfseBlockInfo, error) { + // LZFSE block types (little-endian u32 representations) + const endOfStream = 0x24787662 // "bvx$" + const uncompressed = 0x2D787662 // "bvx-" + const compressedV1 = 0x31787662 // "bvx1" + const compressedV2 = 0x32787662 // "bvx2" + const compressedLZVN = 0x6E787662 // "bvxn" + + blockTypeStructure := []StructField{{"block_type", "u32"}} + + hdr, err := Parse(lzfseData, blockTypeStructure, "little") + if err != nil { + return nil, fmt.Errorf("block header parse error: %w", err) + } + + blockType := hdr["block_type"] + + switch blockType { + case endOfStream: + return &lzfseBlockInfo{eof: true, dataSize: 0, headerSize: 4}, nil + + case uncompressed: + return parseLZFSEUncompressedBlock(lzfseData) + + case compressedV1: + return parseLZFSECompressedV1Block(lzfseData) + + case compressedV2: + return parseLZFSECompressedV2Block(lzfseData) + + case compressedLZVN: + return parseLZFSECompressedLZVNBlock(lzfseData) + } + + return nil, fmt.Errorf("unknown LZFSE block type: 0x%08X", blockType) +} + +func parseLZFSEUncompressedBlock(lzfseData []byte) (*lzfseBlockInfo, error) { + const headerSize = 8 + + structure := []StructField{ + {"magic", "u32"}, + {"n_raw_bytes", "u32"}, + } + + hdr, err := Parse(lzfseData, structure, "little") + if err != nil { + return nil, fmt.Errorf("uncompressed block parse error: %w", err) + } + + return &lzfseBlockInfo{ + eof: false, + dataSize: hdr["n_raw_bytes"], + headerSize: headerSize, + }, nil +} + +func parseLZFSECompressedV1Block(lzfseData []byte) (*lzfseBlockInfo, error) { + const headerSize = 770 + + structure := []StructField{ + {"magic", "u32"}, + {"n_raw_bytes", "u32"}, + {"n_payload_bytes", "u32"}, + {"n_literals", "u32"}, + {"n_matches", "u32"}, + {"n_literal_payload_bytes", "u32"}, + {"n_lmd_payload_bytes", "u32"}, + {"literal_bits", "u32"}, + {"literal_state", "u64"}, + {"lmd_bits", "u32"}, + {"l_state", "u16"}, + {"m_state", "u16"}, + {"d_state", "u16"}, + } + + hdr, err := Parse(lzfseData, structure, "little") + if err != nil { + return nil, fmt.Errorf("compressed v1 block parse error: %w", err) + } + + return &lzfseBlockInfo{ + eof: false, + dataSize: hdr["n_literal_payload_bytes"] + hdr["n_lmd_payload_bytes"], + headerSize: headerSize, + }, nil +} + +func parseLZFSECompressedV2Block(lzfseData []byte) (*lzfseBlockInfo, error) { + const nPayloadShift = 20 + const lmdPayloadShift = 40 + const payloadMask = 0b11111_11111_11111_11111 + + structure := []StructField{ + {"magic", "u32"}, + {"uncompressed_size", "u32"}, + {"packed_field_1", "u64"}, + {"packed_field_2", "u64"}, + {"header_size", "u32"}, + {"state_fields", "u32"}, + } + + hdr, err := Parse(lzfseData, structure, "little") + if err != nil { + return nil, fmt.Errorf("compressed v2 block parse error: %w", err) + } + + nLMDPayloadBytes := (hdr["packed_field_2"] >> lmdPayloadShift) & payloadMask + nLiteralPayloadBytes := (hdr["packed_field_1"] >> nPayloadShift) & payloadMask + + return &lzfseBlockInfo{ + eof: false, + dataSize: nLMDPayloadBytes + nLiteralPayloadBytes, + headerSize: hdr["header_size"], + }, nil +} + +func parseLZFSECompressedLZVNBlock(lzfseData []byte) (*lzfseBlockInfo, error) { + const headerSize = 12 + + structure := []StructField{ + {"magic", "u32"}, + {"n_raw_bytes", "u32"}, + {"n_payload_bytes", "u32"}, + } + + hdr, err := Parse(lzfseData, structure, "little") + if err != nil { + return nil, fmt.Errorf("compressed LZVN block parse error: %w", err) + } + + return &lzfseBlockInfo{ + eof: false, + dataSize: hdr["n_payload_bytes"], + headerSize: headerSize, + }, nil +} + +// --------------------------------------------------------------------------- +// Zlib +// --------------------------------------------------------------------------- + +const ZlibDescription = "Zlib compressed file" + +func ZlibMagic() [][]byte { + return [][]byte{ + {0x78, 0x9C}, + {0x78, 0xDA}, + {0x78, 0x5E}, + } +} + +func ZlibParser(data []byte, offset int) (*SignatureResult, error) { + if offset+2 > len(data) { + return nil, fmt.Errorf("zlib: not enough data") + } + + // Zlib header: CMF (byte 0) and FLG (byte 1) + // The two-byte value (CMF*256 + FLG) must be a multiple of 31 (FCHECK) + cmf := int(data[offset]) + flg := int(data[offset+1]) + if (cmf*256+flg)%31 != 0 { + return nil, fmt.Errorf("zlib: FCHECK validation failed") + } + + // CM (compression method) should be 8 (deflate) + cm := cmf & 0x0F + if cm != 8 { + return nil, fmt.Errorf("zlib: invalid compression method: %d", cm) + } + + // CINFO should be <= 7 + cinfo := (cmf >> 4) & 0x0F + if cinfo > 7 { + return nil, fmt.Errorf("zlib: invalid CINFO: %d", cinfo) + } + + return &SignatureResult{ + Offset: offset, + Description: ZlibDescription, + Confidence: ConfidenceHigh, + }, nil +} + +// --------------------------------------------------------------------------- +// compress'd +// --------------------------------------------------------------------------- + +const CompressdDescription = "compress'd data" + +func CompressdMagic() [][]byte { + return [][]byte{ + {0x1F, 0x9D, 0x90}, + } +} + +func CompressdParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: CompressdDescription, + Confidence: ConfidenceLow, + } + + // Higher confidence when found at the beginning of a file + if offset == 0 { + result.Confidence = ConfidenceMedium + } + + return result, nil +} diff --git a/internal/leviathan/binsnoop/sig_crypto.go b/internal/leviathan/binsnoop/sig_crypto.go new file mode 100644 index 0000000..f1424a1 --- /dev/null +++ b/internal/leviathan/binsnoop/sig_crypto.go @@ -0,0 +1,795 @@ +package binsnoop + +import ( + "encoding/base64" + "fmt" + "strings" +) + +// ---- AES descriptions ---- + +const ( + DescriptionAESSBox = "AES S-Box" + DescriptionAESFT = "AES Forward Table" + DescriptionAESRT = "AES Reverse Table" + DescriptionAESRCON = "AES RCON" + DescriptionAESAcc = "AES Acceleration Table" +) + +// aesSBoxMagic returns magic bytes for AES S-Box (forward and reverse). +func aesSBoxMagic() [][]byte { + return [][]byte{ + {0x63, 0x7C, 0x77, 0x7B, 0xF2, 0x6B, 0x6F, 0xC5}, // Forward S-Box + {0x52, 0x09, 0x6A, 0xD5, 0x30, 0x36, 0xA5, 0x38}, // Reverse S-Box + } +} + +// aesSBoxParser validates AES S-Box signatures. +func aesSBoxParser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Description: DescriptionAESSBox, + Confidence: ConfidenceLow, + }, nil +} + +// aesForwardTableMagic returns magic bytes for AES Forward Table. +func aesForwardTableMagic() [][]byte { + return [][]byte{ + {0xC6, 0x63, 0x63, 0xA5, 0xF8, 0x7C, 0x7C, 0x84, 0xEE, 0x77, 0x77, 0x99, 0xF6, 0x7B, 0x7B, 0x8D}, + } +} + +// aesForwardTableParser validates AES Forward Table signatures. +func aesForwardTableParser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Description: DescriptionAESFT, + Confidence: ConfidenceLow, + }, nil +} + +// aesReverseTableMagic returns magic bytes for AES Reverse Table. +func aesReverseTableMagic() [][]byte { + return [][]byte{ + {0x51, 0xF4, 0xA7, 0x50, 0x7E, 0x41, 0x65, 0x53, 0x1A, 0x17, 0xA4, 0xC3, 0x3A, 0x27, 0x5E, 0x96}, + } +} + +// aesReverseTableParser validates AES Reverse Table signatures. +func aesReverseTableParser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Description: DescriptionAESRT, + Confidence: ConfidenceLow, + }, nil +} + +// aesRconMagic returns magic bytes for AES RCON. +func aesRconMagic() [][]byte { + return [][]byte{ + {0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1B, 0x36}, + {0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, + 0x10, 0x00, 0x00, 0x00, 0x20, 0x00, 0x00, 0x00, 0x40, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x1B, 0x00, 0x00, 0x00, 0x36, 0x00, 0x00, 0x00}, + } +} + +// aesRconParser validates AES RCON signatures. +func aesRconParser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Description: DescriptionAESRCON, + Confidence: ConfidenceLow, + }, nil +} + +// aesAccelerationTableMagic returns magic bytes for AES Acceleration Table. +func aesAccelerationTableMagic() [][]byte { + return [][]byte{ + // combined sbox x2 + {0xA5, 0x84, 0x99, 0x8D, 0x0D, 0xBD, 0xB1, 0x54, 0x50, 0x03, 0xA9, 0x7D, 0x19, 0x62, 0xE6, 0x9A}, + // combined sbox x3 + {0xC6, 0xF8, 0xEE, 0xF6, 0xFF, 0xD6, 0xDE, 0x91, 0x60, 0x02, 0xCE, 0x56, 0xE7, 0xB5, 0x4D, 0xEC}, + // Galois mult 2 + {0x00, 0x02, 0x04, 0x06, 0x08, 0x0a, 0x0c, 0x0e, 0x10, 0x12, 0x14, 0x16, 0x18, 0x1a, 0x1c, 0x1e, + 0x20, 0x22, 0x24, 0x26, 0x28, 0x2a, 0x2c, 0x2e}, + // Galois mult 3 + {0x00, 0x03, 0x06, 0x05, 0x0c, 0x0f, 0x0a, 0x09, 0x18, 0x1b, 0x1e, 0x1d, 0x14, 0x17, 0x12, 0x11, + 0x30, 0x33, 0x36, 0x35, 0x3c, 0x3f, 0x3a, 0x39}, + // Galois mult 9 + {0x00, 0x09, 0x12, 0x1b, 0x24, 0x2d, 0x36, 0x3f, 0x48, 0x41, 0x5a, 0x53, 0x6c, 0x65, 0x7e, 0x77, + 0x90, 0x99, 0x82, 0x8b, 0xb4, 0xbd, 0xa6, 0xaf}, + // Galois mult 11 + {0x00, 0x0b, 0x16, 0x1d, 0x2c, 0x27, 0x3a, 0x31, 0x58, 0x53, 0x4e, 0x45, 0x74, 0x7f, 0x62, 0x69, + 0xb0, 0xbb, 0xa6, 0xad, 0x9c, 0x97, 0x8a, 0x81}, + // Galois mult 13 + {0x00, 0x0d, 0x1a, 0x17, 0x34, 0x39, 0x2e, 0x23, 0x68, 0x65, 0x72, 0x7f, 0x5c, 0x51, 0x46, 0x4b, + 0xd0, 0xdd, 0xca, 0xc7, 0xe4, 0xe9, 0xfe, 0xf3}, + // Galois mult 14 + {0x00, 0x0e, 0x1c, 0x12, 0x38, 0x36, 0x24, 0x2a, 0x70, 0x7e, 0x6c, 0x62, 0x48, 0x46, 0x54, 0x5a, + 0xe0, 0xee, 0xfc, 0xf2, 0xd8, 0xd6, 0xc4, 0xca}, + } +} + +// aesAccelerationTableParser validates AES Acceleration Table signatures. +func aesAccelerationTableParser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Description: DescriptionAESAcc, + Confidence: ConfidenceLow, + }, nil +} + +// ---- RSA ---- + +const DescriptionRSA = "RSA encrypted session key" + +type rsaKeyDefinition struct { + magic []byte + keySize int + usageOffset int + keyIDOffset int + validChecks map[int][][]byte + terminatorOffset int +} + +func rsaKeyDefinitions() []rsaKeyDefinition { + return []rsaKeyDefinition{ + // 1024b RSA key + { + magic: []byte{0x84, 0x8C, 0x03}, + keySize: 1024, + keyIDOffset: 3, + terminatorOffset: 142, + usageOffset: 11, + validChecks: map[int][][]byte{ + 12: { + {0x04, 0x00}, {0x03, 0xff}, {0x03, 0xfe}, {0x03, 0xfd}, + {0x03, 0xfc}, {0x03, 0xfb}, {0x03, 0xfa}, {0x03, 0xf9}, + }, + }, + }, + // 2048b RSA key + { + magic: []byte{0x85, 0x01, 0x0c, 0x03}, + keySize: 2048, + keyIDOffset: 4, + terminatorOffset: 271, + usageOffset: 12, + validChecks: map[int][][]byte{ + 13: { + {0x08, 0x00}, {0x07, 0xff}, {0x07, 0xfe}, {0x07, 0xfd}, + {0x07, 0xfc}, {0x07, 0xfb}, {0x07, 0xfa}, {0x07, 0xf9}, + }, + }, + }, + // 3072b RSA key + { + magic: []byte{0x85, 0x01, 0x8c, 0x03}, + keySize: 3072, + keyIDOffset: 4, + terminatorOffset: 399, + usageOffset: 12, + validChecks: map[int][][]byte{ + 13: { + {0x0c, 0x00}, {0x0b, 0xff}, {0x0b, 0xfe}, {0x0b, 0xfd}, + {0x0b, 0xfc}, {0x0b, 0xfb}, {0x0b, 0xfa}, {0x0b, 0xf9}, + }, + }, + }, + // 4096b RSA key + { + magic: []byte{0x85, 0x02, 0x0c, 0x03}, + keySize: 4096, + keyIDOffset: 4, + terminatorOffset: 527, + usageOffset: 12, + validChecks: map[int][][]byte{ + 13: { + {0x10, 0x00}, {0x0f, 0xff}, {0x0f, 0xfe}, {0x0f, 0xfd}, + {0x0f, 0xfc}, {0x0f, 0xfb}, {0x0f, 0xfa}, {0x0f, 0xf9}, + }, + }, + }, + // 8192b RSA key + { + magic: []byte{0x85, 0x04, 0x0c, 0x03}, + keySize: 8192, + keyIDOffset: 4, + terminatorOffset: 1039, + usageOffset: 12, + validChecks: map[int][][]byte{ + 13: { + {0x20, 0x00}, {0x1f, 0xff}, {0x1f, 0xfe}, {0x1f, 0xfd}, + {0x1f, 0xfc}, {0x1f, 0xfb}, {0x1f, 0xfa}, {0x1f, 0xf9}, + }, + }, + }, + } +} + +// rsaMagic returns magic bytes for all known RSA key sizes. +func rsaMagic() [][]byte { + var magics [][]byte + for _, def := range rsaKeyDefinitions() { + m := make([]byte, len(def.magic)) + copy(m, def.magic) + magics = append(magics, m) + } + return magics +} + +type rsaKeyInfo struct { + bits int + canSign bool + canEncrypt bool + keyID int + dataSize int +} + +func rsaKeyParse(rawData []byte, def *rsaKeyDefinition) (*rsaKeyInfo, error) { + const terminatorByte byte = 0xD2 + const encryptOnly byte = 2 + const signAndEncrypt byte = 1 + const validBytesSize = 2 + + keyIDFields := []StructField{{"id", "u64"}} + + keyDataLen := def.terminatorOffset + 1 + if len(rawData) < keyDataLen { + return nil, fmt.Errorf("insufficient RSA key data") + } + keyData := rawData[:keyDataLen] + + if keyData[def.terminatorOffset] != terminatorByte { + return nil, fmt.Errorf("invalid RSA terminator byte") + } + + if def.keyIDOffset+8 > len(keyData) { + return nil, fmt.Errorf("key ID offset out of bounds") + } + keyIDParsed, err := Parse(keyData[def.keyIDOffset:], keyIDFields, "big") + if err != nil { + return nil, fmt.Errorf("failed to parse RSA key ID") + } + + info := &rsaKeyInfo{keyID: keyIDParsed["id"]} + info.canEncrypt = keyData[def.usageOffset] == encryptOnly + info.canSign = keyData[def.usageOffset] == signAndEncrypt + if info.canSign { + info.canEncrypt = true + } + + if !info.canSign && !info.canEncrypt { + return nil, fmt.Errorf("RSA key cannot sign or encrypt") + } + + for checkOffset, expectedList := range def.validChecks { + if checkOffset+validBytesSize > len(keyData) { + continue + } + actual := keyData[checkOffset : checkOffset+validBytesSize] + for _, expected := range expectedList { + if len(expected) == validBytesSize && actual[0] == expected[0] && actual[1] == expected[1] { + info.bits = def.keySize + info.dataSize = keyDataLen + return info, nil + } + } + } + + return nil, fmt.Errorf("RSA key validation bytes mismatch") +} + +// rsaParser validates RSA encrypted session key signatures. +func rsaParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: DescriptionRSA, + Confidence: ConfidenceMedium, + } + + for _, def := range rsaKeyDefinitions() { + magicEnd := offset + len(def.magic) + if magicEnd > len(data) { + continue + } + match := true + for i, b := range def.magic { + if data[offset+i] != b { + match = false + break + } + } + if !match { + continue + } + + info, err := rsaKeyParse(data[offset:], &def) + if err != nil { + break + } + result.Size = info.dataSize + result.Description = fmt.Sprintf("%s, %d bits, can sign: %t, can encrypt: %t, key ID: %#018X", + result.Description, info.bits, info.canSign, info.canEncrypt, info.keyID) + return result, nil + } + + return nil, fmt.Errorf("invalid RSA signature") +} + +// ---- OpenSSL ---- + +const DescriptionOpenSSL = "OpenSSL encryption" + +// opensslMagic returns magic bytes for OpenSSL encrypted files. +func opensslMagic() [][]byte { + return [][]byte{ + []byte("Salted__"), + } +} + +func isSaltInvalid(salt int) bool { + const saltLen = 8 + badByteCount := 0 + for i := 0; i < saltLen; i++ { + saltByte := byte((salt >> (8 * i)) & 0xFF) + if saltByte == 0 || IsPrintableASCII(saltByte) { + badByteCount++ + } + } + return badByteCount == saltLen +} + +// opensslParser validates OpenSSL encryption signatures. +func opensslParser(data []byte, offset int) (*SignatureResult, error) { + sslFields := []StructField{{"magic", "u64"}, {"salt", "u64"}} + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionOpenSSL, + Confidence: ConfidenceLow, + } + + if offset+StructSize(sslFields) > len(data) { + return nil, fmt.Errorf("insufficient OpenSSL data") + } + + sslHeader, err := Parse(data[offset:], sslFields, "big") + if err != nil { + return nil, fmt.Errorf("failed to parse OpenSSL header") + } + + salt := sslHeader["salt"] + if salt == 0 { + return nil, fmt.Errorf("OpenSSL salt is zero") + } + + if isSaltInvalid(salt) { + return nil, fmt.Errorf("OpenSSL salt is invalid") + } + + if offset == 0 { + result.Confidence = ConfidenceMedium + } + + result.Description = fmt.Sprintf("%s, salt: %#X", result.Description, salt) + return result, nil +} + +// ---- GPG ---- + +const DescriptionGPGSigned = "GPG signed file" + +// gpgSignedMagic returns magic bytes for GPG signed files. +func gpgSignedMagic() [][]byte { + return [][]byte{ + {0xA3, 0x01}, + } +} + +// gpgSignedParser validates GPG signed file signatures. +// The Rust version does a zlib decompression dry run; here we do a simplified validation. +func gpgSignedParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Confidence: ConfidenceHigh, + Description: DescriptionGPGSigned, + } + + // GPG signed files are zlib compressed with magic bytes replaced. + // Validate there is enough data and the zlib-like stream follows. + if offset+4 > len(data) { + return nil, fmt.Errorf("insufficient GPG data") + } + + // The third byte should be a valid zlib CMF byte (deflate method = 8) + cmf := data[offset+2] + if cmf&0x0F != 8 { + return nil, fmt.Errorf("GPG: invalid deflate method in underlying zlib stream") + } + + result.Description = fmt.Sprintf("%s, compressed data at offset %d", result.Description, offset) + return result, nil +} + +// ---- PEM ---- + +const ( + DescriptionPEMCertificate = "PEM certificate" + DescriptionPEMPublicKey = "PEM public key" + DescriptionPEMPrivateKey = "PEM private key" +) + +// pemCertificateMagic returns magic bytes for PEM certificates. +func pemCertificateMagic() [][]byte { + return [][]byte{ + []byte("-----BEGIN CERTIFICATE-----"), + } +} + +// pemPublicKeyMagic returns magic bytes for PEM public keys. +func pemPublicKeyMagic() [][]byte { + return [][]byte{ + []byte("-----BEGIN PUBLIC KEY-----"), + []byte("-----BEGIN RSA PUBLIC KEY-----"), + []byte("-----BEGIN DSA PUBLIC KEY-----"), + []byte("-----BEGIN ECDSA PUBLIC KEY-----"), + } +} + +// pemPrivateKeyMagic returns magic bytes for PEM private keys. +func pemPrivateKeyMagic() [][]byte { + return [][]byte{ + []byte("-----BEGIN PRIVATE KEY-----"), + []byte("-----BEGIN EC PRIVATE KEY-----"), + []byte("-----BEGIN RSA PRIVATE KEY-----"), + []byte("-----BEGIN DSA PRIVATE KEY-----"), + []byte("-----BEGIN OPENSSH PRIVATE KEY-----"), + []byte("-----BEGIN ANY PRIVATE KEY-----"), + []byte("-----BEGIN ENCRYPTED PRIVATE KEY-----"), + []byte("-----BEGIN TSS2 PRIVATE KEY-----"), + } +} + +func decodePEMData(pemData []byte) (int, error) { + const delim = "--" + + pemStr := string(pemData) + delimCount := 0 + var base64Str strings.Builder + + for _, line := range strings.Split(pemStr, "\n") { + line = strings.TrimRight(line, "\r") + if strings.HasPrefix(line, delim) { + delimCount++ + if delimCount == 2 { + break + } + continue + } + base64Str.WriteString(line) + } + + if base64Str.Len() == 0 { + return 0, fmt.Errorf("no PEM base64 data found") + } + + decoded, err := base64.StdEncoding.DecodeString(base64Str.String()) + if err != nil { + return 0, fmt.Errorf("PEM base64 decode failed: %v", err) + } + + return len(decoded), nil +} + +// pemFindEnd locates the end delimiter and returns the total PEM size. +func pemFindEnd(data []byte, offset int) (int, bool) { + const endPrefix = "-----END " + pemData := string(data[offset:]) + idx := strings.Index(pemData, endPrefix) + if idx < 0 { + return 0, false + } + // Find end of the delimiter line + rest := pemData[idx:] + nlIdx := strings.Index(rest, "\n") + if nlIdx < 0 { + return idx + len(rest), true + } + return idx + nlIdx + 1, true +} + +// pemParser validates PEM certificate and key signatures. +func pemParser(data []byte, offset int) (*SignatureResult, error) { + const minPEMLen = 26 + + result := &SignatureResult{ + Offset: offset, + Confidence: ConfidenceHigh, + } + + if offset+minPEMLen > len(data) { + return nil, fmt.Errorf("insufficient PEM data") + } + + pemMagic := data[offset : offset+minPEMLen] + pemMagicStr := string(pemMagic) + + // Determine type based on first 26 bytes + matched := false + for _, m := range pemPublicKeyMagic() { + if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) { + result.Description = DescriptionPEMPublicKey + matched = true + break + } + } + if !matched { + for _, m := range pemPrivateKeyMagic() { + if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) { + result.Description = DescriptionPEMPrivateKey + matched = true + break + } + } + } + if !matched { + for _, m := range pemCertificateMagic() { + if len(m) >= minPEMLen && strings.HasPrefix(string(m), pemMagicStr[:minPEMLen]) { + result.Description = DescriptionPEMCertificate + matched = true + break + } + } + } + if !matched { + return nil, fmt.Errorf("PEM magic not recognised") + } + + // Find the end of PEM data + pemSize, found := pemFindEnd(data, offset) + if !found || pemSize == 0 { + return nil, fmt.Errorf("PEM end delimiter not found") + } + + // Validate base64 content + if offset+pemSize > len(data) { + pemSize = len(data) - offset + } + if _, err := decodePEMData(data[offset : offset+pemSize]); err != nil { + return nil, fmt.Errorf("PEM validation failed: %v", err) + } + + if offset == 0 && pemSize == len(data) { + result.ExtractionDeclined = true + } + + result.Size = pemSize + return result, nil +} + +// ---- PKCS DER ---- + +const DescriptionDERHash = "PKCS DER hash" + +func derHashLookups() map[string][]byte { + return map[string][]byte{ + "MD5": {0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, 0x05, 0x00, 0x04, 0x10}, + "SHA1": {0x30, 0x21, 0x30, 0x09, 0x06, 0x05, 0x2b, 0x0e, 0x03, 0x02, 0x1a, 0x05, 0x00, 0x04, 0x14}, + "SHA256": {0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, 0x05, 0x00, 0x04, 0x20}, + "SHA384": {0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, 0x05, 0x00, 0x04, 0x30}, + "SHA512": {0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, 0x05, 0x00, 0x04}, + } +} + +// derHashMagic returns magic bytes for all DER hash types. +func derHashMagic() [][]byte { + var magics [][]byte + for _, m := range derHashLookups() { + mc := make([]byte, len(m)) + copy(mc, m) + magics = append(magics, mc) + } + return magics +} + +// derHashParser validates PKCS DER hash signatures. +func derHashParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: DescriptionDERHash, + Confidence: ConfidenceMedium, + } + + for name, magic := range derHashLookups() { + hashEnd := offset + len(magic) + if hashEnd > len(data) { + continue + } + match := true + for i, b := range magic { + if data[offset+i] != b { + match = false + break + } + } + if match { + result.Size = len(magic) + result.Description = fmt.Sprintf("%s, %s", result.Description, name) + return result, nil + } + } + + return nil, fmt.Errorf("DER hash validation failed") +} + +// ---- LUKS ---- + +const DescriptionLUKS = "LUKS header" + +// luksMagic returns magic bytes for LUKS headers. +func luksMagic() [][]byte { + return [][]byte{ + {0x4C, 0x55, 0x4B, 0x53, 0xBA, 0xBE}, // "LUKS\xBA\xBE" + } +} + +// luksParser validates LUKS header signatures. +func luksParser(data []byte, offset int) (*SignatureResult, error) { + const cipherAlgoStart = 8 + const cipherAlgoEnd = 40 + const cipherModeStart = 40 + const cipherModeEnd = 72 + const hashFnStart = 72 + const hashFnEnd = 104 + const luks2MinHeaderSize = 4032 + + luksBaseFields := []StructField{ + {"magic_1", "u32"}, + {"magic_2", "u16"}, + {"version", "u16"}, + {"header_size", "u64"}, + } + + result := &SignatureResult{ + Offset: offset, + Name: "luks", + Description: DescriptionLUKS, + Confidence: ConfidenceMedium, + } + + if offset+StructSize(luksBaseFields) > len(data) { + return nil, fmt.Errorf("insufficient LUKS data") + } + + luksBase, err := Parse(data[offset:], luksBaseFields, "big") + if err != nil { + return nil, fmt.Errorf("failed to parse LUKS header") + } + + version := luksBase["version"] + + // Hash function string + if offset+hashFnEnd > len(data) { + return nil, fmt.Errorf("LUKS hash function data out of bounds") + } + hashFn := GetCString(data[offset+hashFnStart : offset+hashFnEnd]) + if hashFn == "" { + return nil, fmt.Errorf("LUKS hash function string empty") + } + + if version == 1 { + if offset+cipherAlgoEnd > len(data) { + return nil, fmt.Errorf("LUKS cipher data out of bounds") + } + cipherAlgo := GetCString(data[offset+cipherAlgoStart : offset+cipherAlgoEnd]) + cipherMode := GetCString(data[offset+cipherModeStart : offset+cipherModeEnd]) + if cipherAlgo == "" || cipherMode == "" { + return nil, fmt.Errorf("LUKS v1 cipher strings empty") + } + result.Description = fmt.Sprintf("%s, version: %d, cipher algorithm: %s, cipher mode: %s, hash fn: %s", + result.Description, version, cipherAlgo, cipherMode, hashFn) + return result, nil + } else if version == 2 { + headerSize := luksBase["header_size"] + availableData := len(data) - offset + if headerSize <= luks2MinHeaderSize || headerSize >= availableData { + return nil, fmt.Errorf("LUKS v2 header size invalid") + } + result.Description = fmt.Sprintf("%s, version: %d, header size: %d bytes, hash fn: %s", + result.Description, version, headerSize, hashFn) + return result, nil + } + + return nil, fmt.Errorf("unsupported LUKS version: %d", version) +} + +// ---- Hash constants ---- + +const hashMagicLen = 16 + +const ( + DescriptionCRC32 = "CRC32 polynomial table" + DescriptionSHA256 = "SHA256 hash constants" + DescriptionMD5 = "MD5 hash constants" +) + +// crc32Magic returns magic bytes for CRC32 polynomial tables. +// Order matters: index 0 is big endian, index 1 is little endian. +func crc32ConstMagic() [][]byte { + return [][]byte{ + // Big endian + {0x00, 0x00, 0x00, 0x00, 0x77, 0x07, 0x30, 0x96, 0xEE, 0x0E, 0x61, 0x2C, 0x99, 0x09, 0x51, 0xBA}, + // Little endian + {0x00, 0x00, 0x00, 0x00, 0x96, 0x30, 0x07, 0x77, 0x2C, 0x61, 0x0E, 0xEE, 0xBA, 0x51, 0x09, 0x99}, + } +} + +// sha256Magic returns magic bytes for SHA256 hash constants. +func sha256ConstMagic() [][]byte { + return [][]byte{ + // Big endian + {0x42, 0x8a, 0x2f, 0x98, 0x71, 0x37, 0x44, 0x91, 0xb5, 0xc0, 0xfb, 0xcf, 0xe9, 0xb5, 0xdb, 0xa5}, + // Little endian + {0x98, 0x2f, 0x8a, 0x42, 0x91, 0x44, 0x37, 0x71, 0xcf, 0xfb, 0xc0, 0xb5, 0xa5, 0xdb, 0xb5, 0xe9}, + } +} + +// md5Magic returns magic bytes for MD5 hash constants. +func md5ConstMagic() [][]byte { + return [][]byte{ + // Big endian + {0xd7, 0x6a, 0xa4, 0x78, 0xe8, 0xc7, 0xb7, 0x56, 0x24, 0x20, 0x70, 0xdb, 0xc1, 0xbd, 0xce, 0xee}, + // Little endian + {0x78, 0xa4, 0x6a, 0xd7, 0x56, 0xb7, 0xc7, 0xe8, 0xdb, 0x70, 0x20, 0x24, 0xee, 0xce, 0xbd, 0xc1}, + } +} + +func hashEndianness(data []byte, offset int, magics [][]byte) string { + if offset+hashMagicLen > len(data) { + return "little" + } + thisMagic := data[offset : offset+hashMagicLen] + if len(magics) > 0 && len(magics[0]) == hashMagicLen { + match := true + for i := 0; i < hashMagicLen; i++ { + if thisMagic[i] != magics[0][i] { + match = false + break + } + } + if match { + return "big" + } + } + return "little" +} + +// crc32Parser validates CRC32 polynomial table signatures. +func crc32Parser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Size: hashMagicLen, + Description: fmt.Sprintf("%s, %s endian", DescriptionCRC32, hashEndianness(data, offset, crc32ConstMagic())), + }, nil +} + +// sha256Parser validates SHA256 hash constant signatures. +func sha256Parser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Size: hashMagicLen, + Description: fmt.Sprintf("%s, %s endian", DescriptionSHA256, hashEndianness(data, offset, sha256ConstMagic())), + }, nil +} + +// md5Parser validates MD5 hash constant signatures. +func md5Parser(data []byte, offset int) (*SignatureResult, error) { + return &SignatureResult{ + Offset: offset, + Size: hashMagicLen, + Description: fmt.Sprintf("%s, %s endian", DescriptionMD5, hashEndianness(data, offset, md5ConstMagic())), + }, nil +} diff --git a/internal/leviathan/binsnoop/sig_executables.go b/internal/leviathan/binsnoop/sig_executables.go new file mode 100644 index 0000000..20b3429 --- /dev/null +++ b/internal/leviathan/binsnoop/sig_executables.go @@ -0,0 +1,485 @@ +package binsnoop + +import ( + "fmt" +) + +// --------------------------------------------------------------------------- +// ELF +// --------------------------------------------------------------------------- + +const ELFDescription = "ELF binary" + +func ELFMagic() [][]byte { + return [][]byte{ + {0x7F, 0x45, 0x4C, 0x46}, // "\x7FELF" + } +} + +func ELFParser(data []byte, offset int) (*SignatureResult, error) { + const elfIdentSize = 16 + const elfInfoStructSize = 8 + const expectedVersion = 1 + + elfIdentFields := []StructField{ + {"magic", "u32"}, + {"class", "u8"}, + {"endianness", "u8"}, + {"version", "u8"}, + {"osabi", "u8"}, + {"abiversion", "u8"}, + {"padding_1", "u32"}, + {"padding_2", "u24"}, + } + + elfInfoFields := []StructField{ + {"type", "u16"}, + {"machine", "u16"}, + {"version", "u32"}, + } + + elfClasses := map[int]int{ + 1: 32, + 2: 64, + } + + elfEndianness := map[int]string{ + 1: "little", + 2: "big", + } + + elfOSABI := map[int]string{ + 0: "System-V (Unix)", + 1: "HP-UX", + 2: "NetBSD", + 3: "Linux", + 4: "GNU Hurd", + 5: "86Open", + 6: "Solaris", + 7: "AIX", + 8: "IRIX", + 9: "FreeBSD", + 10: "Tru64", + 11: "Novell Modesto", + 12: "OpenBSD", + 13: "OpenVMS", + 14: "NonStop Kernel", + 15: "AROS", + 16: "FenixOS", + 17: "Nuxi CloudABI", + 18: "OpenVOS", + 97: "ARM ABI", + 102: "Cell LV2", + 202: "Cafe OS", + 255: "embedded", + } + + elfTypes := map[int]string{ + 0: "no file type", + 1: "relocatable", + 2: "executable", + 3: "shared object", + 4: "core file", + } + + elfMachines := map[int]string{ + 0: "no machine", + 1: "AT&T WE 32100", + 2: "SPARC", + 3: "x86", + 4: "Motorola 68k", + 5: "Motorola 88k", + 6: "Intel MCU", + 7: "Intel 80860", + 8: "MIPS", + 9: "IBM System/370", + 10: "MIPS RS3000", + 11: "RS6000", + 15: "HP PA-RISC", + 16: "nCUBE", + 17: "Fujitsu VPP500", + 18: "SPARC32PLUS", + 19: "Intel 80960", + 20: "PowerPC", + 21: "PowerPC 64-bit", + 22: "S390", + 23: "IBM SPU/SPC", + 24: "cisco SVIP", + 25: "cisco 7200", + 36: "NEC V800", + 37: "Fujitsu FR20", + 38: "TRW RH-32", + 39: "Motorola RCE", + 40: "ARM", + 41: "Digital Alpha", + 42: "SuperH", + 43: "SPARCv9", + 44: "Siemens TriCore embedded processor", + 45: "Argonaut RISC Core", + 46: "Hitachi H8/300", + 47: "Hitachi H8/300H", + 48: "Hitachi H8S", + 49: "Hitachi H8/500", + 50: "IA-64", + 51: "Stanford MIPS-X", + 52: "Motorola ColdFire", + 53: "Motorola M68HC12", + 54: "Fujitsu MMA Multimedia Accelerator", + 55: "Siemens PCP", + 56: "Sony nCPU embedded RISC processor", + 57: "Denso NDR1 microprocessor", + 58: "Motorola StarCore", + 59: "Toyota ME16", + 60: "STMicroelectronics ST100", + 61: "Advanced Logic TinyJ embedded processor", + 62: "AMD X86-64", + 63: "Sony DSP processor", + 64: "PDP-10", + 65: "PDP-11", + 66: "Siemens FX66", + 67: "STMicroelectronics ST9+", + 68: "STMicroelectronics ST7", + 69: "Motorola MC68HC16", + 70: "Motorola MC68HC11", + 71: "Motorola MC68HC08", + 72: "Motorola MC68HC05", + 73: "Silicon Graphics SVx", + 74: "STMicroelectonrics ST19", + 75: "Digital VAX", + 76: "Axis Communications 32-bit CPU", + 77: "Infineon Technologies 32-bit CPU", + 78: "Element 14 64-bit DSP", + 79: "LSI Logic 16-bit DSP", + 80: "MMIX", + 81: "Harvard machine-independent", + 82: "SiTera Prism", + 83: "Atmel AVR 8-bit", + 84: "Fujitsu FR30", + 85: "Mitsubishi D10V", + 86: "Mitsubishi D30V", + 87: "NEC v850", + 88: "Renesas M32R", + 89: "Matsushita MN10300", + 90: "Matsushita MN10200", + 91: "picoJava", + 92: "OpenRISC", + 93: "Synopsys ARCompact ARC700 cores", + 94: "Tensilica Xtensa", + 95: "Alphamosaic VideoCore", + 96: "Thompson Multimedia", + 97: "NatSemi 32k", + 98: "Tenor Network TPC", + 99: "Trebia SNP 1000", + 100: "STMicroelectronics ST200", + 101: "Ubicom IP2022", + 102: "MAX Processor", + 103: "NatSemi CompactRISC", + 104: "Fujitsu F2MC16", + 105: "TI msp430", + 106: "Analog Devices Blackfin", + 107: "S1C33 Family of Seiko Epson", + 108: "Sharp embedded", + 109: "Arca RISC", + 110: "PKU-Unity Ltd.", + 111: "eXcess: 16/32/64-bit", + 112: "Icera Deep Execution Processor", + 113: "Altera Nios II", + 114: "NatSemi CRX", + 115: "Motorola XGATE", + 116: "Infineon C16x/XC16x", + 117: "Renesas M16C series", + 118: "Microchip dsPIC30F", + 119: "Freescale RISC core", + 120: "Renesas M32C series", + 131: "Altium TSK3000 core", + 132: "Freescale RS08", + 134: "Cyan Technology eCOG2", + 135: "Sunplus S+core7 RISC", + 136: "New Japan Radio (NJR) 24-bit DSP", + 137: "Broadcom VideoCore III processor", + 138: "LatticeMico32", + 139: "Seiko Epson C17 family", + 140: "TMS320C6000", + 141: "TMS320C2000", + 142: "TMS320C55x", + 144: "TI Programmable Realtime Unit", + 160: "STMicroelectronics 64bit VLIW DSP", + 161: "Cypress M8C", + 162: "Renesas R32C series", + 163: "NXP TriMedia family", + 164: "Qualcomm DSP6", + 165: "Intel 8051 and variants", + 166: "STMicroelectronics STxP7x family", + 167: "Andes embedded RISC", + 168: "Cyan eCOG1X family", + 169: "Dallas MAXQ30", + 170: "New Japan Radio (NJR) 16-bit DSP", + 171: "M2000 Reconfigurable RISC", + 172: "Cray NV2 vector architecture", + 173: "Renesas RX family", + 174: "META", + 175: "MCST Elbrus e2k", + 176: "Cyan Technology eCOG16 family", + 177: "NatSemi CompactRISC", + 178: "Freescale Extended Time Processing Unit", + 179: "Infineon SLE9X", + 180: "Intel L1OM", + 181: "Intel K1OM", + 183: "ARM 64-bit", + 185: "Atmel 32-bit family", + 186: "STMicroeletronics STM8 8-bit", + 187: "Tilera TILE64", + 188: "Tilera TILEPro", + 189: "Xilinx MicroBlaze 32-bit RISC", + 190: "NVIDIA CUDA architecture", + 191: "Tilera TILE-Gx", + 195: "Synopsys ARCv2/HS3x/HS4x cores", + 197: "Renesas RL78 family", + 199: "Renesas 78K0R", + 200: "Freescale 56800EX", + 201: "Beyond BA1", + 202: "Beyond BA2", + 203: "XMOS xCORE", + 204: "Microchip 8-bit PIC(r)", + 210: "KM211 KM32", + 211: "KM211 KMX32", + 212: "KM211 KMX16", + 213: "KM211 KMX8", + 214: "KM211 KVARC", + 215: "Paneve CDP", + 216: "Cognitive Smart Memory", + 217: "iCelero CoolEngine", + 218: "Nanoradio Optimized RISC", + 219: "CSR Kalimba architecture family", + 220: "Zilog Z80", + 221: "Controls and Data Services VISIUMcore processor", + 222: "FTDI Chip FT32 high performance 32-bit RISC architecture", + 223: "Moxie processor family", + 224: "AMD GPU architecture", + 243: "RISC-V", + 244: "Lanai 32-bit processor", + 245: "CEVA Processor Architecture Family", + 246: "CEVA X2 Processor Family", + 247: "Berkeley Packet Filter", + 248: "Graphcore Intelligent Processing Unit", + 249: "Imagination Technologies", + 250: "Netronome Flow Processor", + 251: "NEC Vector Engine", + 252: "C-SKY processor family", + 253: "Synopsys ARCv3 64-bit ISA/HS6x cores", + 254: "MOS Technology MCS 6502 processor", + 255: "Synopsys ARCv3 32-bit", + 256: "Kalray VLIW core of the MPPA family", + 257: "WDC 65C816", + 258: "LoongArch", + 259: "ChipON KungFu32", + } + + elfData := data[offset:] + if len(elfData) < elfIdentSize+elfInfoStructSize { + return nil, fmt.Errorf("ELF: not enough data") + } + + // Parse the e_ident (endianness doesn't matter for single-byte fields) + eIdent, err := Parse(elfData, elfIdentFields, "little") + if err != nil { + return nil, fmt.Errorf("ELF: e_ident parse error: %w", err) + } + + // Sanity check padding fields + if eIdent["padding_1"] != 0 || eIdent["padding_2"] != 0 { + return nil, fmt.Errorf("ELF: non-zero padding in e_ident") + } + if eIdent["version"] != expectedVersion { + return nil, fmt.Errorf("ELF: unexpected e_ident version %d", eIdent["version"]) + } + + classVal, classOK := elfClasses[eIdent["class"]] + if !classOK { + return nil, fmt.Errorf("ELF: unknown class %d", eIdent["class"]) + } + + endianStr, endianOK := elfEndianness[eIdent["endianness"]] + if !endianOK { + return nil, fmt.Errorf("ELF: unknown endianness %d", eIdent["endianness"]) + } + + osabiStr, osabiOK := elfOSABI[eIdent["osabi"]] + if !osabiOK { + return nil, fmt.Errorf("ELF: unknown OS/ABI %d", eIdent["osabi"]) + } + + // Parse the rest of the ELF header with the correct endianness + elfInfoData := elfData[elfIdentSize : elfIdentSize+elfInfoStructSize] + elfInfo, err := Parse(elfInfoData, elfInfoFields, endianStr) + if err != nil { + return nil, fmt.Errorf("ELF: info parse error: %w", err) + } + + if elfInfo["version"] != expectedVersion { + return nil, fmt.Errorf("ELF: unexpected info version %d", elfInfo["version"]) + } + + typeStr, typeOK := elfTypes[elfInfo["type"]] + if !typeOK { + return nil, fmt.Errorf("ELF: unknown type %d", elfInfo["type"]) + } + + machineStr := "Unknown" + if m, ok := elfMachines[elfInfo["machine"]]; ok { + machineStr = m + } + + return &SignatureResult{ + Offset: offset, + Name: "elf", + Description: fmt.Sprintf("%s, %d-bit %s, %s for %s, %s endian", ELFDescription, classVal, typeStr, machineStr, osabiStr, endianStr), + Confidence: ConfidenceMedium, + }, nil +} + +// --------------------------------------------------------------------------- +// PE +// --------------------------------------------------------------------------- + +const PEDescription = "Windows PE binary" + +func PEMagic() [][]byte { + // First 16 bytes of common DOS headers (e_magic through e_ss) + return [][]byte{ + {0x4D, 0x5A, 0x90, 0x00, 0x03, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0xFF, 0xFF, 0x00, 0x00}, + {0x4D, 0x5A, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + } +} + +func PEParser(data []byte, offset int) (*SignatureResult, error) { + const peMagicValue = 0x00004550 // "PE\x00\x00" + + dosFields := []StructField{ + {"e_magic", "u16"}, + {"e_cblp", "u16"}, + {"e_cp", "u16"}, + {"e_crlc", "u16"}, + {"e_cparhdr", "u16"}, + {"e_minalloc", "u16"}, + {"e_maxalloc", "u16"}, + {"e_ss", "u16"}, + {"e_sp", "u16"}, + {"e_csum", "u16"}, + {"e_ip", "u16"}, + {"e_cs", "u16"}, + {"e_lfarlc", "u16"}, + {"e_ovno", "u16"}, + {"e_res_1", "u16"}, + {"e_res_2", "u16"}, + {"e_res_3", "u16"}, + {"e_res_4", "u16"}, + {"e_oemid", "u16"}, + {"e_oeminfo", "u16"}, + {"e_res_5", "u16"}, + {"e_res_6", "u16"}, + {"e_res_7", "u16"}, + {"e_res_8", "u16"}, + {"e_res_9", "u16"}, + {"e_res_10", "u16"}, + {"e_res_11", "u16"}, + {"e_res_12", "u16"}, + {"e_res_13", "u16"}, + {"e_res_14", "u16"}, + {"e_lfanew", "u32"}, + } + + peFields := []StructField{ + {"magic", "u32"}, + {"machine", "u16"}, + {"number_of_sections", "u16"}, + {"timestamp", "u32"}, + {"symbol_table_ptr", "u32"}, + {"number_of_symbols", "u32"}, + {"optional_header_size", "u16"}, + {"characteristics", "u16"}, + } + + knownMachineTypes := map[int]string{ + 0x0: "Unknown", + 0x184: "Alpha32", + 0x284: "Alpha64", + 0x1D3: "Matsushita AM33", + 0x8664: "Intel x86-64", + 0x1C0: "ARM", + 0xAA64: "ARM-64", + 0x1C4: "ARM Thumb2", + 0xEBC: "EFI", + 0x14C: "Intel x86", + 0x200: "Intel Itanium", + 0x6232: "LoongArch 32-bit", + 0x6264: "LoongArch 64-bit", + 0x9041: "Mitsubishi M32R", + 0x266: "MIPS16", + 0x366: "MIPS with FPU", + 0x466: "MIPS16 with FPU", + 0x1F0: "PowerPC", + 0x1F1: "PowerPC with FPU", + 0x5032: "RISC-V 32-bit", + 0x5064: "RISC-V 64-bit", + 0x5128: "RISC-V 128-bit", + 0x1A2: "Hitachi SH3", + 0x1A3: "Hitachi SH3 DSP", + 0x1A6: "Hitachi SH4", + 0x1A8: "Hitachi SH5", + 0x1C2: "Thumb", + 0x169: "MIPS WCEv2", + } + + peData := data[offset:] + dosStructSize := StructSize(dosFields) + peStructSize := StructSize(peFields) + + if len(peData) < dosStructSize { + return nil, fmt.Errorf("PE: not enough data for DOS header") + } + + dosHeader, err := Parse(peData, dosFields, "little") + if err != nil { + return nil, fmt.Errorf("PE: DOS header parse error: %w", err) + } + + // Sanity check the reserved header fields; they should all be 0 + for _, field := range []string{ + "e_res_1", "e_res_2", "e_res_3", "e_res_4", + "e_res_5", "e_res_6", "e_res_7", "e_res_8", + "e_res_9", "e_res_10", "e_res_11", "e_res_12", + "e_res_13", "e_res_14", + } { + if dosHeader[field] != 0 { + return nil, fmt.Errorf("PE: reserved field %s is non-zero", field) + } + } + + peHeaderStart := dosHeader["e_lfanew"] + peHeaderEnd := peHeaderStart + peStructSize + + if peHeaderEnd > len(peData) { + return nil, fmt.Errorf("PE: e_lfanew points beyond available data") + } + + peHeader, err := Parse(peData[peHeaderStart:peHeaderEnd], peFields, "little") + if err != nil { + return nil, fmt.Errorf("PE: PE header parse error: %w", err) + } + + if peHeader["magic"] != peMagicValue { + return nil, fmt.Errorf("PE: invalid PE magic 0x%08X", peHeader["magic"]) + } + + machineStr, machineOK := knownMachineTypes[peHeader["machine"]] + if !machineOK { + return nil, fmt.Errorf("PE: unknown machine type 0x%X", peHeader["machine"]) + } + + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, machine type: %s", PEDescription, machineStr), + Confidence: ConfidenceMedium, + }, nil +} diff --git a/internal/leviathan/binsnoop/sig_filesystems.go b/internal/leviathan/binsnoop/sig_filesystems.go new file mode 100644 index 0000000..4f59ffb --- /dev/null +++ b/internal/leviathan/binsnoop/sig_filesystems.go @@ -0,0 +1,1289 @@ +package binsnoop + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" +) + +// --------------------------------------------------------------------------- +// SquashFS +// --------------------------------------------------------------------------- + +const SquashFSDescription = "SquashFS file system" + +func squashfs_magic() [][]byte { + return [][]byte{ + []byte("sqsh"), + []byte("hsqs"), + []byte("sqlz"), + []byte("qshs"), + []byte("tqsh"), + []byte("hsqt"), + []byte("shsq"), + } +} + +func squashfs_parser(data []byte, offset int) (*SignatureResult, error) { + const squashfsV4 = 4 + const maxSquashfsVersion uint16 = 4 + const minSquashfsHeaderSize = 120 + const versionStart = 28 + const versionEnd = 30 + + compressionTypes := map[int]string{ + 0: "unknown", 1: "gzip", 2: "lzma", 3: "lzo", 4: "xz", 5: "lz4", 6: "zstd", + } + + available := len(data) - offset + sqshData := data[offset:] + + if len(sqshData) <= minSquashfsHeaderSize { + return nil, fmt.Errorf("not enough data for squashfs header") + } + + // Determine endianness from version field at fixed offset + endianness := "little" + squashfsVersion := int(binary.LittleEndian.Uint16(sqshData[versionStart:versionEnd])) + if squashfsVersion == 0 || squashfsVersion > int(maxSquashfsVersion) { + endianness = "big" + squashfsVersion = int(binary.BigEndian.Uint16(sqshData[versionStart:versionEnd])) + } + if squashfsVersion <= 0 || squashfsVersion > int(maxSquashfsVersion) { + return nil, fmt.Errorf("invalid squashfs version") + } + + // Pick the right structure + var fields []StructField + if squashfsVersion == squashfsV4 { + fields = []StructField{ + {"magic", "u32"}, {"inode_count", "u32"}, {"modification_time", "u32"}, + {"block_size", "u32"}, {"fragment_count", "u32"}, {"compression_id", "u16"}, + {"block_log", "u16"}, {"flags", "u16"}, {"id_count", "u16"}, + {"major_version", "u16"}, {"minor_version", "u16"}, + {"root_inode_ref", "u64"}, {"image_size", "u64"}, {"uid_start", "u64"}, + } + } else { + fields = []StructField{ + {"magic", "u32"}, {"inode_count", "u32"}, + {"bytes_used_2", "u32"}, {"uid_start_2", "u32"}, {"guid_start_2", "u32"}, + {"inode_table_start_2", "u32"}, {"directory_table_start_2", "u32"}, + {"major_version", "u16"}, {"minor_version", "u16"}, + {"block_size_1", "u16"}, {"block_log", "u16"}, + {"flags", "u8"}, {"uid_count", "u8"}, {"guid_count", "u8"}, + {"modification_time", "u32"}, + {"root_inode_ref", "u64"}, {"block_size", "u32"}, + {"fragment_entry_count", "u32"}, {"fragment_table_start_2", "u32"}, + {"image_size", "u64"}, {"uid_start", "u64"}, {"guid_start", "u64"}, + {"inode_table_start", "u64"}, {"directory_table_start", "u64"}, + {"fragment_table_start", "u64"}, {"lookup_table_start", "u64"}, + } + } + + hdr, err := Parse(sqshData, fields, endianness) + if err != nil { + return nil, fmt.Errorf("squashfs header parse failed") + } + + // For v1/v2 images, adjust fields + if squashfsVersion < 3 { + hdr["uid_start"] = hdr["uid_start_2"] + hdr["image_size"] = hdr["bytes_used_2"] + } + + headerSize := StructSize(fields) + imageSize := hdr["image_size"] + + if imageSize <= minSquashfsHeaderSize { + return nil, fmt.Errorf("squashfs image size too small") + } + if imageSize > available { + return nil, fmt.Errorf("squashfs image size exceeds available data") + } + + // Validate block_size vs block_log + blockSize := hdr["block_size"] + blockLog := hdr["block_log"] + if blockSize <= 0 || blockLog != int(math.Log2(float64(blockSize))) { + return nil, fmt.Errorf("squashfs block_size/block_log mismatch") + } + + // Validate UID table + uidTableStart := offset + hdr["uid_start"] + if uidTableStart <= headerSize { + return nil, fmt.Errorf("squashfs uid table offset invalid") + } + if uidTableStart >= len(data) { + return nil, fmt.Errorf("squashfs uid table out of bounds") + } + + // Parse one UID entry + uidData := data[uidTableStart:] + var uidFields []StructField + if squashfsVersion == squashfsV4 { + uidFields = []StructField{{"uid_block_ptr", "u64"}} + } else { + uidFields = []StructField{{"uid_block_ptr", "u32"}} + } + uidEntry, err := Parse(uidData, uidFields, endianness) + if err != nil { + return nil, fmt.Errorf("squashfs uid entry parse failed") + } + uidPtr := uidEntry["uid_block_ptr"] + if uidPtr != 0 && (uidPtr <= headerSize || uidPtr > imageSize) { + return nil, fmt.Errorf("squashfs uid table pointer invalid") + } + + compressionID := hdr["compression_id"] + if squashfsVersion < 4 { + // v3 headers don't have compression_id; use 0 (unknown) + compressionID = 0 + } + compStr, ok := compressionTypes[compressionID] + if !ok { + return nil, fmt.Errorf("squashfs unknown compression type") + } + + createDate := EpochToString(uint32(hdr["modification_time"])) + + return &SignatureResult{ + Offset: offset, + Size: imageSize, + Confidence: ConfidenceHigh, + Description: fmt.Sprintf( + "%s, %s endian, version: %d.%d, compression: %s, inode count: %d, block size: %d, image size: %d bytes, created: %s", + SquashFSDescription, endianness, + hdr["major_version"], hdr["minor_version"], + compStr, hdr["inode_count"], blockSize, imageSize, createDate, + ), + }, nil +} + +// --------------------------------------------------------------------------- +// CramFS +// --------------------------------------------------------------------------- + +const CramFSDescription = "CramFS filesystem" + +func cramfs_magic() [][]byte { + return [][]byte{ + []byte("Compressed ROMFS"), + } +} + +func cramfs_parser(data []byte, offset int) (*SignatureResult, error) { + const signatureOffset = 16 + const crcStartOffset = 32 + const crcEndOffset = 36 + const bigEndianMagic = 0x453DCD28 + const littleEndianMagic = 0x28CD3D45 + + realOffset := offset - signatureOffset + if realOffset < 0 || realOffset >= len(data) { + return nil, fmt.Errorf("cramfs offset out of bounds") + } + + cramfsData := data[realOffset:] + + fields := []StructField{ + {"magic", "u32"}, {"size", "u32"}, {"flags", "u32"}, {"future", "u32"}, + {"signature_p1", "u64"}, {"signature_p2", "u64"}, + {"checksum", "u32"}, {"edition", "u32"}, {"block_count", "u32"}, {"file_count", "u32"}, + } + + endianness := "little" + hdr, err := Parse(cramfsData, fields, endianness) + if err != nil { + return nil, fmt.Errorf("cramfs parse failed") + } + + magic := hdr["magic"] + if magic != bigEndianMagic && magic != littleEndianMagic { + return nil, fmt.Errorf("cramfs invalid magic") + } + if magic == bigEndianMagic { + endianness = "big" + hdr, err = Parse(cramfsData, fields, endianness) + if err != nil { + return nil, fmt.Errorf("cramfs big endian parse failed") + } + } + + structSize := StructSize(fields) + if hdr["size"] <= structSize { + return nil, fmt.Errorf("cramfs reported size too small") + } + + imageSize := hdr["size"] + checksum := uint32(hdr["checksum"]) + fileCount := hdr["file_count"] + + if realOffset+imageSize > len(data) { + return nil, fmt.Errorf("cramfs image exceeds available data") + } + + // Copy image data and null out checksum field for CRC validation + cramfsImage := make([]byte, imageSize) + copy(cramfsImage, cramfsData[:imageSize]) + for i := crcStartOffset; i < crcEndOffset && i < imageSize; i++ { + cramfsImage[i] = 0 + } + + errorMsg := "" + confidence := ConfidenceHigh + if CRC32(cramfsImage) != checksum { + errorMsg = " (checksum error)" + confidence = ConfidenceMedium + } + + return &SignatureResult{ + Offset: realOffset, + Size: imageSize, + Confidence: confidence, + Description: fmt.Sprintf("%s, %s endian, %d files, total size: %d bytes%s", + CramFSDescription, endianness, fileCount, imageSize, errorMsg), + }, nil +} + +// --------------------------------------------------------------------------- +// JFFS2 +// --------------------------------------------------------------------------- + +const JFFS2Description = "JFFS2 filesystem" + +func jffs2_magic() [][]byte { + return [][]byte{ + {0x19, 0x85, 0xE0, 0x01}, + {0x19, 0x85, 0xE0, 0x02}, + {0x19, 0x85, 0x20, 0x03}, + {0x85, 0x19, 0x01, 0xE0}, + {0x85, 0x19, 0x02, 0xE0}, + {0x85, 0x19, 0x03, 0x20}, + } +} + +func jffs2_parser(data []byte, offset int) (*SignatureResult, error) { + const maxPageSize = 0x20000 + const minValidNodeCount = 2 + const jffs2CorrectMagic = 0x1985 + const headerCRCSize = 8 + + jffs2NodeFields := []StructField{ + {"magic", "u16"}, {"type", "u16"}, {"size", "u32"}, {"crc", "u32"}, + } + jffs2NodeStructSize := StructSize(jffs2NodeFields) // 12 + + // Parse first node + firstNode, endianness, err := parseJFFS2NodeHeader(data[offset:], jffs2NodeFields, headerCRCSize, jffs2CorrectMagic) + if err != nil { + return nil, err + } + + jffs2EOF := offset + jffs2Roundup(firstNode["size"]) + if jffs2EOF >= len(data) { + return nil, fmt.Errorf("jffs2 first node exceeds data") + } + + var nodeMagic []byte + if endianness == "big" { + nodeMagic = []byte{0x19, 0x85} + } else { + nodeMagic = []byte{0x85, 0x19} + } + + nodeCount := 1 + + // Scan for subsequent nodes + searchData := data[jffs2EOF:] + for i := 0; i < len(searchData); { + idx := bytes.Index(searchData[i:], nodeMagic) + if idx < 0 { + break + } + absPos := jffs2EOF + i + idx + if absPos < jffs2EOF { + i += idx + 1 + continue + } + if absPos-jffs2EOF > maxPageSize { + break + } + headerEnd := absPos + jffs2NodeStructSize + if headerEnd > len(data) { + break + } + nodeHdr, _, err := parseJFFS2NodeHeader(data[absPos:headerEnd], jffs2NodeFields, headerCRCSize, jffs2CorrectMagic) + if err == nil { + nodeCount++ + jffs2EOF = absPos + jffs2Roundup(nodeHdr["size"]) + } + i += idx + 1 + } + + if nodeCount <= minValidNodeCount { + return nil, fmt.Errorf("jffs2 too few valid nodes: %d", nodeCount) + } + + totalSize := jffs2EOF - offset + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Confidence: ConfidenceHigh, + Description: fmt.Sprintf("%s, %s endian, nodes: %d, total size: %d bytes", + JFFS2Description, endianness, nodeCount, totalSize), + }, nil +} + +func parseJFFS2NodeHeader(nodeData []byte, fields []StructField, headerCRCSize, correctMagic int) (map[string]int, string, error) { + endianness := "little" + hdr, err := Parse(nodeData, fields, endianness) + if err != nil { + return nil, "", fmt.Errorf("jffs2 node parse failed") + } + if hdr["magic"] != correctMagic { + endianness = "big" + hdr, err = Parse(nodeData, fields, endianness) + if err != nil { + return nil, "", fmt.Errorf("jffs2 node big endian parse failed") + } + } + if hdr["magic"] != correctMagic { + return nil, "", fmt.Errorf("jffs2 node invalid magic") + } + + // Validate header CRC + if headerCRCSize > len(nodeData) { + return nil, "", fmt.Errorf("jffs2 not enough data for crc") + } + calcCRC := int(jffs2NodeCRC(nodeData[:headerCRCSize])) + if calcCRC != hdr["crc"] { + return nil, "", fmt.Errorf("jffs2 node header crc mismatch") + } + + return hdr, endianness, nil +} + +func jffs2NodeCRC(data []byte) uint32 { + return CRC32(data) // CRC32 uses IEEE polynomial; JFFS2: crc32(0xFFFFFFFF, data) ^ 0xFFFFFFFF which is standard CRC32 +} + +func jffs2Roundup(num int) int { + base := 4 + return ((num + base - 1) / base) * base +} + +// --------------------------------------------------------------------------- +// YAFFS +// --------------------------------------------------------------------------- + +const YAFFSDescription = "YAFFSv2 filesystem" + +func yaffs_magic() [][]byte { + return [][]byte{ + {0x03, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF}, + {0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x01, 0xFF, 0xFF}, + {0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0xFF, 0xFF}, + {0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0xFF, 0xFF}, + } +} + +func yaffs_parser(data []byte, offset int) (*SignatureResult, error) { + const maxObjSize = 16896 + const minNumberOfObjs = 2 + const bigEndianFirstByte byte = 0 + const fileType = 1 + + available := len(data) + requiredMinOffset := offset + (maxObjSize * minNumberOfObjs) + if !IsOffsetSafe(available, requiredMinOffset, nil) { + return nil, fmt.Errorf("yaffs not enough data") + } + + endianness := "little" + if data[offset] == bigEndianFirstByte { + endianness = "big" + } + + yaffsData := data[offset:] + + // Determine page size + pageSize, err := yaffsGetPageSize(yaffsData) + if err != nil { + return nil, err + } + + // Determine spare size + spareSize, err := yaffsGetSpareSize(yaffsData, pageSize, endianness) + if err != nil { + return nil, err + } + + // Get total image size + imageSize, err := yaffsGetImageSize(yaffsData, pageSize, spareSize, endianness) + if err != nil { + return nil, err + } + + return &SignatureResult{ + Offset: offset, + Size: imageSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf("%s, %s endian, page size: %d, spare size: %d, image size: %d bytes", + YAFFSDescription, endianness, pageSize, spareSize, imageSize), + }, nil +} + +func yaffsGetPageSize(fileData []byte) (int, error) { + spareMagics := [][]byte{ + {0x00, 0x00, 0x10, 0x00}, + {0x00, 0x10, 0x00, 0x00}, + {0xFF, 0xFF, 0x00, 0x00, 0x10, 0x00}, + {0xFF, 0xFF, 0x00, 0x10, 0x00, 0x00}, + } + pageSizes := []int{512, 1024, 2048, 4096, 8192, 16384} + + for _, ps := range pageSizes { + for _, sm := range spareMagics { + start := ps + end := start + len(sm) + if end <= len(fileData) { + if bytes.Equal(fileData[start:end], sm) { + return ps, nil + } + } + } + } + return 0, fmt.Errorf("yaffs page size not detected") +} + +func yaffsGetSpareSize(fileData []byte, pageSize int, endianness string) (int, error) { + const minNumberOfObjs = 2 + spareSizes := []int{16, 32, 64, 128, 256, 512} + + yaffsObjFields := []StructField{ + {"type", "u32"}, {"parent_id", "u32"}, {"name_checksum", "u16"}, + } + + for _, ss := range spareSizes { + nextObjOffset := (pageSize + ss) * minNumberOfObjs + if nextObjOffset < len(fileData) { + hdr, err := Parse(fileData[nextObjOffset:], yaffsObjFields, endianness) + if err == nil { + allowedTypes := []int{0, 1, 2, 3, 4, 5} + if containsInt(allowedTypes, hdr["type"]) && hdr["parent_id"] > 0 && hdr["name_checksum"] == 0xFFFF { + return ss, nil + } + } + } + } + return 0, fmt.Errorf("yaffs spare size not detected") +} + +func yaffsGetImageSize(fileData []byte, pageSize, spareSize int, endianness string) (int, error) { + const fileType = 1 + const minNumberOfObjs = 2 + const infoStructStart = 268 + + yaffsObjFields := []StructField{ + {"type", "u32"}, {"parent_id", "u32"}, {"name_checksum", "u16"}, + } + yaffsFileInfo := []StructField{ + {"mode", "u32"}, {"uid", "u32"}, {"gid", "u32"}, + {"atime", "u32"}, {"mtime", "u32"}, {"ctime", "u32"}, {"file_size", "u32"}, + } + + imageSize := 0 + nextObjOffset := 0 + var previousObjOffset *int + availableData := len(fileData) + blockSize := pageSize + spareSize + + for IsOffsetSafe(availableData, nextObjOffset, previousObjOffset) { + objData := fileData[nextObjOffset:] + hdr, err := Parse(objData, yaffsObjFields, endianness) + if err != nil { + break + } + allowedTypes := []int{0, 1, 2, 3, 4, 5} + if !containsInt(allowedTypes, hdr["type"]) || hdr["parent_id"] <= 0 || hdr["name_checksum"] != 0xFFFF { + break + } + + dataBlocks := 1 + if hdr["type"] == fileType { + if infoStructStart < len(objData) { + fileInfo, err := Parse(objData[infoStructStart:], yaffsFileInfo, endianness) + if err != nil { + return 0, fmt.Errorf("yaffs file header parse failed") + } + fileBlockCount := int(math.Ceil(float64(fileInfo["file_size"]) / float64(pageSize))) + dataBlocks += fileBlockCount + } else { + return 0, fmt.Errorf("yaffs not enough data for file header") + } + } + + prev := nextObjOffset + previousObjOffset = &prev + imageSize += dataBlocks * blockSize + nextObjOffset = imageSize + } + + if imageSize > blockSize*minNumberOfObjs && imageSize <= availableData { + return imageSize, nil + } + return 0, fmt.Errorf("yaffs image size invalid") +} + +// --------------------------------------------------------------------------- +// RomFS +// --------------------------------------------------------------------------- + +const RomFSDescription = "RomFS filesystem" + +func romfs_magic() [][]byte { + return [][]byte{ + []byte("-rom1fs-"), + } +} + +func romfs_parser(data []byte, offset int) (*SignatureResult, error) { + const maxHeaderCRCDataLen = 512 + + headerFields := []StructField{ + {"magic", "u64"}, {"image_size", "u32"}, {"checksum", "u32"}, + } + + romfsData := data[offset:] + headerSize := StructSize(headerFields) + + hdr, err := Parse(romfsData, headerFields, "big") + if err != nil { + return nil, fmt.Errorf("romfs header parse failed") + } + + if hdr["image_size"] <= headerSize { + return nil, fmt.Errorf("romfs image size too small") + } + + // Get volume name + volumeName := "" + if headerSize < len(romfsData) { + volumeName = GetCString(romfsData[headerSize:]) + } + + // Validate CRC + crcDataLen := maxHeaderCRCDataLen + if hdr["image_size"] < crcDataLen { + crcDataLen = hdr["image_size"] + } + if crcDataLen > len(romfsData) { + return nil, fmt.Errorf("romfs not enough data for crc") + } + if !romfsCRCValid(romfsData[:crcDataLen]) { + return nil, fmt.Errorf("romfs crc invalid") + } + + return &SignatureResult{ + Offset: offset, + Size: hdr["image_size"], + Confidence: ConfidenceHigh, + Description: fmt.Sprintf(`%s, volume name: "%s", total size: %d bytes`, + RomFSDescription, volumeName, hdr["image_size"]), + }, nil +} + +func romfsCRCValid(crcData []byte) bool { + wordSize := 4 + if len(crcData)%wordSize != 0 { + return false + } + var sum uint32 + for i := 0; i < len(crcData); i += wordSize { + sum += binary.BigEndian.Uint32(crcData[i : i+wordSize]) + } + return sum == 0 +} + +// --------------------------------------------------------------------------- +// EXT +// --------------------------------------------------------------------------- + +const EXTDescription = "EXT filesystem" + +func ext_magic() [][]byte { + return [][]byte{ + {0x53, 0xEF, 0x01, 0x00, 0x01, 0x00, 0x00, 0x00}, + {0x53, 0xEF, 0x01, 0x00, 0x02, 0x00, 0x00, 0x00}, + {0x53, 0xEF, 0x01, 0x00, 0x03, 0x00, 0x00, 0x00}, + {0x53, 0xEF, 0x02, 0x00, 0x01, 0x00, 0x00, 0x00}, + {0x53, 0xEF, 0x02, 0x00, 0x02, 0x00, 0x00, 0x00}, + {0x53, 0xEF, 0x02, 0x00, 0x03, 0x00, 0x00, 0x00}, + } +} + +const EXTMagicOffset = 1080 + +func ext_parser(data []byte, offset int) (*SignatureResult, error) { + const magicOffset = 1080 + const superblockOffset = 1024 + const superblockSize = 1024 + const maxBlockLog = 2 + + realOffset := offset - magicOffset + if realOffset < 0 { + return nil, fmt.Errorf("ext offset out of bounds") + } + + extData := data[realOffset:] + if len(extData) < superblockOffset+superblockSize { + return nil, fmt.Errorf("ext not enough data for superblock") + } + + extSBFields := []StructField{ + {"inodes_count", "u32"}, {"blocks_count", "u32"}, {"reserved_blocks_count", "u32"}, + {"free_blocks_count", "u32"}, {"free_inodes_count", "u32"}, {"first_data_block", "u32"}, + {"log_block_size", "u32"}, {"log_frag_size", "u32"}, {"blocks_per_group", "u32"}, + {"frags_per_group", "u32"}, {"inodes_per_group", "u32"}, {"modification_time", "u32"}, + {"write_time", "u32"}, {"mount_count", "u16"}, {"max_mount_count", "u16"}, + {"magic", "u16"}, {"state", "u16"}, {"errors", "u16"}, {"s_minor_rev_level", "u16"}, + {"last_check", "u32"}, {"check_interval", "u32"}, {"creator_os", "u32"}, + {"s_rev_level", "u32"}, {"resuid", "u16"}, {"resgid", "u16"}, + } + + hdr, err := Parse(extData[superblockOffset:], extSBFields, "little") + if err != nil { + return nil, fmt.Errorf("ext superblock parse failed") + } + + supportedOS := map[int]string{0: "Linux", 1: "GNU HURD", 2: "MASIX", 3: "FreeBSD", 4: "Lites"} + osName, osOk := supportedOS[hdr["creator_os"]] + if !osOk { + return nil, fmt.Errorf("ext unknown OS") + } + + allowedRevLevels := []int{0, 1} + if !containsInt(allowedRevLevels, hdr["s_rev_level"]) { + return nil, fmt.Errorf("ext invalid rev level") + } + + allowedFirstDataBlocks := []int{0, 1} + if !containsInt(allowedFirstDataBlocks, hdr["first_data_block"]) { + return nil, fmt.Errorf("ext invalid first_data_block") + } + + if hdr["log_block_size"] > maxBlockLog { + return nil, fmt.Errorf("ext invalid log_block_size") + } + + blockSize := 1024 << hdr["log_block_size"] + imageSize := blockSize * hdr["blocks_count"] + + return &SignatureResult{ + Offset: realOffset, + Size: imageSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf( + "%s for %s, inodes: %d, block size: %d, block count: %d, free blocks: %d, reserved blocks: %d, total size: %d bytes", + EXTDescription, osName, hdr["inodes_count"], blockSize, + hdr["free_blocks_count"], hdr["reserved_blocks_count"], hdr["blocks_count"], imageSize), + }, nil +} + +// --------------------------------------------------------------------------- +// FAT +// --------------------------------------------------------------------------- + +const FATDescription = "FAT file system" +const FATMagicOffset = 0x01FE + +func fat_magic() [][]byte { + return [][]byte{ + {0x55, 0xAA}, + } +} + +func fat_parser(data []byte, offset int) (*SignatureResult, error) { + const expectedFATCount = 2 + + validOpcode1 := []int{0xEB, 0xE9} + validSectorSizes := []int{512, 1024, 2048, 4096} + validSectorsPerCluster := []int{1, 2, 4, 8, 16, 32, 64, 128} + validMediaTypes := []int{0xF0, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE} + + if offset < FATMagicOffset { + return nil, fmt.Errorf("fat offset too small") + } + realOffset := offset - FATMagicOffset + + fatData := data[realOffset:] + + fields := []StructField{ + {"opcode1", "u8"}, {"opcode2", "u8"}, {"opcode3", "u8"}, + {"oem_name", "u64"}, {"bytes_per_sector", "u16"}, {"sectors_per_cluster", "u8"}, + {"reserved_sectors", "u16"}, {"fat_count", "u8"}, {"root_entries_count_16", "u16"}, + {"total_sectors_16", "u16"}, {"media_type", "u8"}, {"fat_size_16", "u16"}, + {"sectors_per_track", "u16"}, {"number_of_heads", "u16"}, + {"hidden_sectors", "u32"}, {"total_sectors_32", "u32"}, + } + + hdr, err := Parse(fatData, fields, "little") + if err != nil { + return nil, fmt.Errorf("fat header parse failed") + } + + if !containsInt(validOpcode1, hdr["opcode1"]) { + return nil, fmt.Errorf("fat invalid opcode1") + } + if !containsInt(validSectorSizes, hdr["bytes_per_sector"]) { + return nil, fmt.Errorf("fat invalid sector size") + } + if !containsInt(validSectorsPerCluster, hdr["sectors_per_cluster"]) { + return nil, fmt.Errorf("fat invalid sectors per cluster") + } + if hdr["reserved_sectors"] <= 0 { + return nil, fmt.Errorf("fat invalid reserved sectors") + } + if hdr["fat_count"] != expectedFATCount { + return nil, fmt.Errorf("fat invalid fat count") + } + if !containsInt(validMediaTypes, hdr["media_type"]) { + return nil, fmt.Errorf("fat invalid media type") + } + + isFAT32 := hdr["fat_size_16"] == 0 + + totalSize := 0 + if hdr["total_sectors_16"] != 0 { + totalSize = hdr["total_sectors_16"] * hdr["bytes_per_sector"] + } else { + totalSize = hdr["total_sectors_32"] * hdr["bytes_per_sector"] + } + if totalSize <= 0 { + return nil, fmt.Errorf("fat total size is zero") + } + + fatTypeDesc := "FAT12/16" + if isFAT32 { + fatTypeDesc = "FAT32" + } + + return &SignatureResult{ + Offset: realOffset, + Size: totalSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf("%s, type: %s, total size: %d bytes", + FATDescription, fatTypeDesc, totalSize), + }, nil +} + +// --------------------------------------------------------------------------- +// NTFS +// --------------------------------------------------------------------------- + +const NTFSDescription = "NTFS partition" + +func ntfs_magic() [][]byte { + return [][]byte{ + {0xEB, 0x52, 0x90, 0x4E, 0x54, 0x46, 0x53, 0x20, 0x20, 0x20, 0x20}, + } +} + +func ntfs_parser(data []byte, offset int) (*SignatureResult, error) { + ntfsFields := []StructField{ + {"opcodes", "u24"}, {"magic", "u64"}, + {"bytes_per_sector", "u16"}, {"sectors_per_cluster", "u8"}, + {"unused1", "u16"}, {"unused2", "u24"}, {"unused3", "u16"}, + {"media_type", "u8"}, {"unused4", "u16"}, + {"sectors_per_track", "u16"}, {"head_count", "u16"}, + {"hidden_sector_count", "u32"}, {"unused5", "u32"}, + {"unknown", "u32"}, {"sector_count", "u64"}, + } + + ntfsData := data[offset:] + hdr, err := Parse(ntfsData, ntfsFields, "little") + if err != nil { + return nil, fmt.Errorf("ntfs header parse failed") + } + + if hdr["unused1"] != 0 || hdr["unused2"] != 0 || hdr["unused3"] != 0 || + hdr["unused4"] != 0 || hdr["unused5"] != 0 { + return nil, fmt.Errorf("ntfs unused fields are not zero") + } + + sectorSize := hdr["bytes_per_sector"] + sectorCount := hdr["sector_count"] + // The reported sector count does not include the NTFS boot sector itself + totalSize := sectorSize * (sectorCount + 1) + + if totalSize <= sectorSize { + return nil, fmt.Errorf("ntfs total size too small") + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf("%s, number of sectors: %d, bytes per sector: %d, total size: %d bytes", + NTFSDescription, sectorCount, sectorSize, totalSize), + }, nil +} + +// --------------------------------------------------------------------------- +// APFS +// --------------------------------------------------------------------------- + +const APFSDescription = "APple File System" +const APFSMagicOffset = 0x20 + +func apfs_magic() [][]byte { + return [][]byte{ + []byte("NXSB"), + } +} + +func apfs_parser(data []byte, offset int) (*SignatureResult, error) { + const mbrBlockSize = 512 + const maxFSCount = 100 + const fsCountBlockSize = 512 + + if offset < APFSMagicOffset { + return nil, fmt.Errorf("apfs offset too small") + } + realOffset := offset - APFSMagicOffset + available := len(data) - realOffset + + apfsData := data[realOffset:] + apfsStructStart := APFSMagicOffset + + fields := []StructField{ + {"magic", "u32"}, {"block_size", "u32"}, {"block_count", "u64"}, + {"nx_features", "u64"}, {"nx_ro_compat_features", "u64"}, {"nx_incompat_features", "u64"}, + {"nx_uuid_p1", "u64"}, {"nx_uuid_p2", "u64"}, + {"nx_next_oid", "u64"}, {"nx_next_xid", "u64"}, + {"nx_xp_desc_blocks", "u32"}, {"nx_xp_data_blocks", "u32"}, + {"nx_xp_desc_base", "u64"}, {"nx_xp_data_base", "u64"}, + {"nx_xp_desc_next", "u32"}, {"nx_xp_data_next", "u32"}, + {"nx_xp_desc_index", "u32"}, {"nx_xp_desc_len", "u32"}, + {"nx_xp_data_index", "u32"}, {"nx_xp_data_len", "u32"}, + {"nx_spaceman_oid", "u64"}, {"nx_omap_oid", "u64"}, {"nx_reaper_oid", "u64"}, + {"nx_xp_test_type", "u32"}, {"nx_xp_max_file_systems", "u32"}, + } + + structEnd := apfsStructStart + StructSize(fields) + if structEnd > len(apfsData) { + return nil, fmt.Errorf("apfs not enough data for header") + } + + hdr, err := Parse(apfsData[apfsStructStart:structEnd], fields, "little") + if err != nil { + return nil, fmt.Errorf("apfs header parse failed") + } + + if hdr["block_size"] == 0 || hdr["block_count"] == 0 { + return nil, fmt.Errorf("apfs zero block size or count") + } + + allowedFeatureFlags := []int{0, 1, 2, 3} + allowedIncompatFlags := []int{0, 1, 2, 3, 0x100, 0x101, 0x102, 0x103} + allowedROCompatFlags := []int{0} + + if !containsInt(allowedFeatureFlags, hdr["nx_features"]) || + !containsInt(allowedROCompatFlags, hdr["nx_ro_compat_features"]) || + !containsInt(allowedIncompatFlags, hdr["nx_incompat_features"]) { + return nil, fmt.Errorf("apfs invalid feature flags") + } + + if hdr["nx_xp_test_type"] != 0 { + return nil, fmt.Errorf("apfs test_type must be null") + } + + fsCount := int(math.Ceil(float64(hdr["nx_xp_max_file_systems"]) / float64(fsCountBlockSize))) + if fsCount <= 0 || fsCount > maxFSCount { + return nil, fmt.Errorf("apfs invalid filesystem count") + } + + totalSize := hdr["block_count"] * hdr["block_size"] + truncatedMsg := "" + + if totalSize > available { + truncatedSize := totalSize - available + if truncatedSize == mbrBlockSize { + totalSize -= truncatedSize + truncatedMsg = fmt.Sprintf(" (truncated by %d bytes)", truncatedSize) + } + } + + return &SignatureResult{ + Offset: realOffset, + Size: totalSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf("%s, block size: %d bytes, block count: %d, total size: %d bytes%s", + APFSDescription, hdr["block_size"], hdr["block_count"], totalSize, truncatedMsg), + }, nil +} + +// --------------------------------------------------------------------------- +// BTRFS +// --------------------------------------------------------------------------- + +const BTRFSDescription = "BTRFS file system" +const BTRFSMagicOffset = 0x10040 + +func btrfs_magic() [][]byte { + return [][]byte{ + []byte("_BHRfS_M"), + } +} + +func btrfs_parser(data []byte, offset int) (*SignatureResult, error) { + const superblockOffset = 0x10000 + const superblockEnd = superblockOffset + 0x1000 + const crcStart = 0x20 + + if offset < BTRFSMagicOffset { + return nil, fmt.Errorf("btrfs offset too small") + } + realOffset := offset - BTRFSMagicOffset + + btrfsData := data[realOffset:] + + if len(btrfsData) < superblockEnd { + return nil, fmt.Errorf("btrfs not enough data") + } + + fields := []StructField{ + {"header_checksum", "u32"}, {"unused1", "u32"}, {"unused2", "u64"}, + {"unused3", "u64"}, {"unused4", "u64"}, + {"uuid_p1", "u64"}, {"uuid_p2", "u64"}, + {"block_phys_addr", "u64"}, {"flags", "u64"}, + {"magic", "u64"}, {"generation", "u64"}, + {"root_tree_address", "u64"}, {"chunk_tree_address", "u64"}, + {"log_tree_address", "u64"}, {"log_root_transid", "u64"}, + {"total_bytes", "u64"}, {"bytes_used", "u64"}, + {"root_dir_objid", "u64"}, {"num_devices", "u64"}, + {"sector_size", "u32"}, {"node_size", "u32"}, + {"leaf_size", "u32"}, {"stripe_size", "u32"}, + } + + headerData := btrfsData[superblockOffset:superblockEnd] + hdr, err := Parse(headerData, fields, "little") + if err != nil { + return nil, fmt.Errorf("btrfs header parse failed") + } + + // Validate CRC32C + calcCRC := crc32c(headerData[crcStart:]) + if hdr["header_checksum"] != int(calcCRC) { + return nil, fmt.Errorf("btrfs crc mismatch") + } + + return &SignatureResult{ + Offset: realOffset, + Size: hdr["total_bytes"], + Confidence: ConfidenceMedium, + Description: fmt.Sprintf( + "%s, node size: %d, sector size: %d, leaf size: %d, stripe size: %d, bytes used: %d, total size: %d bytes", + BTRFSDescription, hdr["node_size"], hdr["sector_size"], + hdr["leaf_size"], hdr["stripe_size"], hdr["bytes_used"], hdr["total_bytes"]), + }, nil +} + +// crc32c computes a CRC-32C (Castagnoli) checksum. +func crc32c(data []byte) uint32 { + // CRC-32C polynomial: 0x1EDC6F41 + const poly = 0x82F63B78 // reversed representation + crc := uint32(0xFFFFFFFF) + for _, b := range data { + crc ^= uint32(b) + for i := 0; i < 8; i++ { + if crc&1 != 0 { + crc = (crc >> 1) ^ poly + } else { + crc >>= 1 + } + } + } + return crc ^ 0xFFFFFFFF +} + +// --------------------------------------------------------------------------- +// LogFS +// --------------------------------------------------------------------------- + +const LogFSDescription = "LogFS file system" +const LogFSMagicOffset = 0x18 + +func logfs_magic() [][]byte { + return [][]byte{ + {0x7A, 0x3A, 0x8E, 0x5C, 0xB9, 0xD5, 0xBF, 0x67}, + } +} + +func logfs_parser(data []byte, offset int) (*SignatureResult, error) { + if offset < LogFSMagicOffset { + return nil, fmt.Errorf("logfs offset too small") + } + realOffset := offset - LogFSMagicOffset + + logfsData := data[realOffset:] + + fields := []StructField{ + {"magic", "u64"}, {"crc32", "u32"}, + {"ifile_levels", "u8"}, {"iblock_levels", "u8"}, + {"data_levels", "u8"}, {"segment_shift", "u8"}, + {"block_shift", "u8"}, {"write_shift", "u8"}, + {"pad0", "u32"}, {"pad1", "u16"}, + {"filesystem_size", "u64"}, {"segment_size", "u32"}, + {"bad_seg_reserved", "u32"}, + {"feature_incompat", "u64"}, {"feature_ro_compat", "u64"}, + {"feature_compat", "u64"}, {"feature_flags", "u64"}, + {"root_reserve", "u64"}, {"speed_reserve", "u64"}, + } + + if LogFSMagicOffset >= len(logfsData) { + return nil, fmt.Errorf("logfs not enough data") + } + + hdr, err := Parse(logfsData[LogFSMagicOffset:], fields, "big") + if err != nil { + return nil, fmt.Errorf("logfs superblock parse failed") + } + + if hdr["pad0"] != 0 || hdr["pad1"] != 0 { + return nil, fmt.Errorf("logfs padding fields not zero") + } + + totalSize := hdr["filesystem_size"] + + return &SignatureResult{ + Offset: realOffset, + Size: totalSize, + Confidence: ConfidenceMedium, + Description: fmt.Sprintf("%s, total size: %d bytes", + LogFSDescription, totalSize), + }, nil +} + +// --------------------------------------------------------------------------- +// UBI +// --------------------------------------------------------------------------- + +const UBIImageDescription = "UBI image" + +func ubi_magic() [][]byte { + return [][]byte{ + {0x55, 0x42, 0x49, 0x23, 0x01}, // "UBI#" + version 1 + } +} + +func ubi_parser(data []byte, offset int) (*SignatureResult, error) { + ubiECFields := []StructField{ + {"magic", "u32"}, {"version", "u8"}, {"padding1", "u24"}, + {"ec", "u64"}, {"volume_id_header_offset", "u32"}, {"data_offset", "u32"}, + {"image_sequence_number", "u32"}, + {"padding2", "u64"}, {"padding3", "u64"}, {"padding4", "u64"}, {"padding5", "u64"}, + {"header_crc", "u32"}, + } + + ecHeaderSize := StructSize(ubiECFields) + crcDataSize := ecHeaderSize - 4 // minus the header_crc field + + ubiData := data[offset:] + ecHdr, err := Parse(ubiData, ubiECFields, "big") + if err != nil { + return nil, fmt.Errorf("ubi ec header parse failed") + } + + if ecHdr["data_offset"] < ecHeaderSize || ecHdr["volume_id_header_offset"] < ecHeaderSize { + return nil, fmt.Errorf("ubi invalid offsets") + } + + // Validate header CRC + if crcDataSize > len(ubiData) { + return nil, fmt.Errorf("ubi not enough data for crc") + } + if ubiCRC(ubiData[:crcDataSize]) != ecHdr["header_crc"] { + return nil, fmt.Errorf("ubi ec header crc mismatch") + } + + dataOffset := offset + ecHdr["data_offset"] + volumeOffset := offset + ecHdr["volume_id_header_offset"] + if dataOffset >= len(data) || volumeOffset >= len(data) { + return nil, fmt.Errorf("ubi offsets out of bounds") + } + + // Get image size by scanning volume headers + imageSize, err := getUBIImageSize(data[offset:]) + if err != nil { + return nil, err + } + + return &SignatureResult{ + Offset: offset, + Size: imageSize, + Confidence: ConfidenceHigh, + Description: fmt.Sprintf("%s, version: %d, image size: %d bytes", + UBIImageDescription, ecHdr["version"], imageSize), + }, nil +} + +func getUBIImageSize(ubiData []byte) (int, error) { + volMagic := []byte{0x55, 0x42, 0x49, 0x21, 0x01} // "UBI!" + version 1 + + ubiVolFields := []StructField{ + {"magic", "u32"}, {"version", "u8"}, {"volume_type", "u8"}, + {"copy_flag", "u8"}, {"compat_type", "u8"}, + {"volume_id", "u32"}, {"logical_erase_block_number", "u32"}, + {"padding1", "u32"}, {"data_size", "u32"}, + {"used_erase_block_count", "u32"}, {"data_padding_size", "u32"}, + {"data_crc", "u32"}, {"padding2", "u32"}, + {"sequence_number", "u64"}, + {"padding3", "u64"}, {"padding4", "u32"}, {"header_crc", "u32"}, + } + + volHeaderSize := StructSize(ubiVolFields) + volCRCDataSize := volHeaderSize - 4 + + blockCount := 0 + previousVolumeOffset := 0 + possibleLEBSizes := make(map[int]int) + + // Scan for volume headers + for i := 0; i <= len(ubiData)-len(volMagic); { + idx := bytes.Index(ubiData[i:], volMagic) + if idx < 0 { + break + } + thisVolumeOffset := i + idx + + if thisVolumeOffset+volHeaderSize > len(ubiData) { + break + } + + volHdr, err := Parse(ubiData[thisVolumeOffset:], ubiVolFields, "big") + if err == nil && + volHdr["padding1"] == 0 && volHdr["padding2"] == 0 && + volHdr["padding3"] == 0 && volHdr["padding4"] == 0 { + // Validate CRC + if thisVolumeOffset+volCRCDataSize <= len(ubiData) { + if ubiCRC(ubiData[thisVolumeOffset:thisVolumeOffset+volCRCDataSize]) == volHdr["header_crc"] { + blockCount++ + if previousVolumeOffset != 0 { + lebSize := thisVolumeOffset - previousVolumeOffset + possibleLEBSizes[lebSize]++ + } + previousVolumeOffset = thisVolumeOffset + } + } + } + i = thisVolumeOffset + 1 + } + + // Pick the most common LEB size + lebSize := 0 + bestCount := 0 + for size, count := range possibleLEBSizes { + if count > bestCount { + lebSize = size + bestCount = count + } + } + + if lebSize != 0 && blockCount != 0 { + return blockCount * lebSize, nil + } + return 0, fmt.Errorf("ubi could not determine image size") +} + +// --------------------------------------------------------------------------- +// UBIFS +// --------------------------------------------------------------------------- + +const UBIFSDescription = "UBIFS image" + +func ubifs_magic() [][]byte { + return [][]byte{ + {0x31, 0x18, 0x10, 0x06}, + } +} + +func ubifs_parser(data []byte, offset int) (*SignatureResult, error) { + const maxGroupType = 2 + const crcStartOffset = 8 + const superblockNodeType = 6 + const superblockStructExtraSize = 3968 + + ubiSBFields := []StructField{ + {"magic", "u32"}, {"header_crc", "u32"}, + {"sequence_number", "u64"}, {"node_len", "u32"}, + {"node_type", "u8"}, {"group_type", "u8"}, + {"padding1", "u32"}, {"key_hash", "u8"}, {"key_format", "u8"}, + {"flags", "u32"}, {"min_io_size", "u32"}, + {"leb_size", "u32"}, {"leb_count", "u32"}, {"max_leb_count", "u32"}, + {"max_bud_bytes", "u64"}, {"log_lebs", "u32"}, {"lpt_lebs", "u32"}, + {"orph_lebs", "u32"}, {"jhead_count", "u32"}, {"fanout", "u32"}, + {"lsave_count", "u32"}, {"fmt_version", "u32"}, + {"default_compression", "u16"}, {"padding2", "u16"}, + {"rp_uid", "u32"}, {"rp_gid", "u32"}, {"rp_size", "u64"}, + {"time_gran", "u32"}, {"uuid_p1", "u64"}, {"uuid_p2", "u64"}, + {"ro_compat_version", "u32"}, + } + + sbStructSize := StructSize(ubiSBFields) + superblockStructExtraSize + + ubisData := data[offset:] + hdr, err := Parse(ubisData, ubiSBFields, "little") + if err != nil { + return nil, fmt.Errorf("ubifs superblock parse failed") + } + + if hdr["padding1"] != 0 || hdr["padding2"] != 0 { + return nil, fmt.Errorf("ubifs padding fields not zero") + } + if hdr["node_type"] != superblockNodeType { + return nil, fmt.Errorf("ubifs not a superblock node") + } + if hdr["group_type"] > maxGroupType { + return nil, fmt.Errorf("ubifs invalid group type") + } + + // Validate header CRC + if crcStartOffset+sbStructSize-crcStartOffset > len(ubisData) { + return nil, fmt.Errorf("ubifs not enough data for crc") + } + crcData := ubisData[crcStartOffset:sbStructSize] + if ubiCRC(crcData) != hdr["header_crc"] { + return nil, fmt.Errorf("ubifs header crc mismatch") + } + + imageSize := hdr["leb_count"] * hdr["leb_size"] + + return &SignatureResult{ + Offset: offset, + Size: imageSize, + Confidence: ConfidenceHigh, + Description: fmt.Sprintf("%s, total size: %d bytes", + UBIFSDescription, imageSize), + }, nil +} + +// ubiCRC computes a UBI checksum: NOT(CRC32(data)) & 0xFFFFFFFF +func ubiCRC(data []byte) int { + const ubiCRCInit = uint32(0xFFFFFFFF) + return int((^CRC32(data)) & ubiCRCInit) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func containsInt(slice []int, val int) bool { + for _, s := range slice { + if s == val { + return true + } + } + return false +} diff --git a/internal/leviathan/binsnoop/sig_firmware.go b/internal/leviathan/binsnoop/sig_firmware.go new file mode 100644 index 0000000..e214ed5 --- /dev/null +++ b/internal/leviathan/binsnoop/sig_firmware.go @@ -0,0 +1,1333 @@ +package binsnoop + +import ( + "encoding/hex" + "fmt" +) + +// ============================================================================ +// uImage +// ============================================================================ + +const uimageDescription = "uImage firmware image" + +func uimageMagic() [][]byte { + return [][]byte{ + {0x27, 0x05, 0x19, 0x56}, + []byte("OKLI"), + } +} + +func uimageParser(data []byte, offset int) (*SignatureResult, error) { + if offset+64 > len(data) { + return nil, fmt.Errorf("insufficient data for uImage header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_crc", "u32"}, {"timestamp", "u32"}, + {"data_size", "u32"}, {"load_address", "u32"}, {"entry_point", "u32"}, + {"data_crc", "u32"}, {"os_type", "u8"}, {"cpu_type", "u8"}, + {"image_type", "u8"}, {"compression_type", "u8"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := 64 + dataSize := parsed["data_size"] + if dataSize == 0 { + return nil, fmt.Errorf("uImage data size is 0") + } + totalSize := headerSize + dataSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("uImage extends beyond EOF") + } + + // Validate header CRC: zero out the header_crc field, calc CRC32 over 64 bytes + headerCopy := make([]byte, headerSize) + copy(headerCopy, hdr[:headerSize]) + headerCopy[4] = 0 + headerCopy[5] = 0 + headerCopy[6] = 0 + headerCopy[7] = 0 + calcCRC := CRC32(headerCopy) + headerCRCValid := calcCRC == uint32(parsed["header_crc"]) + + osMap := map[int]string{0: "OpenBSD", 1: "NetBSD", 2: "FreeBSD", 3: "4.4BSD", 4: "Linux", 5: "SVR4", 6: "Esix", 7: "Solaris", 8: "Irix", 9: "SCO", 10: "Dell", 11: "NCR", 12: "LynxOS", 13: "VxWorks", 14: "pSOS", 15: "QNX", 16: "U-Boot", 17: "RTEMS", 18: "ARTOS", 19: "Unity OS", 20: "INTEGRITY", 21: "OSE", 22: "Plan 9", 23: "OpenRTOS", 24: "ARM Trusted Firmware", 25: "Trusted Execution Environment", 26: "RISC-V OpenSBI", 27: "EFI Firmware"} + cpuMap := map[int]string{0: "Invalid", 1: "Alpha", 2: "ARM", 3: "Intel x86", 4: "IA64", 5: "MIPS", 6: "MIPS 64-bit", 7: "PowerPC", 8: "IBM S390", 9: "SuperH", 10: "Sparc", 11: "Sparc 64-bit", 12: "M68K", 13: "Nios-32", 14: "MicroBlaze", 15: "Nios-II", 16: "Blackfin", 17: "AVR32", 18: "ST200", 19: "Sandbox", 20: "NDS32", 21: "OpenRISC 1000", 22: "ARM64", 23: "ARC", 24: "x86_64", 25: "Xtensa", 26: "RISC-V"} + compMap := map[int]string{0: "none", 1: "gzip", 2: "bzip2", 3: "lzma", 4: "lzo", 5: "lz4", 6: "zstd"} + imgMap := map[int]string{0: "Invalid", 1: "Standalone", 2: "Kernel", 3: "RAMDisk", 4: "Multi-File", 5: "Firmware", 6: "Script", 7: "Filesystem", 8: "Flat DT", 9: "Kwb", 10: "IMX", 11: "AIS", 12: "Kernel (no load)", 13: "PBL", 14: "MXS", 15: "GP Header", 16: "ATMEL", 17: "IMX (Silicon Labs)", 18: "TEE", 19: "FW IVT", 20: "PMC", 21: "STM32", 22: "Renesas SPKG", 23: "ST SunXI", 24: "FDT Legacy"} + + osType := lookupMap(osMap, parsed["os_type"], "Unknown") + cpuType := lookupMap(cpuMap, parsed["cpu_type"], "Unknown") + compType := lookupMap(compMap, parsed["compression_type"], "Unknown") + imageType := lookupMap(imgMap, parsed["image_type"], "Unknown") + name := GetCString(hdr[32:64]) + + confidence := ConfidenceHigh + desc := fmt.Sprintf("%s, header size: %d bytes, data size: %d bytes, compression: %s, CPU: %s, OS: %s, image type: %s, load address: 0x%X, entry point: 0x%X, creation time: %s, image name: \"%s\"", + uimageDescription, headerSize, dataSize, compType, cpuType, osType, imageType, + parsed["load_address"], parsed["entry_point"], + EpochToString(uint32(parsed["timestamp"])), name) + + if !headerCRCValid { + if offset == 0 { + confidence = ConfidenceMedium + } else { + confidence = ConfidenceLow + } + desc += ", invalid checksum" + } + + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: desc, + Confidence: confidence, + ExtractionDeclined: !headerCRCValid || dataSize == 0, + }, nil +} + +// ============================================================================ +// U-Boot version string +// ============================================================================ + +const ubootDescription = "U-Boot version string" + +func ubootMagic() [][]byte { + return [][]byte{[]byte("U-Boot ")} +} + +func ubootParser(data []byte, offset int) (*SignatureResult, error) { + numOff := offset + 7 + if numOff >= len(data) { + return nil, fmt.Errorf("insufficient data") + } + if !IsASCIINumber(data[numOff]) { + return nil, fmt.Errorf("expected version number") + } + ver := GetCString(data[numOff:]) + if len(ver) == 0 { + return nil, fmt.Errorf("empty version string") + } + if len(ver) > 100 { + ver = ver[:100] + } + return &SignatureResult{ + Offset: offset, + Size: len(ver), + Description: fmt.Sprintf("%s: %s", ubootDescription, ver), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// TRX +// ============================================================================ + +const trxDescription = "TRX firmware image" + +func trxMagic() [][]byte { + return [][]byte{[]byte("HDR0")} +} + +func trxParser(data []byte, offset int) (*SignatureResult, error) { + if offset+28 > len(data) { + return nil, fmt.Errorf("insufficient data for TRX header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"total_size", "u32"}, {"header_crc", "u32"}, + {"version", "u16"}, {"flags", "u16"}, + {"partition1", "u32"}, {"partition2", "u32"}, {"partition3", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + totalSize := parsed["total_size"] + if totalSize <= 28 || offset+totalSize > len(data) { + return nil, fmt.Errorf("invalid TRX size") + } + // CRC32 validation (over data after the CRC field) + if offset+12 < len(data) { + calcCRC := CRC32(hdr[12:totalSize]) + if int(calcCRC) != parsed["header_crc"] { + return nil, fmt.Errorf("TRX CRC mismatch") + } + } + partCount := 0 + for _, p := range []string{"partition1", "partition2", "partition3"} { + if parsed[p] > 0 { + partCount++ + } + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, version %d, partition count: %d, header size: 28 bytes, total size: %d bytes", trxDescription, parsed["version"], partCount, totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// TP-Link firmware header +// ============================================================================ + +const tplinkDescription = "TP-Link firmware header" + +func tplinkMagic() [][]byte { + return [][]byte{ + {0x01, 0x00, 0x00, 0x00, 0x54, 0x50, 0x2D, 0x4C, 0x49, 0x4E, 0x4B, 0x20, 0x54, 0x65, 0x63, 0x68, 0x6E, 0x6F, 0x6C, 0x6F, 0x67, 0x69, 0x65, 0x73, 0x00, 0x00, 0x00, 0x00, 0x76, 0x65, 0x72, 0x2E, 0x20, 0x31, 0x2E, 0x30}, + } +} + +func tplinkParser(data []byte, offset int) (*SignatureResult, error) { + if offset+0x200 > len(data) { + return nil, fmt.Errorf("insufficient data for TP-Link header") + } + hdr := data[offset:] + fields := []StructField{ + {"version", "u32"}, {"fw_version", "u32"}, {"hw_id", "u32"}, {"hw_rev", "u32"}, + {"unk1", "u32"}, {"kernel_data_offset", "u32"}, {"kernel_load_address", "u32"}, + {"kernel_entry_point", "u32"}, {"kernel_data_length", "u32"}, + } + parsed, err := Parse(hdr[0x40:], fields, "big") + if err != nil { + return nil, err + } + headerSize := 0x200 + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, kernel load address: 0x%X, kernel entry point: 0x%X, header size: %d bytes", tplinkDescription, parsed["kernel_load_address"], parsed["kernel_entry_point"], headerSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// TP-Link RTOS firmware +// ============================================================================ + +const tplinkRTOSDescription = "TP-Link RTOS firmware" + +func tplinkRTOSMagic() [][]byte { + return [][]byte{{0x00, 0x14, 0x2F, 0xC0}} +} + +func tplinkRTOSParser(data []byte, offset int) (*SignatureResult, error) { + if offset+0x200 > len(data) { + return nil, fmt.Errorf("insufficient data for TP-Link RTOS header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"total_size", "u32"}, + {"model_number", "u32"}, {"hw_rev_major", "u32"}, {"hw_rev_minor", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + return &SignatureResult{ + Offset: offset, + Size: parsed["header_size"], + Description: fmt.Sprintf("%s, model number: %X, hardware version: %X.%X, header size: %d bytes, total size: %d bytes", tplinkRTOSDescription, parsed["model_number"], parsed["hw_rev_major"], parsed["hw_rev_minor"], parsed["header_size"], parsed["total_size"]), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// DLOB firmware header +// ============================================================================ + +const dlobDescription = "DLOB firmware header" + +func dlobMagic() [][]byte { + return [][]byte{{0x5e, 0xa3, 0xa4, 0x17}} +} + +func dlobParser(data []byte, offset int) (*SignatureResult, error) { + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data for DLOB header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + availableData := len(data) - offset + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + if availableData < headerSize+dataSize { + return nil, fmt.Errorf("DLOB size exceeds available data") + } + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, header size: %d bytes, data size: %d", dlobDescription, headerSize, dataSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// Arcadyan obfuscated LZMA +// ============================================================================ + +const arcadyanDescription = "Arcadyan obfuscated LZMA" + +func arcadyanMagic() [][]byte { + return [][]byte{{0x00, 0xD5, 0x08, 0x00}} +} + +func arcadyanParser(data []byte, offset int) (*SignatureResult, error) { + const magicOffset = 0x68 + if offset < magicOffset { + return nil, fmt.Errorf("arcadyan magic offset too small") + } + startOffset := offset - magicOffset + return &SignatureResult{ + Offset: startOffset, + Description: arcadyanDescription, + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// CHK firmware header (Netgear) +// ============================================================================ + +const chkDescription = "CHK firmware header" + +func chkMagic() [][]byte { + return [][]byte{{0x2A, 0x23, 0x24, 0x5E}} +} + +func chkParser(data []byte, offset int) (*SignatureResult, error) { + if offset+40 > len(data) { + return nil, fmt.Errorf("insufficient data for CHK header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"reserved", "u32"}, + {"kernel_checksum", "u32"}, {"rootfs_checksum", "u32"}, + {"kernel_size", "u32"}, {"rootfs_size", "u32"}, + {"image_checksum", "u32"}, {"header_checksum", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + kernelSize := parsed["kernel_size"] + rootfsSize := parsed["rootfs_size"] + totalSize := headerSize + kernelSize + rootfsSize + availableData := len(data) - offset + if availableData < totalSize || totalSize <= headerSize { + return nil, fmt.Errorf("invalid CHK sizes") + } + boardID := GetCString(hdr[40:]) + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, board ID: %s, header size: %d bytes, data size: %d bytes", chkDescription, boardID, headerSize, kernelSize+rootfsSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// SEAMA firmware header +// ============================================================================ + +const seamaDescription = "SEAMA firmware header" + +func seamaMagic() [][]byte { + return [][]byte{ + {0x5E, 0xA3, 0xA4, 0x17, 0x00, 0x00}, + {0x17, 0xA4, 0xA3, 0x5E, 0x00, 0x00}, + } +} + +func seamaParser(data []byte, offset int) (*SignatureResult, error) { + if offset+12 > len(data) { + return nil, fmt.Errorf("insufficient data for SEAMA header") + } + hdr := data[offset:] + endianness := "big" + if hdr[0] == 0x17 { + endianness = "little" + } + fields := []StructField{ + {"magic", "u32"}, {"meta_size", "u16"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr, fields, endianness) + if err != nil { + return nil, err + } + headerSize := 12 + parsed["meta_size"] + dataSize := parsed["data_size"] + totalSize := headerSize + dataSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("SEAMA size exceeds data") + } + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, header size: %d bytes, data size: %d bytes", seamaDescription, headerSize, dataSize), + Confidence: ConfidenceLow, + }, nil +} + +// ============================================================================ +// CFE bootloader +// ============================================================================ + +const cfeDescription = "CFE bootloader" + +func cfeMagic() [][]byte { + return [][]byte{[]byte("CFE1CFE1")} +} + +func cfeParser(_ []byte, offset int) (*SignatureResult, error) { + const cfeMagicOffset = 28 + if offset < cfeMagicOffset { + return nil, fmt.Errorf("CFE magic offset too small") + } + confidence := ConfidenceLow + startOffset := offset - cfeMagicOffset + if startOffset == 0 { + confidence = ConfidenceMedium + } + return &SignatureResult{ + Offset: startOffset, + Description: cfeDescription, + Confidence: confidence, + }, nil +} + +// ============================================================================ +// JBOOT ARM firmware +// ============================================================================ + +const jbootARMDescription = "JBOOT firmware header" + +func jbootARMMagic() [][]byte { + return [][]byte{ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x42, 0x48}, + } +} + +func jbootARMParser(data []byte, offset int) (*SignatureResult, error) { + const magicOff = 48 + if offset < magicOff { + return nil, fmt.Errorf("JBOOT ARM magic offset too small") + } + headerStart := offset - magicOff + if headerStart+80 > len(data) { + return nil, fmt.Errorf("insufficient data for JBOOT ARM header") + } + hdr := data[headerStart:] + fields := []StructField{ + {"rom_id_size", "u32"}, {"rom_id", "u32"}, {"product_id", "u32"}, + {"func_id", "u32"}, {"erase_offset", "u32"}, {"erase_size", "u32"}, + {"data_offset", "u32"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr[20:], fields, "big") + if err != nil { + return nil, err + } + headerSize := 80 + return &SignatureResult{ + Offset: headerStart, + Size: headerSize, + Description: fmt.Sprintf("%s, header size: %d bytes, ROM ID: %d, erase offset: 0x%X, erase size: 0x%X, data flash offset: 0x%X, data size: 0x%X", jbootARMDescription, headerSize, parsed["rom_id"], parsed["erase_offset"], parsed["erase_size"], parsed["data_offset"], parsed["data_size"]), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// JBOOT STAG header +// ============================================================================ + +const jbootSTAGDescription = "JBOOT STAG header" + +func jbootSTAGMagic() [][]byte { + return [][]byte{ + {0x04, 0x04, 0x24, 0x2B}, + {0xFF, 0x04, 0x24, 0x2B}, + } +} + +func jbootSTAGParser(data []byte, offset int) (*SignatureResult, error) { + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data for JBOOT STAG header") + } + hdr := data[offset:] + fields := []StructField{ + {"tag", "u32"}, {"header_size", "u16"}, {"flags", "u16"}, {"image_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + imageSize := parsed["image_size"] + if offset+headerSize+imageSize >= len(data) { + return nil, fmt.Errorf("JBOOT STAG size exceeds data") + } + imageType := "factory image" + if parsed["flags"]&0x01 != 0 { + imageType = "system upgrade image" + } + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, %s, header size: %d bytes, kernel data size: %d bytes", jbootSTAGDescription, imageType, headerSize, imageSize), + Confidence: ConfidenceLow, + }, nil +} + +// ============================================================================ +// JBOOT SCH2 header +// ============================================================================ + +const jbootSCH2Description = "JBOOT SCH2 header" + +func jbootSCH2Magic() [][]byte { + return [][]byte{ + {0x24, 0x21, 0x00, 0x02}, + {0x24, 0x21, 0x01, 0x02}, + {0x24, 0x21, 0x02, 0x02}, + {0x24, 0x21, 0x03, 0x02}, + } +} + +func jbootSCH2Parser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for JBOOT SCH2 header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"kernel_size", "u32"}, + {"kernel_checksum", "u32"}, {"kernel_entry_point", "u32"}, + {"compression", "u8"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + compTypes := map[int]string{0: "none", 1: "gzip", 2: "bzip2", 3: "lzma"} + comp := lookupMap(compTypes, parsed["compression"], "unknown") + headerSize := parsed["header_size"] + kernelSize := parsed["kernel_size"] + totalSize := headerSize + kernelSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("JBOOT SCH2 extends beyond EOF") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, header size: %d bytes, kernel size: %d bytes, kernel compression: %s, kernel entry point: 0x%X", jbootSCH2Description, headerSize, kernelSize, comp, parsed["kernel_entry_point"]), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// PackImg firmware header +// ============================================================================ + +const packimgDescription = "PackImg firmware header" + +func packimgMagic() [][]byte { + return [][]byte{[]byte("--PaCkImGs--")} +} + +func packimgParser(data []byte, offset int) (*SignatureResult, error) { + if offset+24 > len(data) { + return nil, fmt.Errorf("insufficient data for PackImg header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic1", "u32"}, {"magic2", "u32"}, {"magic3", "u32"}, + {"data_size", "u32"}, {"header_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + if len(data)-offset < headerSize+dataSize { + return nil, fmt.Errorf("PackImg size exceeds data") + } + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, header size: %d bytes, data size: %d bytes", packimgDescription, headerSize, dataSize), + Confidence: ConfidenceLow, + }, nil +} + +// ============================================================================ +// BIN firmware header +// ============================================================================ + +const binhdrDescription = "BIN firmware header" + +func binhdrMagic() [][]byte { + return [][]byte{[]byte("U2ND")} +} + +func binhdrParser(data []byte, offset int) (*SignatureResult, error) { + const magicOff = 14 + if offset < magicOff { + return nil, fmt.Errorf("BIN header magic offset too small") + } + startOffset := offset - magicOff + if startOffset+24 > len(data) { + return nil, fmt.Errorf("insufficient data for BIN header") + } + hdr := data[startOffset:] + boardID := GetCString(hdr[:14]) + fields := []StructField{ + {"magic", "u32"}, {"hw_rev", "u16"}, {"fw_major", "u8"}, {"fw_minor", "u8"}, + } + parsed, err := Parse(hdr[14:], fields, "big") + if err != nil { + return nil, err + } + return &SignatureResult{ + Offset: startOffset, + Description: fmt.Sprintf("%s, board ID: %s, hardware revision: %d, firmware version: %d.%d", binhdrDescription, boardID, parsed["hw_rev"], parsed["fw_major"], parsed["fw_minor"]), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// Autel obfuscated firmware +// ============================================================================ + +const autelDescription = "Autel obfuscated firmware" + +func autelMagic() [][]byte { + return [][]byte{[]byte("ECC0101\x00")} +} + +func autelParser(data []byte, offset int) (*SignatureResult, error) { + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data for Autel header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u64"}, {"data_size", "u32"}, {"header_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + totalSize := headerSize + dataSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("Autel size exceeds data") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, header size: %d bytes, data size: %d, total size: %d", autelDescription, headerSize, dataSize, totalSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// D-Link TLV firmware +// ============================================================================ + +const dlinkTLVDescription = "D-Link TLV firmware" + +func dlinkTLVMagic() [][]byte { + return [][]byte{{0x64, 0x80, 0x19, 0x40}} +} + +func dlinkTLVParser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for D-Link TLV header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + totalSize := headerSize + dataSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("D-Link TLV size exceeds data") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, header size: %d bytes, data size: %d bytes", dlinkTLVDescription, headerSize, dataSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// DLKE encrypted firmware +// ============================================================================ + +const dlkeDescription = "DLK encrypted firmware" + +func dlkeMagic() [][]byte { + return [][]byte{ + []byte("DLK6E8202001"), + []byte("DLK6E6110002"), + } +} + +func dlkeParser(data []byte, offset int) (*SignatureResult, error) { + if offset+80 > len(data) { + return nil, fmt.Errorf("insufficient data for DLKE header") + } + // Parse first JBOOT-like header + hdr1 := data[offset:] + fields := []StructField{ + {"rom_id_size", "u32"}, {"rom_id", "u32"}, {"product_id", "u32"}, + {"func_id", "u32"}, {"erase_offset", "u32"}, {"erase_size", "u32"}, + {"data_offset", "u32"}, {"data_size", "u32"}, + } + parsed1, err := Parse(hdr1[20:], fields, "big") + if err != nil { + return nil, err + } + hdr1Size := 80 + sig1DataSize := parsed1["data_size"] + + nextOffset := offset + hdr1Size + sig1DataSize + if nextOffset+80 > len(data) { + return nil, fmt.Errorf("DLKE second header beyond EOF") + } + hdr2 := data[nextOffset:] + parsed2, err := Parse(hdr2[20:], fields, "big") + if err != nil { + return nil, err + } + hdr2Size := 80 + cryptDataSize := parsed2["data_size"] + + totalSize := hdr1Size + sig1DataSize + hdr2Size + cryptDataSize + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, signature size: %d bytes, encrypted data size: %d bytes", dlkeDescription, sig1DataSize, cryptDataSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// SHRS encrypted firmware +// ============================================================================ + +const shrsDescription = "SHRS encrypted firmware" + +func shrsMagic() [][]byte { + return [][]byte{[]byte("SHRS")} +} + +func shrsParser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for SHRS header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + totalSize := headerSize + dataSize + if offset+totalSize > len(data) { + return nil, fmt.Errorf("SHRS size exceeds data") + } + // Extract IV (16 bytes after the fixed header) + ivStr := "" + if headerSize >= 28 && offset+28 <= len(data) { + iv := hdr[12:28] + ivStr = hex.EncodeToString(iv) + } + confidence := ConfidenceLow + if offset == 0 { + confidence = ConfidenceMedium + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, header size: %d bytes, encrypted data size: %d bytes, IV: %s", shrsDescription, headerSize, dataSize, ivStr), + Confidence: confidence, + }, nil +} + +// ============================================================================ +// DKBS firmware header +// ============================================================================ + +const dkbsDescription = "DKBS firmware header" + +func dkbsMagic() [][]byte { + return [][]byte{[]byte("_dkbs_")} +} + +func dkbsParser(data []byte, offset int) (*SignatureResult, error) { + const magicOff = 7 + if offset < magicOff { + return nil, fmt.Errorf("DKBS magic offset too small") + } + startOffset := offset - magicOff + if startOffset+64 > len(data) { + return nil, fmt.Errorf("insufficient data for DKBS header") + } + hdr := data[startOffset:] + fields := []StructField{ + {"version", "u32"}, {"magic", "u32"}, {"padding", "u32"}, + {"header_size", "u32"}, {"data_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + dataSize := parsed["data_size"] + availableData := len(data) - startOffset + if availableData < headerSize+dataSize { + return nil, fmt.Errorf("DKBS size exceeds data") + } + confidence := ConfidenceMedium + if startOffset == 0 { + confidence = ConfidenceHigh + } + boardID := GetCString(hdr[24:]) + return &SignatureResult{ + Offset: startOffset, + Size: headerSize, + Description: fmt.Sprintf("%s, board ID: %s, header size: %d bytes, data size: %d", dkbsDescription, boardID, headerSize, dataSize), + Confidence: confidence, + }, nil +} + +// ============================================================================ +// DMS firmware image +// ============================================================================ + +const dmsDescription = "DMS firmware image" + +func dmsMagic() [][]byte { + return [][]byte{[]byte("0><1")} +} + +func dmsParser(data []byte, offset int) (*SignatureResult, error) { + const magicOff = 4 + const minSize = 0x100 + if offset < magicOff { + return nil, fmt.Errorf("DMS magic offset too small") + } + startOffset := offset - magicOff + if startOffset+minSize > len(data) { + return nil, fmt.Errorf("insufficient data for DMS header") + } + + // DMS images have byte-swapped u16 pairs + dmsData := make([]byte, minSize) + copy(dmsData, data[startOffset:startOffset+minSize]) + for i := 0; i+1 < len(dmsData); i += 2 { + dmsData[i], dmsData[i+1] = dmsData[i+1], dmsData[i] + } + + fields := []StructField{ + {"magic", "u32"}, {"unk1", "u32"}, {"image_size", "u32"}, + } + parsed, err := Parse(dmsData, fields, "big") + if err != nil { + return nil, err + } + imageSize := parsed["image_size"] + if imageSize <= 0 || startOffset+imageSize > len(data) { + return nil, fmt.Errorf("invalid DMS image size") + } + return &SignatureResult{ + Offset: startOffset, + Size: imageSize, + Description: fmt.Sprintf("%s, total size: %d bytes", dmsDescription, imageSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// Known encrypted firmware (encfw) +// ============================================================================ + +const encfwDescription = "Known encrypted firmware" + +func encfwKnownFirmware() map[string]string { + return map[string]string{ + "\xdf\x8c\x39\x0d": "D-Link DIR-822 rev C", + "\x35\x66\x6f\x68": "D-Link DAP-1665", + "\xf5\x2a\xa0\xb4": "D-Link DIR-842 rev C", + "\xe3\x13\x00\x5b": "D-Link DIR-850 rev A", + "\x0a\x14\xe4\x24": "D-Link DIR-850 rev B", + } +} + +func encfwMagic() [][]byte { + var magics [][]byte + for k := range encfwKnownFirmware() { + magics = append(magics, []byte(k)) + } + return magics +} + +func encfwParser(data []byte, offset int) (*SignatureResult, error) { + if offset+4 > len(data) { + return nil, fmt.Errorf("insufficient data for encfw") + } + key := string(data[offset : offset+4]) + known := encfwKnownFirmware() + model, ok := known[key] + if !ok { + return nil, fmt.Errorf("unknown encfw magic") + } + confidence := ConfidenceMedium + if offset != 0 { + confidence = ConfidenceLow + } + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, %s", encfwDescription, model), + Confidence: confidence, + }, nil +} + +// ============================================================================ +// D-Link encrpted_img +// ============================================================================ + +const encrptedImgDescription = "D-Link Encrpted Image" + +func encrptedImgMagic() [][]byte { + return [][]byte{[]byte("encrpted_img")} +} + +func encrptedImgParser(_ []byte, offset int) (*SignatureResult, error) { + confidence := ConfidenceMedium + if offset != 0 { + confidence = ConfidenceLow + } + return &SignatureResult{ + Offset: offset, + Description: encrptedImgDescription, + Confidence: confidence, + }, nil +} + +// ============================================================================ +// Android boot image +// ============================================================================ + +const androidBootimgDescription = "Android boot image" + +func androidBootimgMagic() [][]byte { + return [][]byte{[]byte("ANDROID!")} +} + +func androidBootimgParser(data []byte, offset int) (*SignatureResult, error) { + if offset+40 > len(data) { + return nil, fmt.Errorf("insufficient data for Android boot image header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u64"}, {"kernel_size", "u32"}, {"kernel_load_addr", "u32"}, + {"ramdisk_size", "u32"}, {"ramdisk_load_addr", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + confidence := ConfidenceLow + if offset == 0 { + confidence = ConfidenceMedium + } + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, kernel size: %d bytes, kernel load address: 0x%X, ramdisk size: %d bytes, ramdisk load address: 0x%X", androidBootimgDescription, parsed["kernel_size"], parsed["kernel_load_addr"], parsed["ramdisk_size"], parsed["ramdisk_load_addr"]), + Confidence: confidence, + }, nil +} + +// ============================================================================ +// Android sparse image +// ============================================================================ + +const androidSparseDescription = "Android sparse image" + +func androidSparseMagic() [][]byte { + return [][]byte{{0x3A, 0xFF, 0x26, 0xED}} +} + +func androidSparseParser(data []byte, offset int) (*SignatureResult, error) { + if offset+28 > len(data) { + return nil, fmt.Errorf("insufficient data for Android sparse header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"major_version", "u16"}, {"minor_version", "u16"}, + {"header_size", "u16"}, {"chunk_header_size", "u16"}, + {"block_size", "u32"}, {"total_blocks", "u32"}, + {"total_chunks", "u32"}, {"checksum", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, version %d.%d, header size: %d, block size: %d, chunk count: %d", androidSparseDescription, parsed["major_version"], parsed["minor_version"], headerSize, parsed["block_size"], parsed["total_chunks"]), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// Windows CE binary image +// ============================================================================ + +const winceDescription = "Windows CE binary image" + +func winceMagic() [][]byte { + return [][]byte{[]byte("B000FF\n")} +} + +func winceParser(data []byte, offset int) (*SignatureResult, error) { + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data for WinCE header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic1", "u32"}, {"magic2", "u32"}, + {"base_address", "u32"}, {"image_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + return &SignatureResult{ + Offset: offset, + Size: parsed["image_size"], + Description: fmt.Sprintf("%s, base address: 0x%X, image size: %d bytes", winceDescription, parsed["base_address"], parsed["image_size"]), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// D-Link MH01 firmware +// ============================================================================ + +const mh01Description = "D-Link MH01 firmware image" + +func mh01Magic() [][]byte { + return [][]byte{[]byte("MH01")} +} + +func mh01Parser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for MH01 header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"total_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + totalSize := parsed["total_size"] + if offset+totalSize > len(data) { + return nil, fmt.Errorf("MH01 size exceeds data") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, total size: %d bytes", mh01Description, totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// CSman DAT +// ============================================================================ + +const csmanDescription = "CSman DAT file" + +func csmanMagic() [][]byte { + return [][]byte{[]byte("SC"), []byte("CS")} +} + +func csmanParser(data []byte, offset int) (*SignatureResult, error) { + if offset+8 > len(data) { + return nil, fmt.Errorf("insufficient data for CSman header") + } + hdr := data[offset:] + endianness := "big" + if hdr[0] == 'C' && hdr[1] == 'S' { + endianness = "little" + } + fields := []StructField{ + {"magic", "u16"}, {"count", "u16"}, {"size", "u32"}, + } + parsed, err := Parse(hdr, fields, endianness) + if err != nil { + return nil, err + } + totalSize := parsed["size"] + if totalSize <= 0 || offset+totalSize > len(data) { + return nil, fmt.Errorf("invalid CSman size") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, total size: %d bytes", csmanDescription, totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// DXBC (DirectX shader bytecode) +// ============================================================================ + +const dxbcDescription = "DirectX shader bytecode" + +func dxbcMagic() [][]byte { + return [][]byte{[]byte("DXBC")} +} + +func dxbcParser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for DXBC header") + } + hdr := data[offset:] + // DXBC header: 4-byte magic, 16-byte hash, 4-byte version, 4-byte total_size, 4-byte chunk_count + fields := []StructField{ + {"magic", "u32"}, {"hash1", "u32"}, {"hash2", "u32"}, + {"hash3", "u32"}, {"hash4", "u32"}, + {"version", "u32"}, {"total_size", "u32"}, {"chunk_count", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + totalSize := parsed["total_size"] + if totalSize <= 32 || offset+totalSize > len(data) { + return nil, fmt.Errorf("invalid DXBC size") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, total size: %d bytes", dxbcDescription, totalSize), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// DTB (Device Tree Blob) +// ============================================================================ + +const dtbDescription = "Device tree blob (DTB)" + +func dtbMagic() [][]byte { + return [][]byte{{0xD0, 0x0D, 0xFE, 0xED}} +} + +func dtbParser(data []byte, offset int) (*SignatureResult, error) { + if offset+40 > len(data) { + return nil, fmt.Errorf("insufficient data for DTB header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"total_size", "u32"}, {"struct_offset", "u32"}, + {"strings_offset", "u32"}, {"mem_reserve_offset", "u32"}, + {"version", "u32"}, {"last_comp_version", "u32"}, + {"cpu_id", "u32"}, {"strings_size", "u32"}, {"struct_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + totalSize := parsed["total_size"] + structEnd := offset + parsed["struct_offset"] + parsed["struct_size"] + stringsEnd := offset + parsed["strings_offset"] + parsed["strings_size"] + if structEnd > len(data) || stringsEnd > len(data) { + return nil, fmt.Errorf("DTB offsets exceed data") + } + if totalSize <= 0 || offset+totalSize > len(data) { + return nil, fmt.Errorf("invalid DTB total size") + } + return &SignatureResult{ + Offset: offset, + Size: totalSize, + Description: fmt.Sprintf("%s, version: %d, CPU ID: %d, total size: %d bytes", dtbDescription, parsed["version"], parsed["cpu_id"], totalSize), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// Matter OTA firmware +// ============================================================================ + +const matterOTADescription = "Matter OTA firmware" + +func matterOTAMagic() [][]byte { + return [][]byte{{0x1e, 0xf1, 0xee, 0x1b}} +} + +func matterOTAParser(data []byte, offset int) (*SignatureResult, error) { + if offset+32 > len(data) { + return nil, fmt.Errorf("insufficient data for Matter OTA header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"total_size", "u32"}, {"header_size", "u32"}, + {"header_tlv_size", "u32"}, {"vendor_id", "u16"}, {"product_id", "u16"}, + {"version", "u32"}, {"min_version", "u32"}, + {"max_version", "u32"}, {"payload_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + headerSize := parsed["header_size"] + return &SignatureResult{ + Offset: offset, + Size: headerSize, + Description: fmt.Sprintf("%s, total size: %d bytes, header size: %d bytes, vendor id: 0x%x, product id: 0x%x, version: %d, payload size: %d bytes", matterOTADescription, parsed["total_size"], headerSize, parsed["vendor_id"], parsed["product_id"], parsed["version"], parsed["payload_size"]), + Confidence: ConfidenceHigh, + }, nil +} + +// ============================================================================ +// RTK firmware header +// ============================================================================ + +const rtkDescription = "RTK firmware header" + +func rtkMagic() [][]byte { + return [][]byte{[]byte("RTK0")} +} + +func rtkParser(data []byte, offset int) (*SignatureResult, error) { + if offset+16 > len(data) { + return nil, fmt.Errorf("insufficient data for RTK header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"header_size", "u32"}, {"image_size", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + availableData := len(data) - offset + if parsed["image_size"] != availableData { + return nil, fmt.Errorf("RTK image size mismatch") + } + return &SignatureResult{ + Offset: offset, + Size: parsed["header_size"], + Description: fmt.Sprintf("%s, header size: %d bytes, image size: %d", rtkDescription, parsed["header_size"], parsed["image_size"]), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// DPAPI blob data +// ============================================================================ + +const dpapiDescription = "DPAPI blob data" + +func dpapiMagic() [][]byte { + return [][]byte{ + {0x01, 0x00, 0x00, 0x00, 0xD0, 0x8c, 0x9d, 0xdf, 0x01, 0x15, 0xd1, 0x11, 0x8c, 0x7a, 0x00, 0xc0, 0x4f, 0xc2, 0x97, 0xeb}, + } +} + +func dpapiParser(data []byte, offset int) (*SignatureResult, error) { + if offset+44 > len(data) { + return nil, fmt.Errorf("insufficient data for DPAPI blob header") + } + hdr := data[offset:] + fields := []StructField{ + {"version", "u32"}, {"provider_guid1", "u32"}, {"provider_guid2", "u32"}, + {"provider_guid3", "u32"}, {"provider_guid4", "u32"}, + {"mk_version", "u32"}, {"mk_guid1", "u32"}, {"mk_guid2", "u32"}, + {"mk_guid3", "u32"}, {"mk_guid4", "u32"}, {"flags", "u32"}, + } + parsed, err := Parse(hdr, fields, "little") + if err != nil { + return nil, err + } + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, version: %d, flags: %d", dpapiDescription, parsed["version"], parsed["flags"]), + Confidence: ConfidenceMedium, + }, nil +} + +// ============================================================================ +// QEMU QCOW Image +// ============================================================================ + +const qcowDescription = "QEMU QCOW Image" + +func qcowMagic() [][]byte { + return [][]byte{[]byte("QFI\xFB")} +} + +func qcowParser(data []byte, offset int) (*SignatureResult, error) { + if offset+24 > len(data) { + return nil, fmt.Errorf("insufficient data for QCOW header") + } + hdr := data[offset:] + fields := []StructField{ + {"magic", "u32"}, {"version", "u32"}, {"backing_file_offset", "u64"}, + {"backing_file_size", "u32"}, {"cluster_bits", "u32"}, + {"disk_size", "u64"}, {"encryption_method", "u32"}, + } + parsed, err := Parse(hdr, fields, "big") + if err != nil { + return nil, err + } + encMap := map[int]string{0: "none", 1: "AES", 2: "LUKS"} + enc := lookupMap(encMap, parsed["encryption_method"], "unknown") + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, version: %d, storage media size: 0x%x bytes, cluster block size: 0x%x bytes, encryption method: %s", qcowDescription, parsed["version"], parsed["disk_size"], 1<= len(data) { + return nil, fmt.Errorf("PNG: not enough data for header") + } + + // Walk chunks starting after the 8-byte PNG header + chunkData := data[offset+pngHeaderLen:] + available := len(chunkData) + chunkOffset := 0 + var previousOffset *int + + for IsOffsetSafe(available, chunkOffset, previousOffset) { + if chunkOffset+chunkHeaderSize > available { + break + } + hdr, err := Parse(chunkData[chunkOffset:], chunkFields, "big") + if err != nil { + break + } + isLast := hdr["type"] == iendChunkType + totalChunkSize := chunkHeaderSize + hdr["length"] + crcSize + prev := chunkOffset + previousOffset = &prev + chunkOffset += totalChunkSize + if isLast { + pngSize := pngHeaderLen + chunkOffset + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", PNGDescription, pngSize), + Confidence: ConfidenceHigh, + Size: pngSize, + }, nil + } + } + + return nil, fmt.Errorf("PNG: failed to walk chunks to IEND") +} + +// --------------------------------------------------------------------------- +// JPEG +// --------------------------------------------------------------------------- + +const JPEGDescription = "JPEG image" + +func JPEGMagic() [][]byte { + return [][]byte{ + {0xFF, 0xD8, 0xFF, 0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00}, // JFIF + {0xFF, 0xD8, 0xFF, 0xE1}, // EXIF + {0xFF, 0xD8, 0xFF, 0xDB}, // DQT + } +} + +func JPEGParser(data []byte, offset int) (*SignatureResult, error) { + const markerMagic byte = 0xFF + const sosMarker byte = 0xDA + const eofMarker byte = 0xD9 + + noLengthMarkers := []byte{ + 0x00, 0x01, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, eofMarker, + } + sosSkipMarkers := []byte{ + 0x00, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, + } + + jpegData := data[offset:] + nextMarkerOffset := 0 + + for { + if nextMarkerOffset >= len(jpegData) { + break + } + magic := jpegData[nextMarkerOffset] + if magic != markerMagic { + break + } + nextMarkerOffset++ + + if nextMarkerOffset >= len(jpegData) { + break + } + markerID := jpegData[nextMarkerOffset] + nextMarkerOffset++ + + // If this marker has a length field + if !byteInSlice(markerID, noLengthMarkers) { + if nextMarkerOffset+2 > len(jpegData) { + break + } + segLen := int(jpegData[nextMarkerOffset])<<8 | int(jpegData[nextMarkerOffset+1]) + nextMarkerOffset += segLen + } + + // SOS marker: scan until next valid marker + if markerID == sosMarker { + for { + if nextMarkerOffset+2 > len(jpegData) { + break + } + if jpegData[nextMarkerOffset] == markerMagic && !byteInSlice(jpegData[nextMarkerOffset+1], sosSkipMarkers) { + break + } + nextMarkerOffset++ + } + } + + // EOF marker + if markerID == eofMarker { + jpegSize := nextMarkerOffset + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", JPEGDescription, jpegSize), + Confidence: ConfidenceMedium, + Size: jpegSize, + }, nil + } + } + + return nil, fmt.Errorf("JPEG: failed to find EOI marker") +} + +// --------------------------------------------------------------------------- +// BMP +// --------------------------------------------------------------------------- + +const BMPDescription = "BMP image (Bitmap)" + +func BMPMagic() [][]byte { + return [][]byte{ + {0x42, 0x4D}, // "BM" + } +} + +func BMPParser(data []byte, offset int) (*SignatureResult, error) { + const bmpFileHeaderSize = 14 + + bmpHeaderFields := []StructField{ + {"bfType", "u16"}, + {"bfSize", "u32"}, + {"bfReserved1", "u16"}, + {"bfReserved2", "u16"}, + {"bfOffBits", "u32"}, + } + + validDIBHeaderSizes := []int{12, 40, 108, 124} + + bmpData := data[offset:] + if len(bmpData) < StructSize(bmpHeaderFields) { + return nil, fmt.Errorf("BMP: not enough data for header") + } + + hdr, err := Parse(bmpData, bmpHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("BMP: parse error: %w", err) + } + + bmpDataSize := len(bmpData) + + // The BMP file size cannot be bigger than available data + if bmpDataSize < hdr["bfSize"] { + return nil, fmt.Errorf("BMP: reported size %d exceeds available data %d", hdr["bfSize"], bmpDataSize) + } + if hdr["bfSize"] == 0 { + return nil, fmt.Errorf("BMP: file size is 0") + } + if hdr["bfOffBits"] == 0 { + return nil, fmt.Errorf("BMP: bitmap bits offset is 0") + } + if hdr["bfOffBits"] > bmpDataSize { + return nil, fmt.Errorf("BMP: bitmap bits offset exceeds available data") + } + + // Validate DIB header size + if offset+bmpFileHeaderSize+4 > len(data) { + return nil, fmt.Errorf("BMP: not enough data for DIB header size") + } + dibHeaderSize := int(LE32(data, offset+bmpFileHeaderSize)) + validDIB := false + for _, vs := range validDIBHeaderSizes { + if dibHeaderSize == vs { + validDIB = true + break + } + } + if !validDIB { + return nil, fmt.Errorf("BMP: invalid DIB header size %d", dibHeaderSize) + } + + // The offset that points to the image data cannot point into the second header + if hdr["bfOffBits"] < bmpFileHeaderSize+dibHeaderSize { + return nil, fmt.Errorf("BMP: bitmap bits offset points into header") + } + + totalSize := hdr["bfSize"] + return &SignatureResult{ + Offset: offset, + Name: "bmp", + Description: fmt.Sprintf("BMP image, total size: %d", totalSize), + Confidence: ConfidenceMedium, + Size: totalSize, + }, nil +} + +// --------------------------------------------------------------------------- +// GIF +// --------------------------------------------------------------------------- + +const GIFDescription = "GIF image" + +func GIFMagic() [][]byte { + return [][]byte{ + {0x47, 0x49, 0x46, 0x38, 0x37, 0x61}, // GIF87a + {0x47, 0x49, 0x46, 0x38, 0x39, 0x61}, // GIF89a + } +} + +func GIFParser(data []byte, offset int) (*SignatureResult, error) { + gifHeaderFields := []StructField{ + {"magic_p1", "u24"}, + {"magic_p2", "u24"}, + {"image_width", "u16"}, + {"image_height", "u16"}, + {"flags", "u8"}, + {"bg_color_index", "u8"}, + {"aspect_ratio", "u8"}, + } + + gifData := data[offset:] + if len(gifData) < StructSize(gifHeaderFields) { + return nil, fmt.Errorf("GIF: not enough data for header") + } + + hdr, err := Parse(gifData, gifHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("GIF: parse error: %w", err) + } + + // Calculate header size including optional global color table + headerSize := StructSize(gifHeaderFields) + gifColorTableSize(hdr["flags"]) + + if offset+headerSize > len(data) { + return nil, fmt.Errorf("GIF: not enough data for header + color table") + } + + // Walk GIF blocks to find terminator + blockData := data[offset+headerSize:] + gifDataSize, err := getGIFDataSize(blockData) + if err != nil { + return nil, fmt.Errorf("GIF: %w", err) + } + + totalSize := headerSize + gifDataSize + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, %dx%d pixels, total size: %d bytes", GIFDescription, hdr["image_width"], hdr["image_height"], totalSize), + Confidence: ConfidenceHigh, + Size: totalSize, + }, nil +} + +func gifColorTableSize(flags int) int { + const hasColorTable = 0x80 + const colorTableSizeMask = 0x07 + + if flags&hasColorTable == 0 { + return 0 + } + encodedSize := (flags & colorTableSizeMask) + 1 + size := 3 + for i := 0; i < encodedSize; i++ { + size *= 2 + } + return size +} + +func getGIFDataSize(gifData []byte) (int, error) { + const extensionBlock byte = 0x21 + const terminatorBlock byte = 0x3B + const imageDescriptorBlock byte = 0x2C + + nextOffset := 0 + var previousOffset *int + available := len(gifData) + + for IsOffsetSafe(available, nextOffset, previousOffset) { + if nextOffset >= available { + break + } + blockType := gifData[nextOffset] + + var blockSize int + var err error + + switch blockType { + case imageDescriptorBlock: + blockSize, err = parseGIFImageDescriptor(gifData[nextOffset:]) + case extensionBlock: + blockSize, err = parseGIFExtension(gifData[nextOffset:]) + case terminatorBlock: + return nextOffset + 1, nil + default: + return 0, fmt.Errorf("unknown GIF block type 0x%02X at offset %d", blockType, nextOffset) + } + + if err != nil { + return 0, err + } + prev := nextOffset + previousOffset = &prev + nextOffset += blockSize + } + + return 0, fmt.Errorf("failed to find GIF terminator block") +} + +func parseGIFImageDescriptor(gifData []byte) (int, error) { + const lzwCodeSize = 1 + + imgDescFields := []StructField{ + {"magic", "u8"}, + {"image_left", "u16"}, + {"image_top", "u16"}, + {"image_width", "u16"}, + {"image_height", "u16"}, + {"flags", "u8"}, + } + + if len(gifData) < StructSize(imgDescFields) { + return 0, fmt.Errorf("GIF image descriptor: not enough data") + } + + hdr, err := Parse(gifData, imgDescFields, "little") + if err != nil { + return 0, err + } + + totalSize := StructSize(imgDescFields) + gifColorTableSize(hdr["flags"]) + lzwCodeSize + + if totalSize > len(gifData) { + return 0, fmt.Errorf("GIF image descriptor: not enough data for sub-blocks") + } + + subBlocksSize, err := parseGIFSubBlocks(gifData[totalSize:]) + if err != nil { + return 0, err + } + totalSize += subBlocksSize + return totalSize, nil +} + +func parseGIFExtension(extensionData []byte) (int, error) { + const plainText = 1 + const application = 0xFF + const headerSize = 2 + + extFields := []StructField{ + {"magic", "u8"}, + {"extension_type", "u8"}, + {"sub_block_offset", "u8"}, + } + + if len(extensionData) < StructSize(extFields) { + return 0, fmt.Errorf("GIF extension: not enough data") + } + + hdr, err := Parse(extensionData, extFields, "little") + if err != nil { + return 0, err + } + + subBlocksOffset := headerSize + extType := hdr["extension_type"] + if extType == application || extType == plainText { + subBlocksOffset += hdr["sub_block_offset"] + 1 + } + + if subBlocksOffset > len(extensionData) { + return 0, fmt.Errorf("GIF extension: sub-blocks offset exceeds data") + } + + subBlocksSize, err := parseGIFSubBlocks(extensionData[subBlocksOffset:]) + if err != nil { + return 0, err + } + return subBlocksOffset + subBlocksSize, nil +} + +func parseGIFSubBlocks(subBlockData []byte) (int, error) { + const subBlockTerminator byte = 0 + + available := len(subBlockData) + nextOffset := 0 + var previousOffset *int + + for IsOffsetSafe(available, nextOffset, previousOffset) { + if nextOffset >= available { + break + } + subBlockSize := subBlockData[nextOffset] + if subBlockSize == subBlockTerminator { + return nextOffset + 1, nil + } + prev := nextOffset + previousOffset = &prev + nextOffset += int(subBlockSize) + 1 + } + + return 0, fmt.Errorf("GIF sub-blocks: terminator not found") +} + +// --------------------------------------------------------------------------- +// SVG +// --------------------------------------------------------------------------- + +const SVGDescription = "SVG image" + +const svgOpenTag = "' and declare the SVG xmlns (which the loop below + // requires anyway via headTagCount). Verifying this up front, with a cap + // on how far we look for the closing '>', keeps the parser O(1) per call + // regardless of file size — so a buffer densely packed with " maxRootTagLen { + rootLimit = maxRootTagLen + } + rootEnd := bytes.IndexByte(svgData[:rootLimit], '>') + if rootEnd < 0 { + return nil, fmt.Errorf("SVG: root tag not terminated") + } + if !bytes.Contains(svgData[:rootEnd+1], headMagic) { + return nil, fmt.Errorf("SVG: root tag missing SVG namespace") + } + + headTagCount := 0 + unclosedSVGTags := 0 + + // Search for all occurrences + searchIdx := 0 + for searchIdx < len(svgData) { + // Find next occurrence of either tag + openIdx := bytes.Index(svgData[searchIdx:], openTag) + closeIdx := bytes.Index(svgData[searchIdx:], closeTag) + + var tagStart int + if openIdx == -1 && closeIdx == -1 { + break + } else if openIdx == -1 { + tagStart = searchIdx + closeIdx + } else if closeIdx == -1 { + tagStart = searchIdx + openIdx + } else if openIdx <= closeIdx { + tagStart = searchIdx + openIdx + } else { + tagStart = searchIdx + closeIdx + } + + tagBytes := svgData[tagStart:] + + // Find the end '>' of this tag + endIdx := bytes.IndexByte(tagBytes, '>') + if endIdx == -1 { + break + } + fullTag := tagBytes[:endIdx+1] + + isOpen := bytes.HasPrefix(fullTag, openTag) + isClose := bytes.HasPrefix(fullTag, closeTag) + isHead := bytes.Contains(fullTag, headMagic) + + if isHead { + headTagCount++ + } + if isOpen { + unclosedSVGTags++ + } + if isClose { + unclosedSVGTags-- + } + + // More than one head tag is invalid + if headTagCount > 1 { + break + } + + // If one head tag and all tags closed, that's EOF + if headTagCount == 1 && unclosedSVGTags == 0 { + totalSize := tagStart + len(svgCloseTag) + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, total size: %d bytes", SVGDescription, totalSize), + Confidence: ConfidenceMedium, + Size: totalSize, + }, nil + } + + searchIdx = tagStart + 1 + } + + return nil, fmt.Errorf("SVG: failed to find matching closing tag") +} + +// --------------------------------------------------------------------------- +// RIFF +// --------------------------------------------------------------------------- + +const RIFFDescription = "RIFF image" + +func RIFFMagic() [][]byte { + return [][]byte{ + {0x52, 0x49, 0x46, 0x46}, // "RIFF" + } +} + +func RIFFParser(data []byte, offset int) (*SignatureResult, error) { + const riffMagicValue = 0x46464952 // "RIFF" in LE + const chunkTypeStart = 8 + const chunkTypeEnd = 12 + const fileSizeOffset = 8 + + riffFields := []StructField{ + {"magic", "u32"}, + {"file_size", "u32"}, + {"chunk_type", "u32"}, + } + + riffData := data[offset:] + if len(riffData) < StructSize(riffFields) { + return nil, fmt.Errorf("RIFF: not enough data for header") + } + + hdr, err := Parse(riffData, riffFields, "little") + if err != nil { + return nil, fmt.Errorf("RIFF: parse error: %w", err) + } + + if hdr["magic"] != riffMagicValue { + return nil, fmt.Errorf("RIFF: invalid magic") + } + + if chunkTypeEnd > len(riffData) { + return nil, fmt.Errorf("RIFF: not enough data for chunk type string") + } + + // Get the RIFF type string (e.g., "WAVE", "AVI ") + typeBytes := riffData[chunkTypeStart:chunkTypeEnd] + typeString := strings.TrimSpace(string(typeBytes)) + + // Validate that the type string is valid UTF-8 / printable + for _, b := range typeBytes { + if b < 0x20 || b > 0x7E { + return nil, fmt.Errorf("RIFF: non-printable chunk type") + } + } + + totalSize := hdr["file_size"] + fileSizeOffset + + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, encoding type: %s, total size: %d bytes", RIFFDescription, typeString, totalSize), + Confidence: ConfidenceMedium, + Size: totalSize, + }, nil +} + +// --------------------------------------------------------------------------- +// PDF +// --------------------------------------------------------------------------- + +const PDFDescription = "PDF document" + +func PDFMagic() [][]byte { + return [][]byte{ + {0x25, 0x50, 0x44, 0x46, 0x2D, 0x31, 0x2E}, // "%PDF-1." + } +} + +func PDFParser(data []byte, offset int) (*SignatureResult, error) { + const minPDFSize = 16 + const newlineOffset = 8 + const minorNumberOffset = 7 + + const asciiZero byte = 0x30 + const asciiNine byte = 0x39 + const asciiNewline byte = 0x0A + const asciiPercent byte = 0x25 + const asciiCarriageReturn byte = 0x0D + + if offset+minPDFSize > len(data) { + return nil, fmt.Errorf("PDF: not enough data for header") + } + + pdfHeader := data[offset : offset+minPDFSize] + + // Get the minor version number + versionMinor := pdfHeader[minorNumberOffset] + if versionMinor < asciiZero || versionMinor > asciiNine { + return nil, fmt.Errorf("PDF: invalid minor version number") + } + + // After the version line, expect newline(s) followed by '%' + for _, b := range pdfHeader[newlineOffset:] { + if b == asciiNewline || b == asciiCarriageReturn { + continue + } else if b == asciiPercent { + return &SignatureResult{ + Offset: offset, + Description: fmt.Sprintf("%s, version 1.%d", PDFDescription, versionMinor-asciiZero), + Confidence: ConfidenceLow, + Size: 0, + }, nil + } else { + break + } + } + + return nil, fmt.Errorf("PDF: invalid header format") +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +func byteInSlice(b byte, s []byte) bool { + return bytes.ContainsRune(s, rune(b)) +} diff --git a/internal/leviathan/binsnoop/sig_misc.go b/internal/leviathan/binsnoop/sig_misc.go new file mode 100644 index 0000000..b353db1 --- /dev/null +++ b/internal/leviathan/binsnoop/sig_misc.go @@ -0,0 +1,1399 @@ +package binsnoop + +import ( + "bytes" + "fmt" + "strings" +) + +// ---- Copyright ---- + +const DescriptionCopyright = "Copyright text" + +// copyrightMagic returns magic bytes for copyright strings. +func copyrightMagic() [][]byte { + return [][]byte{ + []byte("copyright"), + []byte("Copyright"), + []byte("COPYRIGHT"), + } +} + +// copyrightParser validates copyright text signatures. +func copyrightParser(data []byte, offset int) (*SignatureResult, error) { + const magicSize = 9 + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionCopyright, + Confidence: ConfidenceHigh, + } + + copyrightStr := GetCString(data[offset:]) + if len(copyrightStr) <= magicSize { + return nil, fmt.Errorf("copyright string too short") + } + + result.Size = len(copyrightStr) + truncated := copyrightStr + if len(truncated) > 100 { + truncated = truncated[:100] + } + result.Description = fmt.Sprintf("%s: \"%s\"", result.Description, truncated) + return result, nil +} + +// ---- VxWorks ---- + +const ( + DescriptionVxWorksSymTab = "VxWorks symbol table" + DescriptionWINDKernelVersion = "VxWorks WIND kernel version" +) + +// windKernelMagic returns magic bytes for VxWorks WIND kernel version. +func windKernelMagic() [][]byte { + return [][]byte{ + []byte("WIND version "), + } +} + +// vxworksSymtabMagic returns magic bytes for VxWorks symbol tables. +func vxworksSymtabMagic() [][]byte { + return [][]byte{ + {0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + {0x00, 0x09, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + } +} + +// windKernelParser validates VxWorks WIND kernel version signatures. +func windKernelParser(data []byte, offset int) (*SignatureResult, error) { + const magicSize = 13 + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionWINDKernelVersion, + Confidence: ConfidenceLow, + } + + versionOffset := offset + magicSize + if versionOffset >= len(data) { + return nil, fmt.Errorf("WIND kernel version offset out of bounds") + } + + versionStr := GetCString(data[versionOffset:]) + if versionStr == "" { + return nil, fmt.Errorf("WIND kernel version string empty") + } + + result.Size = magicSize + len(versionStr) + result.Description = fmt.Sprintf("%s %s", result.Description, versionStr) + return result, nil +} + +// vxworksSymtabParser validates VxWorks symbol table signatures. +func vxworksSymtabParser(data []byte, offset int) (*SignatureResult, error) { + const symtabMagicOffset = 8 + + result := &SignatureResult{ + Description: DescriptionVxWorksSymTab, + Confidence: ConfidenceHigh, + } + + if offset < symtabMagicOffset { + return nil, fmt.Errorf("VxWorks symbol table: offset too small") + } + + symtabStart := offset - symtabMagicOffset + + // Determine endianness from the type field at offset 9 from symtab start + endianness := "little" + typeFieldOffset := symtabStart + 9 + if typeFieldOffset < len(data) && data[typeFieldOffset] == 0 { + endianness = "big" + } + + // Validate the first symbol table entry + symtabFields := []StructField{ + {"name_ptr", "u32"}, + {"value_ptr", "u32"}, + {"type", "u32"}, + {"group", "u32"}, + } + entrySize := StructSize(symtabFields) + + allowedTypes := map[int]string{ + 0x500: "function", + 0x700: "initialized data", + 0x900: "uninitialized data", + } + + if symtabStart+entrySize > len(data) { + return nil, fmt.Errorf("VxWorks symbol table: insufficient data") + } + + entry, err := Parse(data[symtabStart:], symtabFields, endianness) + if err != nil { + return nil, fmt.Errorf("VxWorks symbol table: parse failed") + } + + if _, ok := allowedTypes[entry["type"]]; !ok { + return nil, fmt.Errorf("VxWorks symbol table: invalid type %#x", entry["type"]) + } + if entry["name_ptr"] == 0 || entry["value_ptr"] == 0 { + return nil, fmt.Errorf("VxWorks symbol table: null pointers") + } + + result.Offset = symtabStart + result.Size = entrySize + result.Description = fmt.Sprintf("%s, total size: %d bytes", result.Description, result.Size) + return result, nil +} + +// ---- eCos ---- + +const DescriptionECosExceptionHandler = "eCos kernel exception handler" + +// ecosExceptionHandlerMagic returns magic bytes for eCos exception handlers (MIPS). +func ecosExceptionHandlerMagic() [][]byte { + return [][]byte{ + {0x00, 0x68, 0x1A, 0x40, 0x00, 0x00, 0x00, 0x00, 0x7F, 0x00, 0x5A, 0x33}, + {0x00, 0x68, 0x1A, 0x40, 0x7F, 0x00, 0x5A, 0x33}, + {0x40, 0x1A, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x33, 0x5A, 0x00, 0x7F}, + {0x40, 0x1A, 0x68, 0x00, 0x33, 0x5A, 0x00, 0x7F}, + } +} + +// ecosExceptionHandlerParser validates eCos exception handler signatures. +func ecosExceptionHandlerParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: DescriptionECosExceptionHandler, + Confidence: ConfidenceLow, + } + + endianness := "big" + if offset < len(data) && data[offset] == 0 { + endianness = "little" + } + + result.Description = fmt.Sprintf("%s, MIPS %s endian", result.Description, endianness) + return result, nil +} + +// ---- Linux ---- + +const ( + DescriptionLinuxARMzImage = "Linux ARM boot executable zImage" + DescriptionLinuxBootImage = "Linux kernel boot image" + DescriptionLinuxKernelVersion = "Linux kernel version" + DescriptionLinuxARM64BootImage = "Linux kernel ARM64 boot image" +) + +// linuxBootImageMagic returns magic bytes for Linux boot images. +func linuxBootImageMagic() [][]byte { + return [][]byte{ + {0xb8, 0xc0, 0x07, 0x8e, 0xd8, 0xb8, 0x00, 0x90, 0x8e, 0xc0, 0xb9, 0x00, 0x01, 0x29, 0xf6, 0x29}, + } +} + +// linuxKernelVersionMagic returns magic bytes for Linux kernel version strings. +func linuxKernelVersionMagic() [][]byte { + return [][]byte{ + []byte("Linux version "), + } +} + +// linuxARM64BootImageMagic returns magic bytes for Linux ARM64 boot images. +func linuxARM64BootImageMagic() [][]byte { + return [][]byte{ + {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x41, 0x52, 0x4D, 0x64}, // "\x00\x00\x00\x00\x00\x00\x00\x00ARMd" + } +} + +// linuxARMzImageMagic returns magic bytes for Linux ARM zImage. +func linuxARMzImageMagic() [][]byte { + return [][]byte{ + {0x18, 0x28, 0x6F, 0x01}, + {0x01, 0x6F, 0x28, 0x18}, + } +} + +// linuxARMzImageParser validates Linux ARM zImage signatures. +func linuxARMzImageParser(data []byte, offset int) (*SignatureResult, error) { + const magicOffset = 36 + const nopLE = 0xE1A00000 + const nopBE = 0x0000A0E1 + + result := &SignatureResult{ + Confidence: ConfidenceMedium, + Description: DescriptionLinuxARMzImage, + } + + if offset < magicOffset { + return nil, fmt.Errorf("ARM zImage: offset too small") + } + + result.Offset = offset - magicOffset + zimageData := data[result.Offset:] + + // Parse 8 NOP instructions at the start + nopFields := []StructField{ + {"nop1", "u32"}, {"nop2", "u32"}, {"nop3", "u32"}, {"nop4", "u32"}, + {"nop5", "u32"}, {"nop6", "u32"}, {"nop7", "u32"}, {"nop8", "u32"}, + } + + if len(zimageData) < StructSize(nopFields) { + return nil, fmt.Errorf("ARM zImage: insufficient data") + } + + nops, err := Parse(zimageData, nopFields, "little") + if err != nil { + return nil, fmt.Errorf("ARM zImage: parse failed") + } + + nop1 := nops["nop1"] + if nop1 != nops["nop2"] || nop1 != nops["nop3"] || nop1 != nops["nop4"] || + nop1 != nops["nop5"] || nop1 != nops["nop6"] || nop1 != nops["nop7"] || nop1 != nops["nop8"] { + return nil, fmt.Errorf("ARM zImage: NOP instructions not uniform") + } + + var endianness string + if nop1 == nopLE { + endianness = "little" + } else if nop1 == nopBE { + endianness = "big" + } else { + return nil, fmt.Errorf("ARM zImage: unrecognized NOP value") + } + + result.Description = fmt.Sprintf("%s, %s endian", result.Description, endianness) + return result, nil +} + +// linuxARM64BootImageParser validates Linux ARM64 boot image signatures. +func linuxARM64BootImageParser(data []byte, offset int) (*SignatureResult, error) { + const magicOffset = 0x30 + const flagsEndianMask = 1 + const bigEndianFlag = 1 + + result := &SignatureResult{ + Confidence: ConfidenceMedium, + Description: DescriptionLinuxARM64BootImage, + } + + if offset < magicOffset { + return nil, fmt.Errorf("ARM64 boot image: offset too small") + } + + result.Offset = offset - magicOffset + imgData := data[result.Offset:] + + bootImgFields := []StructField{ + {"code0", "u32"}, + {"code1", "u32"}, + {"image_load_offset", "u64"}, + {"image_size", "u64"}, + {"flags", "u64"}, + {"reserved1", "u64"}, + {"reserved2", "u64"}, + {"reserved3", "u64"}, + {"magic", "u32"}, + {"pe_offset", "u32"}, + } + + structSize := StructSize(bootImgFields) + if len(imgData) < structSize { + return nil, fmt.Errorf("ARM64 boot image: insufficient data") + } + + imgHeader, err := Parse(imgData, bootImgFields, "little") + if err != nil { + return nil, fmt.Errorf("ARM64 boot image: parse failed") + } + + if imgHeader["reserved1"] != 0 || imgHeader["reserved2"] != 0 || imgHeader["reserved3"] != 0 { + return nil, fmt.Errorf("ARM64 boot image: reserved fields not zero") + } + + peOffset := imgHeader["pe_offset"] + peEnd := peOffset + 2 + if peEnd > len(imgData) { + return nil, fmt.Errorf("ARM64 boot image: PE offset out of bounds") + } + if imgData[peOffset] != 'P' || imgData[peOffset+1] != 'E' { + return nil, fmt.Errorf("ARM64 boot image: PE signature not found") + } + + flags := imgHeader["flags"] + // Reserved bits are the upper 60 bits (mask 0xFFFFFFFFFFFFFFF0) + if (uint64(flags) & 0xFFFFFFFFFFFFFFF0) != 0 { + return nil, fmt.Errorf("ARM64 boot image: reserved flag bits set") + } + + endianness := "little" + if (flags & flagsEndianMask) == bigEndianFlag { + endianness = "big" + } + + imageSize := imgHeader["image_size"] + result.Size = structSize + result.Description = fmt.Sprintf("%s, %s endian, effective image size: %d bytes", + result.Description, endianness, imageSize) + return result, nil +} + +// linuxBootImageParser validates Linux boot image signatures. +func linuxBootImageParser(data []byte, offset int) (*SignatureResult, error) { + const hdrsOffset = 514 + const hdrsExpected = "!HdrS" + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionLinuxBootImage, + } + + hdrsStart := offset + hdrsOffset + hdrsEnd := hdrsStart + len(hdrsExpected) + + if hdrsEnd > len(data) { + return nil, fmt.Errorf("Linux boot image: insufficient data for HdrS check") + } + + actual := string(data[hdrsStart:hdrsEnd]) + if actual != hdrsExpected { + return nil, fmt.Errorf("Linux boot image: HdrS signature not found") + } + + return result, nil +} + +// linuxKernelVersionParser validates Linux kernel version string signatures. +func linuxKernelVersionParser(data []byte, offset int) (*SignatureResult, error) { + const periodByte byte = 0x2E + const periodOffset1 = 15 + const periodOffset2 = 17 + const periodOffset3 = 18 + const minFileSize = 100 * 1024 + const minVersionStrLen = 75 + const gccVersionStr = "gcc " + + result := &SignatureResult{ + Offset: offset, + Confidence: ConfidenceLow, + } + + fileSize := len(data) + if fileSize <= minFileSize { + return nil, fmt.Errorf("Linux kernel version: file too small") + } + + kvString := GetCString(data[offset:]) + if len(kvString) <= minVersionStrLen { + return nil, fmt.Errorf("Linux kernel version: string too short") + } + + if !strings.Contains(kvString, gccVersionStr) { + return nil, fmt.Errorf("Linux kernel version: no gcc version string") + } + + if !strings.Contains(kvString, "@") { + return nil, fmt.Errorf("Linux kernel version: no ampersand") + } + + if !strings.HasSuffix(kvString, "\n") { + return nil, fmt.Errorf("Linux kernel version: does not end with newline") + } + + kvBytes := []byte(kvString) + if periodOffset1 >= len(kvBytes) || periodOffset3 >= len(kvBytes) { + return nil, fmt.Errorf("Linux kernel version: string too short for period checks") + } + + if kvBytes[periodOffset1] != periodByte || (kvBytes[periodOffset2] != periodByte && kvBytes[periodOffset3] != periodByte) { + return nil, fmt.Errorf("Linux kernel version: periods not at expected offsets") + } + + // Search for Linux kernel symbol table + symtabMagic := []byte("\x000\x001\x002\x003\x004\x005\x006\x007\x008\x009\x00") + symtabPresent := bytes.Count(data, symtabMagic) == 1 + + if symtabPresent { + result.Offset = 0 + result.Size = len(data) + } else { + result.Size = len(kvString) + result.ExtractionDeclined = true + } + + result.Description = fmt.Sprintf("%s, has symbol table: %t", + strings.TrimSpace(kvString), symtabPresent) + return result, nil +} + +// ---- EFI GPT ---- + +const DescriptionEFIGPT = "EFI Global Partition Table" + +// efigptMagic returns magic bytes for EFI GPT headers. +func efigptMagic() [][]byte { + return [][]byte{ + {0x55, 0xAA, 0x45, 0x46, 0x49, 0x20, 0x50, 0x41, 0x52, 0x54}, // "\x55\xAAEFI PART" + } +} + +func lbaToOffset(lba int) int { + return lba * 512 +} + +// efigptParser validates EFI GPT header signatures. +func efigptParser(data []byte, offset int) (*SignatureResult, error) { + const magicOffset = 0x01FE + const blockSize = 512 + const expectedRevision = 0x00010000 + + result := &SignatureResult{ + Description: DescriptionEFIGPT, + Confidence: ConfidenceHigh, + } + + availableData := len(data) - offset + if offset < magicOffset { + return nil, fmt.Errorf("EFI GPT: offset too small") + } + + result.Offset = offset - magicOffset + efiData := data[result.Offset:] + + if len(efiData) <= blockSize { + return nil, fmt.Errorf("EFI GPT: insufficient data for GPT structure") + } + + gptData := efiData[blockSize:] + + gptFields := []StructField{ + {"magic", "u64"}, + {"revision", "u32"}, + {"header_size", "u32"}, + {"header_crc", "u32"}, + {"reserved", "u32"}, + {"my_lba", "u64"}, + {"alternate_lba", "u64"}, + {"first_usable_lba", "u64"}, + {"last_usable_lba", "u64"}, + {"disk_guid_p1", "u64"}, + {"disk_guid_p2", "u64"}, + {"partition_entry_lba", "u64"}, + {"partition_entry_count", "u32"}, + {"partition_entry_size", "u32"}, + {"partition_entries_crc", "u32"}, + } + + if len(gptData) < StructSize(gptFields) { + return nil, fmt.Errorf("EFI GPT: insufficient GPT header data") + } + + gptHeader, err := Parse(gptData, gptFields, "little") + if err != nil { + return nil, fmt.Errorf("EFI GPT: parse failed") + } + + if gptHeader["reserved"] != 0 { + return nil, fmt.Errorf("EFI GPT: reserved field not zero") + } + if gptHeader["revision"] != expectedRevision { + return nil, fmt.Errorf("EFI GPT: unexpected revision") + } + + partEntriesStart := lbaToOffset(gptHeader["partition_entry_lba"]) + partEntriesEnd := partEntriesStart + (gptHeader["partition_entry_count"] * gptHeader["partition_entry_size"]) + + if partEntriesEnd > len(efiData) || partEntriesStart >= partEntriesEnd { + return nil, fmt.Errorf("EFI GPT: partition entries out of bounds") + } + + partEntriesData := efiData[partEntriesStart:partEntriesEnd] + + // Validate partition entries CRC + if CRC32(partEntriesData) != uint32(gptHeader["partition_entries_crc"]) { + return nil, fmt.Errorf("EFI GPT: partition entries CRC mismatch") + } + + // Parse partition entries to find total size + entryFields := []StructField{ + {"type_guid_p1", "u64"}, + {"type_guid_p2", "u64"}, + {"partition_guid_p1", "u64"}, + {"partition_guid_p2", "u64"}, + {"starting_lba", "u64"}, + {"ending_lba", "u64"}, + {"attributes", "u64"}, + } + + entrySize := gptHeader["partition_entry_size"] + totalSize := 0 + + for i := 0; i+entrySize <= len(partEntriesData); i += entrySize { + entry, err := Parse(partEntriesData[i:], entryFields, "little") + if err != nil { + continue + } + if entry["type_guid_p1"] == 0 && entry["type_guid_p2"] == 0 { + continue + } + startOff := lbaToOffset(entry["starting_lba"]) + endOff := lbaToOffset(entry["ending_lba"]) + if startOff < endOff && endOff > totalSize { + totalSize = endOff + } + } + + if totalSize == 0 { + return nil, fmt.Errorf("EFI GPT: no valid partitions found") + } + + if totalSize > availableData { + result.Size = availableData + } else { + result.Size = totalSize + } + result.Description = fmt.Sprintf("%s, total size: %d", result.Description, result.Size) + return result, nil +} + +// ---- MBR ---- + +const DescriptionMBR = "DOS Master Boot Record" +const MBRMagicOffset = 0x01FE + +// mbrMagic returns magic bytes for MBR signatures. +func mbrMagic() [][]byte { + return [][]byte{ + {0x55, 0xAA}, + } +} + +// mbrParser validates MBR header signatures. +func mbrParser(data []byte, offset int) (*SignatureResult, error) { + const blockSize = 512 + const minImageSize = blockSize * 2 + const partitionCount = 4 + const partitionTableOffset = 446 + + result := &SignatureResult{ + Description: DescriptionMBR, + Confidence: ConfidenceMedium, + } + + if offset != MBRMagicOffset { + return nil, fmt.Errorf("MBR: magic not at expected offset") + } + + result.Offset = offset - MBRMagicOffset + + partEntryFields := []StructField{ + {"status", "u8"}, + {"chs_start", "u24"}, + {"os_type", "u8"}, + {"chs_end", "u24"}, + {"lba_start", "u32"}, + {"lba_size", "u32"}, + } + + knownOSTypes := map[int]string{ + 0x07: "NTFS_IFS_HPFS_exFAT", + 0x0B: "FAT32", + 0x0C: "FAT32", + 0x43: "Linux", + 0x4D: "QNX Primary Volume", + 0x4E: "QNX Secondary Volume", + 0x81: "Minix", + 0x83: "Linux", + 0x8E: "Linux LVM", + 0x96: "ISO-9660", + 0xB1: "QNXv6 File System", + 0xB2: "QNXv6 File System", + 0xB3: "QNXv6 File System", + 0xEE: "EFI GPT Protective", + 0xEF: "EFI System Partition", + } + + allowedStatus := map[int]bool{0: true, 0x80: true} + entrySize := StructSize(partEntryFields) + + partTableStart := partitionTableOffset + partTableEnd := partTableStart + (entrySize * partitionCount) + + if partTableEnd > len(data) { + return nil, fmt.Errorf("MBR: insufficient data for partition table") + } + + partTable := data[partTableStart:partTableEnd] + + type mbrPartition struct { + start int + size int + name string + } + + var partitions []mbrPartition + imageSize := 0 + + for i := 0; i < partitionCount; i++ { + entryStart := i * entrySize + entry, err := Parse(partTable[entryStart:], partEntryFields, "little") + if err != nil { + return nil, fmt.Errorf("MBR: partition entry parse failed") + } + + if entry["os_type"] == 0 && entry["lba_size"] == 0 { + continue + } + + if !allowedStatus[entry["status"]] { + continue + } + + partName := "Unknown" + if name, ok := knownOSTypes[entry["os_type"]]; ok { + partName = name + } + + partStart := entry["lba_start"] * blockSize + partSize := entry["lba_size"] * blockSize + partEnd := partStart + partSize + + if partEnd > len(data) { + continue + } + + if partStart != result.Offset { + partitions = append(partitions, mbrPartition{ + start: partStart, + size: partSize, + name: partName, + }) + } + + if partEnd > imageSize { + imageSize = partEnd + } + } + + if len(partitions) == 0 { + return nil, fmt.Errorf("MBR: no valid partitions") + } + + if imageSize <= minImageSize { + return nil, fmt.Errorf("MBR: image size too small") + } + + result.Size = imageSize + for _, p := range partitions { + if p.start == result.Offset { + result.ExtractionDeclined = true + } + result.Description = fmt.Sprintf("%s, partition: %s", result.Description, p.name) + } + result.Description = fmt.Sprintf("%s, image size: %d bytes", result.Description, result.Size) + return result, nil +} + +// ---- UEFI ---- + +const ( + DescriptionUEFIVolume = "UEFI PI firmware volume" + DescriptionUEFICapsule = "UEFI capsule image" +) + +// uefiVolumeMagic returns magic bytes for UEFI PI firmware volumes. +func uefiVolumeMagic() [][]byte { + return [][]byte{ + []byte("_FVH"), + } +} + +// uefiCapsuleMagic returns magic bytes for UEFI capsule images. +func uefiCapsuleMagic() [][]byte { + return [][]byte{ + // EFI capsule GUID + {0xBD, 0x86, 0x66, 0x3B, 0x76, 0x0D, 0x30, 0x40, 0xB7, 0x0E, 0xB5, 0x51, 0x9E, 0x2F, 0xC5, 0xA0}, + // EFI2 capsule GUID + {0x8B, 0xA6, 0x3C, 0x4A, 0x23, 0x77, 0xFB, 0x48, 0x80, 0x3D, 0x57, 0x8C, 0xC1, 0xFE, 0xC4, 0x4D}, + // UEFI capsule GUID + {0xB9, 0x82, 0x91, 0x53, 0xB5, 0xAB, 0x91, 0x43, 0xB6, 0x9A, 0xE3, 0xA9, 0x43, 0xF7, 0x2F, 0xCC}, + } +} + +// uefiVolumeParser validates UEFI PI firmware volume signatures. +func uefiVolumeParser(data []byte, offset int) (*SignatureResult, error) { + const uefiMagicOffset = 40 + + result := &SignatureResult{ + Description: DescriptionUEFIVolume, + Confidence: ConfidenceMedium, + } + + if offset < uefiMagicOffset { + return nil, fmt.Errorf("UEFI volume: offset too small") + } + + result.Offset = offset - uefiMagicOffset + uefiData := data[result.Offset:] + + validRevisions := map[int]bool{1: true, 2: true} + + volFields := []StructField{ + {"volume_size", "u64"}, + {"magic", "u32"}, + {"attributes", "u32"}, + {"header_size", "u16"}, + {"header_crc", "u16"}, + {"extended_header_offset", "u16"}, + {"reserved", "u8"}, + {"revision", "u8"}, + } + + if len(uefiData) < StructSize(volFields) { + return nil, fmt.Errorf("UEFI volume: insufficient data") + } + + volHeader, err := Parse(uefiData, volFields, "little") + if err != nil { + return nil, fmt.Errorf("UEFI volume: parse failed") + } + + if volHeader["header_size"] >= volHeader["volume_size"] { + return nil, fmt.Errorf("UEFI volume: header size >= volume size") + } + if volHeader["reserved"] != 0 { + return nil, fmt.Errorf("UEFI volume: reserved field not zero") + } + if !validRevisions[volHeader["revision"]] { + return nil, fmt.Errorf("UEFI volume: invalid revision") + } + + volumeSize := volHeader["volume_size"] + if len(data) < result.Offset+volumeSize { + return nil, fmt.Errorf("UEFI volume: volume size exceeds data") + } + + result.Size = volumeSize + result.Description = fmt.Sprintf("%s, header CRC: %#X, header size: %d bytes, total size: %d bytes", + result.Description, uint32(volHeader["header_crc"]), volHeader["header_size"], volumeSize) + return result, nil +} + +// uefiCapsuleParser validates UEFI capsule image signatures. +func uefiCapsuleParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: DescriptionUEFICapsule, + Confidence: ConfidenceMedium, + } + + capsuleFields := []StructField{ + {"guid_p1", "u64"}, + {"guid_p2", "u64"}, + {"header_size", "u32"}, + {"flags", "u32"}, + {"total_size", "u32"}, + } + + if offset+StructSize(capsuleFields) > len(data) { + return nil, fmt.Errorf("UEFI capsule: insufficient data") + } + + capsuleHeader, err := Parse(data[offset:], capsuleFields, "little") + if err != nil { + return nil, fmt.Errorf("UEFI capsule: parse failed") + } + + headerSize := capsuleHeader["header_size"] + totalSize := capsuleHeader["total_size"] + availableData := len(data) - offset + + if headerSize >= totalSize { + return nil, fmt.Errorf("UEFI capsule: header size >= total size") + } + + if totalSize < availableData { + return nil, fmt.Errorf("UEFI capsule: total size smaller than available data") + } + + result.Size = totalSize + result.Description = fmt.Sprintf("%s, header size: %d bytes, total size: %d bytes", + result.Description, headerSize, totalSize) + return result, nil +} + +// ---- PCH ROM ---- + +const DescriptionPCHRom = "Intel serial flash for PCH ROM" + +// pchRomMagic returns magic bytes for PCH ROM signatures. +func pchRomMagic() [][]byte { + return [][]byte{ + {0x5a, 0xa5, 0xf0, 0x0f}, + } +} + +// pchRomParser validates PCH ROM signatures. +func pchRomParser(data []byte, offset int) (*SignatureResult, error) { + const pchMagicOffset = 16 + const headerStructOffset = 16 + const headerStructSize = 8 + const expectedFCBA = 3 + const expectedFRBA = 4 + const flashRegionCount = 5 + const flashRegionEntrySize = 4 + + result := &SignatureResult{ + Description: DescriptionPCHRom, + } + + if offset < pchMagicOffset { + return nil, fmt.Errorf("PCH ROM: offset too small") + } + + result.Offset = offset - pchMagicOffset + pchData := data[result.Offset:] + + structStart := headerStructOffset + structEnd := structStart + headerStructSize + if structEnd > len(pchData) { + return nil, fmt.Errorf("PCH ROM: insufficient data for header") + } + + pchHeaderFields := []StructField{ + {"flmagic", "u32"}, + {"flmap0_fcba", "u8"}, + {"flmap0_nc", "u8"}, + {"flmap0_frba_nr", "u16"}, + } + + pchHeader, err := Parse(pchData[structStart:structEnd], pchHeaderFields, "little") + if err != nil { + return nil, fmt.Errorf("PCH ROM: header parse failed") + } + + if pchHeader["flmap0_fcba"] != expectedFCBA { + return nil, fmt.Errorf("PCH ROM: unexpected FCBA value") + } + if pchHeader["flmap0_frba_nr"] != expectedFRBA { + return nil, fmt.Errorf("PCH ROM: unexpected FRBA value") + } + + nc := pchHeader["flmap0_nc"] + if nc != 0 && nc != 1 { + return nil, fmt.Errorf("PCH ROM: unexpected NC value") + } + + // Parse flash region entries + fcba := pchHeader["flmap0_fcba"] + flashRegionBaseAddr := ((fcba >> 16) & 0xFF) << 4 + + imageSize := 0 + for i := 0; i < flashRegionCount; i++ { + regionEntryStart := flashRegionBaseAddr + (i * flashRegionEntrySize) + regionEntryEnd := regionEntryStart + flashRegionEntrySize + + if regionEntryEnd > len(pchData) { + return nil, fmt.Errorf("PCH ROM: region entry out of bounds") + } + + regionFields := []StructField{{"region_value", "u32"}} + regionEntry, err := Parse(pchData[regionEntryStart:regionEntryEnd], regionFields, "little") + if err != nil { + return nil, fmt.Errorf("PCH ROM: region entry parse failed") + } + + regionValue := regionEntry["region_value"] + regionBase := (regionValue & 0x1FFF) << 12 + regionLimit := (((regionValue & 0x1FFF0000) >> 4) | 0xFFFF) + 1 + regionSize := regionLimit - regionBase + + if regionSize > 0 && regionLimit > imageSize { + imageSize = regionLimit + } + } + + if imageSize == 0 { + return nil, fmt.Errorf("PCH ROM: no valid flash regions") + } + + result.Size = headerStructOffset + imageSize + return result, nil +} + +// ---- PCAP ---- + +const DescriptionPcapNG = "Pcap-NG capture file" + +// pcapngMagic returns magic bytes for Pcap-NG files. +func pcapngMagic() [][]byte { + return [][]byte{ + {0x0A, 0x0D, 0x0D, 0x0A}, + } +} + +// pcapngParser validates Pcap-NG capture file signatures. +func pcapngParser(data []byte, offset int) (*SignatureResult, error) { + const sectionHeaderBlockType = 0x0A0D0D0A + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionPcapNG, + Confidence: ConfidenceHigh, + } + + sectionFields := []StructField{ + {"block_type", "u32"}, + {"block_size", "u32"}, + {"endian_magic", "u32"}, + {"major_version", "u16"}, + {"minor_version", "u16"}, + {"section_length", "u32"}, + } + + if offset+StructSize(sectionFields) > len(data) { + return nil, fmt.Errorf("Pcap-NG: insufficient data") + } + + sectionHeader, err := Parse(data[offset:], sectionFields, "little") + if err != nil { + return nil, fmt.Errorf("Pcap-NG: parse failed") + } + + // Determine endianness + endianMagic := sectionHeader["endian_magic"] + var endianness string + switch endianMagic { + case 0x1A2B3C4D: + endianness = "little" + case 0x4D3C2B1A: + endianness = "big" + default: + return nil, fmt.Errorf("Pcap-NG: invalid endian magic") + } + + // Re-parse with correct endianness + blockFields := []StructField{ + {"block_type", "u32"}, + {"block_size", "u32"}, + } + footerFields := []StructField{{"block_size", "u32"}} + footerSize := StructSize(footerFields) + + blockHeader, err := Parse(data[offset:], blockFields, endianness) + if err != nil { + return nil, fmt.Errorf("Pcap-NG: block header parse failed") + } + + if blockHeader["block_type"] != sectionHeaderBlockType { + return nil, fmt.Errorf("Pcap-NG: unexpected block type") + } + + blockSize := blockHeader["block_size"] + if blockSize < footerSize { + return nil, fmt.Errorf("Pcap-NG: block size too small") + } + + // Validate block size via footer + footerStart := offset + blockSize - footerSize + if footerStart+footerSize > len(data) { + return nil, fmt.Errorf("Pcap-NG: block footer out of bounds") + } + + blockFooter, err := Parse(data[footerStart:], footerFields, endianness) + if err != nil { + return nil, fmt.Errorf("Pcap-NG: block footer parse failed") + } + + if blockFooter["block_size"] != blockSize { + return nil, fmt.Errorf("Pcap-NG: block size mismatch between header and footer") + } + + // Reserved bit check + if (blockHeader["block_type"] & 0x80000000) != 0 { + return nil, fmt.Errorf("Pcap-NG: reserved bit set in block type") + } + + result.Size = blockSize + result.Description = fmt.Sprintf("%s, total size: %d bytes", result.Description, result.Size) + + if offset == 0 && result.Size == len(data) { + result.ExtractionDeclined = true + } + + return result, nil +} + +// ---- PJL ---- + +const DescriptionPJL = "HP Printer Job Language data" + +// pjlMagic returns magic bytes for PJL data. +func pjlMagic() [][]byte { + return [][]byte{ + {0x1B, 0x25, 0x2D, 0x31, 0x32, 0x33, 0x34, 0x35, 0x58, 0x40, 0x50, 0x4A, 0x4C}, // "\x1B%-12345X@PJL" + } +} + +// pjlParser validates PJL data signatures. +func pjlParser(data []byte, offset int) (*SignatureResult, error) { + const pjlCommandsOffset = 9 + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionPJL, + Confidence: ConfidenceLow, + } + + if offset+pjlCommandsOffset >= len(data) { + return nil, fmt.Errorf("PJL: insufficient data") + } + + pjlText := GetCString(data[offset+pjlCommandsOffset:]) + if len(pjlText) == 0 { + return nil, fmt.Errorf("PJL: no command text found") + } + + result.Size = len(pjlText) + displayText := strings.ReplaceAll(pjlText, "\r", " ") + displayText = strings.ReplaceAll(displayText, "\n", "") + result.Description = fmt.Sprintf("%s: \"%s\"", result.Description, displayText) + return result, nil +} + +// ---- S-record ---- + +const ( + DescriptionSRecord = "Motorola S-record" + DescriptionSRecordGeneric = "Motorola S-record (generic)" +) + +// srecordMagic returns magic bytes for Motorola S-records. +func srecordMagic() [][]byte { + return [][]byte{ + []byte("S00600004844521B"), + } +} + +// srecordGenericMagic returns short/generic magic bytes for S-records. +func srecordGenericMagic() [][]byte { + return [][]byte{ + []byte("S0"), + } +} + +// srecordParser validates Motorola S-record signatures. +func srecordParser(data []byte, offset int) (*SignatureResult, error) { + const unixTermChar byte = 0x0A + const windowsTermChar byte = 0x0D + + result := &SignatureResult{ + Offset: offset, + Description: DescriptionSRecord, + Confidence: ConfidenceHigh, + } + + availableData := len(data) + + // Search for srec footer lines + srecFooters := [][]byte{[]byte("\nS9"), []byte("\nS8"), []byte("\nS7")} + footerLen := 3 + + searchData := data[offset:] + + for _, footer := range srecFooters { + idx := bytes.Index(searchData, footer) + if idx < 0 { + continue + } + + osType := "Unix" + srecEOF := offset + idx + footerLen + + for srecEOF < availableData { + b := data[srecEOF] + if b == windowsTermChar || b == unixTermChar { + if b == windowsTermChar { + srecEOF++ + osType = "Windows" + } + if srecEOF >= availableData { + return nil, fmt.Errorf("S-record: unexpected end after footer") + } + if data[srecEOF] != unixTermChar { + return nil, fmt.Errorf("S-record: invalid termination") + } + srecEOF++ + result.Size = srecEOF - offset + result.Description = fmt.Sprintf("%s, origin OS: %s, total size: %d bytes", + result.Description, osType, result.Size) + return result, nil + } + srecEOF++ + } + } + + return nil, fmt.Errorf("S-record: no valid footer found") +} + +// ---- DMG ---- + +const DescriptionDMG = "Apple Disk iMaGe" + +// dmgMagic returns magic bytes for DMG files (footer signature). +func dmgMagic() [][]byte { + return [][]byte{ + {0x6B, 0x6F, 0x6C, 0x79, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x02, 0x00}, // "koly\x00\x00\x00\x04\x00\x00\x02\x00" + } +} + +// dmgParser validates DMG footer signatures. +func dmgParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Description: DescriptionDMG, + Confidence: ConfidenceHigh + 1, + } + + dmgFooterFields := []StructField{ + {"magic", "u32"}, + {"version", "u32"}, + {"header_size", "u32"}, + {"flags", "u32"}, + {"running_data_fork_offset", "u64"}, + {"data_fork_offset", "u64"}, + {"data_fork_length", "u64"}, + {"rsrc_fork_offset", "u64"}, + {"rsrc_fork_length", "u64"}, + {"segment_number", "u32"}, + {"segment_count", "u32"}, + {"segment_id_p1", "u64"}, + {"segment_id_p2", "u64"}, + {"data_checksum_type", "u32"}, + {"data_checksum_size", "u32"}, + {"data_checksum_1", "u32"}, + {"data_checksum_2", "u32"}, + {"data_checksum_3", "u32"}, + {"data_checksum_4", "u32"}, + {"data_checksum_5", "u32"}, + {"data_checksum_6", "u32"}, + {"data_checksum_7", "u32"}, + {"data_checksum_8", "u32"}, + {"data_checksum_9", "u32"}, + {"data_checksum_10", "u32"}, + {"data_checksum_11", "u32"}, + {"data_checksum_12", "u32"}, + {"data_checksum_13", "u32"}, + {"data_checksum_14", "u32"}, + {"data_checksum_15", "u32"}, + {"data_checksum_16", "u32"}, + {"data_checksum_17", "u32"}, + {"data_checksum_18", "u32"}, + {"data_checksum_19", "u32"}, + {"data_checksum_20", "u32"}, + {"data_checksum_21", "u32"}, + {"data_checksum_22", "u32"}, + {"data_checksum_23", "u32"}, + {"data_checksum_24", "u32"}, + {"data_checksum_25", "u32"}, + {"data_checksum_26", "u32"}, + {"data_checksum_27", "u32"}, + {"data_checksum_28", "u32"}, + {"data_checksum_29", "u32"}, + {"data_checksum_30", "u32"}, + {"data_checksum_31", "u32"}, + {"data_checksum_32", "u32"}, + {"xml_offset", "u64"}, + {"xml_length", "u64"}, + {"reserved_1", "u64"}, + {"reserved_2", "u64"}, + {"reserved_3", "u64"}, + {"reserved_4", "u64"}, + {"reserved_5", "u64"}, + {"reserved_6", "u64"}, + {"reserved_7", "u64"}, + {"reserved_8", "u64"}, + {"reserved_9", "u64"}, + {"reserved_10", "u64"}, + {"reserved_11", "u64"}, + {"reserved_12", "u64"}, + {"reserved_13", "u64"}, + {"reserved_14", "u64"}, + {"reserved_15", "u64"}, + {"checksum_type", "u32"}, + {"checksum_size", "u32"}, + {"checksum_1", "u32"}, + {"checksum_2", "u32"}, + {"checksum_3", "u32"}, + {"checksum_4", "u32"}, + {"checksum_5", "u32"}, + {"checksum_6", "u32"}, + {"checksum_7", "u32"}, + {"checksum_8", "u32"}, + {"checksum_9", "u32"}, + {"checksum_10", "u32"}, + {"checksum_11", "u32"}, + {"checksum_12", "u32"}, + {"checksum_13", "u32"}, + {"checksum_14", "u32"}, + {"checksum_15", "u32"}, + {"checksum_16", "u32"}, + {"checksum_17", "u32"}, + {"checksum_18", "u32"}, + {"checksum_19", "u32"}, + {"checksum_20", "u32"}, + {"checksum_21", "u32"}, + {"checksum_22", "u32"}, + {"checksum_23", "u32"}, + {"checksum_24", "u32"}, + {"checksum_25", "u32"}, + {"checksum_26", "u32"}, + {"checksum_27", "u32"}, + {"checksum_28", "u32"}, + {"checksum_29", "u32"}, + {"checksum_30", "u32"}, + {"checksum_31", "u32"}, + {"checksum_32", "u32"}, + {"image_variant", "u32"}, + {"sector_count", "u64"}, + {"reserved_16", "u32"}, + {"reserved_17", "u32"}, + {"reserved_18", "u32"}, + } + + footerSize := StructSize(dmgFooterFields) + + if offset+footerSize > len(data) { + return nil, fmt.Errorf("DMG: insufficient data for footer") + } + + dmgFooter, err := Parse(data[offset:], dmgFooterFields, "big") + if err != nil { + return nil, fmt.Errorf("DMG: footer parse failed") + } + + if dmgFooter["header_size"] != footerSize { + return nil, fmt.Errorf("DMG: header size mismatch (got %d, expected %d)", dmgFooter["header_size"], footerSize) + } + + dataLength := dmgFooter["data_fork_length"] + xmlLength := dmgFooter["xml_length"] + + if dataLength+xmlLength > offset { + return nil, fmt.Errorf("DMG: data + xml length exceeds offset") + } + + // Search for XML property list with blkx key + xmlSig := []byte("blkx" + const minXMLLen = 1024 + + xmlOffset := -1 + searchStart := 0 + for { + idx := bytes.Index(data[searchStart:], xmlSig) + if idx < 0 { + break + } + xmlStart := searchStart + idx + xmlEnd := xmlStart + minXMLLen + if xmlEnd > len(data) { + searchStart = xmlStart + len(xmlSig) + continue + } + xmlStr := string(data[xmlStart:xmlEnd]) + if strings.Contains(xmlStr, blkxKey) { + xmlOffset = xmlStart + break + } + searchStart = xmlStart + len(xmlSig) + } + + if xmlOffset < 0 { + return nil, fmt.Errorf("DMG: XML property list not found") + } + + if xmlOffset < dataLength { + return nil, fmt.Errorf("DMG: XML data before image data") + } + + result.Size = offset + footerSize + result.Offset = xmlOffset - dataLength + result.Description = fmt.Sprintf("%s, total size: %d bytes", result.Description, result.Size) + return result, nil +} + +// ---- QNX IFS ---- + +const DescriptionQNXIFS = "QNX IFS image" + +// qnxIFSMagic returns magic bytes for QNX IFS images. +func qnxIFSMagic() [][]byte { + return [][]byte{ + {0xEB, 0x7E, 0xFF, 0x00, 0x01, 0x00}, + } +} + +// qnxIFSParser validates QNX IFS image signatures. +func qnxIFSParser(data []byte, offset int) (*SignatureResult, error) { + result := &SignatureResult{ + Offset: offset, + Description: DescriptionQNXIFS, + } + + ifsFields := []StructField{ + {"magic", "u32"}, + {"version", "u16"}, + {"flags1", "u8"}, + {"flags2", "u8"}, + {"header_size", "u16"}, + {"machine", "u16"}, + {"startup_vaddr", "u32"}, + {"paddr_bias", "u32"}, + {"image_paddr", "u32"}, + {"ram_paddr", "u32"}, + {"ram_size", "u32"}, + {"startup_size", "u32"}, + {"stored_size", "u32"}, + {"imagefs_paddr", "u32"}, + {"imagefs_size", "u32"}, + {"preboot_size", "u16"}, + {"zero_0", "u16"}, + {"zero_1", "u32"}, + {"zero_2", "u32"}, + {"zero_3", "u32"}, + } + + if offset+StructSize(ifsFields) > len(data) { + return nil, fmt.Errorf("QNX IFS: insufficient data") + } + + ifsHeader, err := Parse(data[offset:], ifsFields, "little") + if err != nil { + return nil, fmt.Errorf("QNX IFS: parse failed") + } + + if ifsHeader["flags2"] != 0 { + return nil, fmt.Errorf("QNX IFS: flags2 not zero") + } + if ifsHeader["zero_0"] != 0 || ifsHeader["zero_1"] != 0 || ifsHeader["zero_2"] != 0 || ifsHeader["zero_3"] != 0 { + return nil, fmt.Errorf("QNX IFS: zero fields not zero") + } + + totalSize := ifsHeader["stored_size"] + availableData := len(data) - offset + if totalSize > availableData { + return nil, fmt.Errorf("QNX IFS: total size exceeds available data") + } + + result.Size = totalSize + result.Description = fmt.Sprintf("%s, total size: %d bytes", result.Description, result.Size) + return result, nil +} diff --git a/internal/leviathan/binsnoop/types.go b/internal/leviathan/binsnoop/types.go new file mode 100644 index 0000000..10d043c --- /dev/null +++ b/internal/leviathan/binsnoop/types.go @@ -0,0 +1,117 @@ +package binsnoop + +import ( + "crypto/rand" + "fmt" + "sort" +) + +// Confidence levels for SignatureResult +const ( + ConfidenceLow uint8 = 0 + ConfidenceMedium uint8 = 128 + ConfidenceHigh uint8 = 250 +) + +// SignatureResult describes a validated file signature found during scanning. +type SignatureResult struct { + Offset int `json:"offset"` + ID string `json:"id"` + Size int `json:"size"` + Name string `json:"name"` + Confidence uint8 `json:"confidence"` + Description string `json:"description"` + AlwaysDisplay bool `json:"always_display"` + ExtractionDeclined bool `json:"extraction_declined"` + PreferredExtractor *Extractor `json:"-"` +} + +// DeepScanResult is a validated signature retained by a non-suppressing scan. +// ParentOffset is -1 for a top-level hit. +type DeepScanResult struct { + SignatureResult + End int `json:"end"` + Depth int `json:"depth"` + ParentOffset int `json:"parent_offset"` + Overlap bool `json:"overlap"` +} + +// DeepScan is a bounded nested-signature analysis result. +type DeepScan struct { + Results []DeepScanResult `json:"results"` + Truncated bool `json:"truncated"` +} + +// SignatureParser validates a magic byte match and returns a SignatureResult or error. +type SignatureParser func(data []byte, offset int) (*SignatureResult, error) + +// Signature defines a file signature to search for. +type Signature struct { + Name string + Short bool + Magic [][]byte + MagicOffset int + Description string + AlwaysDisplay bool + Parser SignatureParser + Extractor *Extractor +} + +// ExtractorType defines the type of extractor. +type ExtractorType int + +const ( + ExtractorTypeNone ExtractorType = iota + ExtractorTypeInternal + ExtractorTypeExternal +) + +// SourceFilePlaceholder is replaced at runtime with the actual file path. +const SourceFilePlaceholder = "%e" + +// InternalExtractorFunc is the function signature for built-in extractors. +type InternalExtractorFunc func(data []byte, offset int, outputDir string) *ExtractionResult + +// Extractor describes how to extract a file type. +type Extractor struct { + Type ExtractorType + Command string + Function InternalExtractorFunc + Extension string + Arguments []string + ExitCodes []int + DoNotRecurse bool +} + +// ExtractionResult stores information about a completed extraction. +type ExtractionResult struct { + Size *int `json:"size,omitempty"` + Success bool `json:"success"` + ExtractorName string `json:"extractor"` + DoNotRecurse bool `json:"do_not_recurse"` + OutputDirectory string `json:"output_directory"` +} + +// AnalysisResults contains all results from analysing a file or buffer. +type AnalysisResults struct { + FilePath string `json:"file_path"` + FileMap []SignatureResult `json:"file_map"` + Extractions map[string]*ExtractionResult `json:"extractions"` +} + +// sortSignatureResults sorts by offset for proper file mapping. +func sortSignatureResults(results []SignatureResult) { + sort.Slice(results, func(i, j int) bool { + return results[i].Offset < results[j].Offset + }) +} + +// newUUID generates a v4 UUID string with no external dependencies. +func newUUID() string { + b := make([]byte, 16) + _, _ = rand.Read(b) + b[6] = (b[6] & 0x0f) | 0x40 + b[8] = (b[8] & 0x3f) | 0x80 + return fmt.Sprintf("%08x-%04x-%04x-%04x-%012x", + b[0:4], b[4:6], b[6:8], b[8:10], b[10:16]) +} diff --git a/internal/leviathan/disasm/aarch64.go b/internal/leviathan/disasm/aarch64.go new file mode 100644 index 0000000..e67495c --- /dev/null +++ b/internal/leviathan/disasm/aarch64.go @@ -0,0 +1,288 @@ +package disasm + +import ( + "encoding/binary" + "fmt" +) + +// diAArch64 is a heuristic AArch64 (ARMv8 A64) decoder. It recognises a +// subset of instructions commonly seen in disassembly and falls back to +// ".word 0x…" for everything else. 32-bit fixed-width little-endian. +func diAArch64(buf []byte, base int64, maxInstr int) []map[string]any { + out := []map[string]any{} + i := 0 + for i+4 <= len(buf) && len(out) < maxInstr { + w := binary.LittleEndian.Uint32(buf[i : i+4]) + mn, ops, conf := decodeA64(w, base+int64(i)) + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+4]), + "mnemonic": mn, + "operands": ops, + "confident": conf, + }) + i += 4 + } + // Tail bytes (not multiple of 4). + for i < len(buf) && len(out) < maxInstr { + end := i + 1 + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i:end]), + "mnemonic": "db", + "operands": hexCommaList(buf[i:end]), + "confident": false, + }) + i++ + } + return out +} + +func decodeA64(w uint32, pc int64) (string, string, bool) { + // NOP + if w == 0xd503201f { + return "nop", "", true + } + // RET Xn (default x30) + if w&0xfffffc1f == 0xd65f0000 { + rn := (w >> 5) & 0x1f + if rn == 30 { + return "ret", "", true + } + return "ret", regX(rn), true + } + // BR Xn + if w&0xfffffc1f == 0xd61f0000 { + return "br", regX((w >> 5) & 0x1f), true + } + // BLR Xn + if w&0xfffffc1f == 0xd63f0000 { + return "blr", regX((w >> 5) & 0x1f), true + } + // B / BL (unconditional branch, imm26 * 4, signed) + if w&0x7c000000 == 0x14000000 { + imm := signExtend64(uint64(w&0x03ffffff)<<2, 28) + target := pc + int64(imm) + if w&0x80000000 != 0 { + return "bl", fmt.Sprintf("0x%x", target), true + } + return "b", fmt.Sprintf("0x%x", target), true + } + // B.cond (conditional branch, imm19*4) + if w&0xff000010 == 0x54000000 { + cond := int(w & 0xf) + imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21) + return "b." + condName(cond), fmt.Sprintf("0x%x", pc+int64(imm)), true + } + // CBZ / CBNZ + if w&0x7e000000 == 0x34000000 { + sf := (w >> 31) & 1 + op := (w >> 24) & 1 + rt := w & 0x1f + imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21) + mn := "cbz" + if op == 1 { + mn = "cbnz" + } + r := regX(rt) + if sf == 0 { + r = regW(rt) + } + return mn, fmt.Sprintf("%s, 0x%x", r, pc+int64(imm)), true + } + // MOVZ / MOVN / MOVK (wide immediate) + if w&0x1f800000 == 0x12800000 { + sf := (w >> 31) & 1 + opc := (w >> 29) & 0x3 + hw := (w >> 21) & 0x3 + imm16 := (w >> 5) & 0xffff + rd := w & 0x1f + mn := "movn" + switch opc { + case 0b10: + mn = "movz" + case 0b11: + mn = "movk" + } + shift := hw * 16 + r := regX(rd) + if sf == 0 { + r = regW(rd) + } + if shift == 0 { + return mn, fmt.Sprintf("%s, #0x%x", r, imm16), true + } + return mn, fmt.Sprintf("%s, #0x%x, lsl #%d", r, imm16, shift), true + } + // ADR / ADRP + if w&0x1f000000 == 0x10000000 { + page := (w >> 31) & 1 + rd := w & 0x1f + immlo := uint64((w >> 29) & 0x3) + immhi := uint64((w >> 5) & 0x7ffff) + imm := (immhi << 2) | immlo + mn := "adr" + shift := uint(21) + signed := signExtend64(imm, shift) + if page == 1 { + mn = "adrp" + signed = signExtend64(imm<<12, 33) + // adrp target is page-aligned: pc & ~0xFFF + return mn, fmt.Sprintf("%s, 0x%x", regX(rd), (pc & ^int64(0xfff))+int64(signed)), true + } + return mn, fmt.Sprintf("%s, 0x%x", regX(rd), pc+int64(signed)), true + } + // ADD / SUB (immediate) + if w&0x1f000000 == 0x11000000 { + sf := (w >> 31) & 1 + op := (w >> 30) & 1 + sh := (w >> 22) & 1 + imm12 := (w >> 10) & 0xfff + rn := (w >> 5) & 0x1f + rd := w & 0x1f + mn := "add" + if op == 1 { + mn = "sub" + } + lhs := regX(rd) + mid := regX(rn) + if sf == 0 { + lhs = regW(rd) + mid = regW(rn) + } + if sh == 1 { + return mn, fmt.Sprintf("%s, %s, #0x%x, lsl #12", lhs, mid, imm12), true + } + return mn, fmt.Sprintf("%s, %s, #0x%x", lhs, mid, imm12), true + } + // LDR / STR (unsigned offset, 32/64-bit int) + if w&0x3fc00000 == 0x39400000 || w&0x3fc00000 == 0x39000000 { + size := (w >> 30) & 0x3 + l := (w >> 22) & 1 + imm12 := (w >> 10) & 0xfff + rn := (w >> 5) & 0x1f + rt := w & 0x1f + var mn string + if l == 1 { + mn = "ldr" + } else { + mn = "str" + } + scale := int(1 << size) + imm := int(imm12) * scale + var reg string + if size == 3 { + reg = regX(rt) + } else { + reg = regW(rt) + } + base := regXorSP(rn) + if imm == 0 { + return mn, fmt.Sprintf("%s, [%s]", reg, base), true + } + return mn, fmt.Sprintf("%s, [%s, #0x%x]", reg, base, imm), true + } + // LDR (literal / PC-rel) + if w&0xbf000000 == 0x18000000 { + sf := (w >> 30) & 1 + imm := signExtend64(uint64((w>>5)&0x7ffff)<<2, 21) + rt := w & 0x1f + reg := regW(rt) + if sf == 1 { + reg = regX(rt) + } + return "ldr", fmt.Sprintf("%s, 0x%x", reg, pc+int64(imm)), true + } + // ORR / AND / EOR (shifted reg) — simplified detection, no shifts reported. + if w&0x7f200000 == 0x2a000000 { // ORR (32/64) w/ shift=0 + sf := (w >> 31) & 1 + rm := (w >> 16) & 0x1f + rn := (w >> 5) & 0x1f + rd := w & 0x1f + // MOV (register) alias: ORR Xd, XZR, Xm → MOV Xd, Xm (when Rn==31 and shift==0) + if rn == 31 && ((w>>10)&0x3f) == 0 && ((w>>22)&0x3) == 0 { + if sf == 1 { + return "mov", fmt.Sprintf("%s, %s", regX(rd), regX(rm)), true + } + return "mov", fmt.Sprintf("%s, %s", regW(rd), regW(rm)), true + } + if sf == 1 { + return "orr", fmt.Sprintf("%s, %s, %s", regX(rd), regX(rn), regX(rm)), true + } + return "orr", fmt.Sprintf("%s, %s, %s", regW(rd), regW(rn), regW(rm)), true + } + // SVC #imm16 + if w&0xffe0001f == 0xd4000001 { + imm16 := (w >> 5) & 0xffff + return "svc", fmt.Sprintf("#0x%x", imm16), true + } + // BRK #imm16 + if w&0xffe0001f == 0xd4200000 { + imm16 := (w >> 5) & 0xffff + return "brk", fmt.Sprintf("#0x%x", imm16), true + } + // HINT family → treat common ones + if w&0xfffff01f == 0xd503201f { + op := (w >> 5) & 0x7f + switch op { + case 0: + return "nop", "", true + case 1: + return "yield", "", true + case 2: + return "wfe", "", true + case 3: + return "wfi", "", true + case 4: + return "sev", "", true + case 5: + return "sevl", "", true + } + return "hint", fmt.Sprintf("#0x%x", op), true + } + // Fallback + return ".word", fmt.Sprintf("0x%08x", w), false +} + +func regX(r uint32) string { + r &= 0x1f + if r == 31 { + return "xzr" + } + return fmt.Sprintf("x%d", r) +} +func regW(r uint32) string { + r &= 0x1f + if r == 31 { + return "wzr" + } + return fmt.Sprintf("w%d", r) +} + +// regXorSP is used for the "Rn" slot in memory ops where 31 means SP, not XZR. +func regXorSP(r uint32) string { + r &= 0x1f + if r == 31 { + return "sp" + } + return fmt.Sprintf("x%d", r) +} + +func condName(c int) string { + names := []string{ + "eq", "ne", "cs", "cc", "mi", "pl", "vs", "vc", + "hi", "ls", "ge", "lt", "gt", "le", "al", "nv", + } + if c >= 0 && c < 16 { + return names[c] + } + return fmt.Sprintf("c%d", c) +} + +func signExtend64(v uint64, bits uint) int64 { + mask := uint64(1) << (bits - 1) + if v&mask != 0 { + v |= ^uint64(0) << bits + } + return int64(v) +} diff --git a/internal/leviathan/disasm/arm32.go b/internal/leviathan/disasm/arm32.go new file mode 100644 index 0000000..f4518bc --- /dev/null +++ b/internal/leviathan/disasm/arm32.go @@ -0,0 +1,209 @@ +package disasm + +import ( + "encoding/binary" + "fmt" + "strings" +) + +// diARM32 is a heuristic ARM state decoder for 32-bit little-endian ARM +// instructions. Thumb/T32 is intentionally not decoded here. +func diARM32(buf []byte, base int64, maxInstr int) []map[string]any { + out := []map[string]any{} + i := 0 + for i+4 <= len(buf) && len(out) < maxInstr { + w := binary.LittleEndian.Uint32(buf[i : i+4]) + mn, ops, conf := decodeARM32(w, base+int64(i)) + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+4]), + "mnemonic": mn, + "operands": ops, + "confident": conf, + }) + i += 4 + } + for i < len(buf) && len(out) < maxInstr { + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+1]), + "mnemonic": "db", + "operands": hexCommaList(buf[i : i+1]), + "confident": false, + }) + i++ + } + return out +} + +func decodeARM32(w uint32, pc int64) (string, string, bool) { + cond := (w >> 28) & 0xf + suf := armCondSuffix(cond) + if cond == 0xf { + suf = ".nv" + } + // BX Rn + if w&0x0ffffff0 == 0x012fff10 { + return "bx" + suf, armReg(w & 0xf), true + } + // SVC #imm24 + if w&0x0f000000 == 0x0f000000 { + return "svc" + suf, fmt.Sprintf("#0x%x", w&0x00ffffff), true + } + // B / BL, offset is sign-extended imm24 << 2 from PC+8. + if w&0x0e000000 == 0x0a000000 { + imm := signExtend64(uint64(w&0x00ffffff)<<2, 26) + target := pc + 8 + int64(imm) + if w&0x01000000 != 0 { + return "bl" + suf, fmt.Sprintf("0x%x", target), true + } + return "b" + suf, fmt.Sprintf("0x%x", target), true + } + // PUSH/POP common aliases. + if w&0xffff0000 == 0xe92d0000 { + return "push", armRegList(w & 0xffff), true + } + if w&0xffff0000 == 0xe8bd0000 { + return "pop", armRegList(w & 0xffff), true + } + // LDR/STR immediate offset. + if w&0x0c000000 == 0x04000000 { + p := (w >> 24) & 1 + u := (w >> 23) & 1 + b := (w >> 22) & 1 + wb := (w >> 21) & 1 + l := (w >> 20) & 1 + rn := (w >> 16) & 0xf + rd := (w >> 12) & 0xf + imm := w & 0xfff + mn := "str" + if l == 1 { + mn = "ldr" + } + if b == 1 { + mn += "b" + } + mn += suf + sign := "" + if u == 0 { + sign = "-" + } + addr := "" + switch { + case p == 1 && imm == 0 && wb == 0: + addr = fmt.Sprintf("[%s]", armReg(rn)) + case p == 1: + addr = fmt.Sprintf("[%s, #%s0x%x]", armReg(rn), sign, imm) + if wb == 1 { + addr += "!" + } + default: + addr = fmt.Sprintf("[%s], #%s0x%x", armReg(rn), sign, imm) + } + return mn, fmt.Sprintf("%s, %s", armReg(rd), addr), true + } + // Data processing immediate/register subset. + if w&0x0c000000 == 0x00000000 || w&0x0e000000 == 0x02000000 { + immediate := w&0x02000000 != 0 + opcode := (w >> 21) & 0xf + s := (w >> 20) & 1 + rn := (w >> 16) & 0xf + rd := (w >> 12) & 0xf + mn := armDataProcName(opcode) + if mn == "" { + return ".word", fmt.Sprintf("0x%08x", w), false + } + mn += suf + if s == 1 && opcode != 8 && opcode != 9 && opcode != 10 && opcode != 11 { + mn += "s" + } + op2 := "" + if immediate { + imm8 := w & 0xff + rot := ((w >> 8) & 0xf) * 2 + imm := (imm8 >> rot) | (imm8 << ((32 - rot) & 31)) + op2 = fmt.Sprintf("#0x%x", imm) + } else { + rm := w & 0xf + shift := (w >> 4) & 0xff + if shift == 0 { + op2 = armReg(rm) + } else { + op2 = fmt.Sprintf("%s, %s", armReg(rm), armShift(shift)) + } + } + switch opcode { + case 8, 9, 10, 11: // tst/teq/cmp/cmn + return mn, fmt.Sprintf("%s, %s", armReg(rn), op2), true + case 13, 15: // mov/mvn + return mn, fmt.Sprintf("%s, %s", armReg(rd), op2), true + default: + return mn, fmt.Sprintf("%s, %s, %s", armReg(rd), armReg(rn), op2), true + } + } + return ".word", fmt.Sprintf("0x%08x", w), false +} + +func armCondSuffix(cond uint32) string { + return [...]string{".eq", ".ne", ".cs", ".cc", ".mi", ".pl", ".vs", ".vc", ".hi", ".ls", ".ge", ".lt", ".gt", ".le", "", ""}[cond&0xf] +} + +func armDataProcName(op uint32) string { + return [...]string{"and", "eor", "sub", "rsb", "add", "adc", "sbc", "rsc", "tst", "teq", "cmp", "cmn", "orr", "mov", "bic", "mvn"}[op&0xf] +} + +func armReg(r uint32) string { + switch r & 0xf { + case 13: + return "sp" + case 14: + return "lr" + case 15: + return "pc" + default: + return fmt.Sprintf("r%d", r&0xf) + } +} + +func armRegList(mask uint32) string { + regs := []string{} + start := -1 + flush := func(end int) { + if start < 0 { + return + } + if start == end { + regs = append(regs, armReg(uint32(start))) + } else { + regs = append(regs, fmt.Sprintf("%s-%s", armReg(uint32(start)), armReg(uint32(end)))) + } + start = -1 + } + for i := 0; i < 16; i++ { + if mask&(1<> 1) & 0x3 + imm := (v >> 3) & 0x1f + names := [...]string{"lsl", "lsr", "asr", "ror"} + if imm == 0 { + if typ == 0 { + return "lsl #0" + } + if typ == 3 { + return "rrx" + } + imm = 32 + } + return fmt.Sprintf("%s #%d", names[typ], imm) +} diff --git a/internal/leviathan/disasm/cfg.go b/internal/leviathan/disasm/cfg.go new file mode 100644 index 0000000..ec0625d --- /dev/null +++ b/internal/leviathan/disasm/cfg.go @@ -0,0 +1,541 @@ +package disasm + +import ( + "sort" + "strconv" + "strings" +) + +// cfg.go builds control-flow graphs and cross-references on top of the +// per-architecture decoders. It performs a recursive-descent sweep from an +// entry point: it follows direct branches and the fall-through after calls +// (calls leave the function, so their targets are recorded as xrefs rather +// than followed), splits the decoded instructions into basic blocks, and +// records every direct code/data reference it can resolve statically. + +// Block is one basic block: a maximal straight-line instruction run. +type Block struct { + Start int64 `json:"start"` + End int64 `json:"end"` // address after the last instruction + Instrs []map[string]any `json:"instrs"` + Succs []int64 `json:"succs"` // successor block start addresses + Kind string `json:"kind"` // ret | jmp | cond | call | fallthrough +} + +// Xref is a single static reference from one address to another. +type Xref struct { + From int64 `json:"from"` + To int64 `json:"to"` + Kind string `json:"kind"` // call | jump | data +} + +// CFG is the analysis result for one entry point. +type CFG struct { + Entry int64 `json:"entry"` + Arch string `json:"arch"` + Blocks []*Block `json:"blocks"` + Calls []int64 `json:"calls"` // distinct call targets (in or out of range) + Xrefs []Xref `json:"xrefs"` // all resolved references + Unknown int `json:"unknown"` // indirect/unresolved branch+call count +} + +// Function is a discovered function and its structural metrics. +type Function struct { + Name string `json:"name"` + Entry int64 `json:"entry"` + End int64 `json:"end"` + Blocks []*Block `json:"blocks"` + Instructions int `json:"instructions"` + Complexity int `json:"complexity"` + Calls []int64 `json:"calls"` + CalledBy []int64 `json:"calledBy"` + Unknown int `json:"unknown"` +} + +// CallEdge is one direct call from a discovered function. +type CallEdge struct { + Caller int64 `json:"caller"` + CallSite int64 `json:"callSite"` + Callee int64 `json:"callee"` + External bool `json:"external"` +} + +// Seed is a known or suspected function entry supplied by executable metadata. +type Seed struct { + Name string `json:"name"` + Entry int64 `json:"entry"` +} + +// Program is a bounded recursive function-discovery result for one byte range. +type Program struct { + Entry int64 `json:"entry"` + Base int64 `json:"base"` + End int64 `json:"end"` + Arch string `json:"arch"` + Functions []*Function `json:"functions"` + Edges []CallEdge `json:"edges"` + ExternalCalls []int64 `json:"externalCalls"` + Instructions int `json:"instructions"` + Unknown int `json:"unknown"` + Truncated bool `json:"truncated"` +} + +type rawInstr struct { + addr int64 + length int + mnem string + class string + target int64 + hasTgt bool + data int64 + hasDat bool + m map[string]any +} + +// BuildCFG decodes from entry (within buf, which starts at address base) and +// returns the control-flow graph plus discovered references. +func BuildCFG(buf []byte, base int64, arch string, entry int64, maxInstr int) *CFG { + if maxInstr <= 0 { + maxInstr = 8192 + } + cfg := &CFG{Entry: entry, Arch: arch} + decoded := map[int64]*rawInstr{} + inRange := func(a int64) bool { return a >= base && a < base+int64(len(buf)) } + + var stack []int64 + push := func(a int64) { + if inRange(a) { + stack = append(stack, a) + } + } + push(entry) + + for len(stack) > 0 && len(decoded) < maxInstr { + addr := stack[len(stack)-1] + stack = stack[:len(stack)-1] + for inRange(addr) && len(decoded) < maxInstr { + if _, seen := decoded[addr]; seen { + break + } + ri := decodeAt(buf, addr, base, arch) + if ri == nil { + break + } + decoded[addr] = ri + next := addr + int64(ri.length) + switch ri.class { + case "ret": + addr = -1 + case "jmp": + if ri.hasTgt { + addr = ri.target + } else { + cfg.Unknown++ + addr = -1 + } + case "cond": + if ri.hasTgt { + push(ri.target) + } else { + cfg.Unknown++ + } + addr = next + case "call": + if !ri.hasTgt { + cfg.Unknown++ + } + addr = next + default: + addr = next + } + if addr < 0 { + break + } + } + } + + // Collect references and call targets. + callSet := map[int64]bool{} + for _, ri := range decoded { + if ri.hasTgt { + switch ri.class { + case "call": + cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.target, Kind: "call"}) + if !callSet[ri.target] { + callSet[ri.target] = true + cfg.Calls = append(cfg.Calls, ri.target) + } + case "jmp", "cond": + cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.target, Kind: "jump"}) + } + } + if ri.hasDat { + cfg.Xrefs = append(cfg.Xrefs, Xref{From: ri.addr, To: ri.data, Kind: "data"}) + } + } + sort.Slice(cfg.Calls, func(i, j int) bool { return cfg.Calls[i] < cfg.Calls[j] }) + sort.Slice(cfg.Xrefs, func(i, j int) bool { + if cfg.Xrefs[i].From != cfg.Xrefs[j].From { + return cfg.Xrefs[i].From < cfg.Xrefs[j].From + } + return cfg.Xrefs[i].To < cfg.Xrefs[j].To + }) + + cfg.Blocks = buildBlocks(decoded, inRange) + return cfg +} + +// DiscoverFunctions recursively follows direct calls from entry, building a +// bounded call graph over buf. Indirect calls are counted but cannot be +// discovered statically. maxInstr is a total budget across all functions. +func DiscoverFunctions(buf []byte, base int64, arch string, entry int64, maxFunctions, maxInstr int) *Program { + return DiscoverFunctionsFromSeeds(buf, base, arch, []Seed{{Entry: entry}}, maxFunctions, maxInstr) +} + +// DiscoverFunctionsFromSeeds analyzes named and unnamed roots plus every +// direct in-range call target reached from them. +func DiscoverFunctionsFromSeeds(buf []byte, base int64, arch string, seeds []Seed, maxFunctions, maxInstr int) *Program { + if maxFunctions <= 0 || maxFunctions > 4096 { + maxFunctions = 256 + } + if maxInstr <= 0 || maxInstr > 1_000_000 { + maxInstr = 65_536 + } + end := base + int64(len(buf)) + entry := base + if len(seeds) > 0 { + entry = seeds[0].Entry + } + program := &Program{Entry: entry, Base: base, End: end, Arch: arch} + inRange := func(addr int64) bool { return addr >= base && addr < end } + queue := make([]int64, 0, len(seeds)) + queued := map[int64]bool{} + names := map[int64]string{} + for _, seed := range seeds { + if !inRange(seed.Entry) { + continue + } + if !queued[seed.Entry] { + queued[seed.Entry] = true + queue = append(queue, seed.Entry) + } + if seed.Name != "" { + names[seed.Entry] = seed.Name + } + } + if len(queue) == 0 { + program.Truncated = true + return program + } + + discovered := map[int64]*Function{} + callerSets := map[int64]map[int64]bool{} + externalSet := map[int64]bool{} + + for len(queue) > 0 && len(discovered) < maxFunctions && program.Instructions < maxInstr { + fnEntry := queue[0] + queue = queue[1:] + if discovered[fnEntry] != nil { + continue + } + remaining := maxInstr - program.Instructions + perFunction := remaining + if perFunction > 8192 { + perFunction = 8192 + } + cfg := BuildCFG(buf, base, arch, fnEntry, perFunction) + fn := summarizeFunction(cfg, names[fnEntry]) + discovered[fnEntry] = fn + program.Functions = append(program.Functions, fn) + program.Instructions += fn.Instructions + program.Unknown += fn.Unknown + + for _, xref := range cfg.Xrefs { + if xref.Kind != "call" { + continue + } + external := !inRange(xref.To) + program.Edges = append(program.Edges, CallEdge{ + Caller: fnEntry, CallSite: xref.From, Callee: xref.To, External: external, + }) + if external { + externalSet[xref.To] = true + continue + } + if callerSets[xref.To] == nil { + callerSets[xref.To] = map[int64]bool{} + } + callerSets[xref.To][fnEntry] = true + if !queued[xref.To] { + queued[xref.To] = true + queue = append(queue, xref.To) + } + } + } + + if len(queue) > 0 || program.Instructions >= maxInstr { + program.Truncated = true + } + for _, fn := range program.Functions { + for caller := range callerSets[fn.Entry] { + fn.CalledBy = append(fn.CalledBy, caller) + } + sort.Slice(fn.CalledBy, func(i, j int) bool { return fn.CalledBy[i] < fn.CalledBy[j] }) + } + for target := range externalSet { + program.ExternalCalls = append(program.ExternalCalls, target) + } + sort.Slice(program.Functions, func(i, j int) bool { return program.Functions[i].Entry < program.Functions[j].Entry }) + sort.Slice(program.Edges, func(i, j int) bool { + if program.Edges[i].Caller != program.Edges[j].Caller { + return program.Edges[i].Caller < program.Edges[j].Caller + } + if program.Edges[i].CallSite != program.Edges[j].CallSite { + return program.Edges[i].CallSite < program.Edges[j].CallSite + } + return program.Edges[i].Callee < program.Edges[j].Callee + }) + sort.Slice(program.ExternalCalls, func(i, j int) bool { return program.ExternalCalls[i] < program.ExternalCalls[j] }) + return program +} + +func summarizeFunction(cfg *CFG, name string) *Function { + if name == "" { + name = unknownFunctionName(cfg.Entry) + } + fn := &Function{ + Name: name, Entry: cfg.Entry, Blocks: cfg.Blocks, + Calls: append([]int64(nil), cfg.Calls...), Unknown: cfg.Unknown, + } + edges := 0 + for _, block := range cfg.Blocks { + if block.End > fn.End { + fn.End = block.End + } + fn.Instructions += len(block.Instrs) + edges += len(block.Succs) + } + fn.Complexity = edges - len(cfg.Blocks) + 2 + if fn.Complexity < 1 { + fn.Complexity = 1 + } + return fn +} + +func unknownFunctionName(entry int64) string { + return "sub_" + strconv.FormatInt(entry, 16) +} + +func buildBlocks(decoded map[int64]*rawInstr, inRange func(int64) bool) []*Block { + if len(decoded) == 0 { + return nil + } + addrs := make([]int64, 0, len(decoded)) + for a := range decoded { + addrs = append(addrs, a) + } + sort.Slice(addrs, func(i, j int) bool { return addrs[i] < addrs[j] }) + + // Leaders: first instr, every branch target, and the instruction after any + // branch/call/ret. + leaders := map[int64]bool{addrs[0]: true} + for _, a := range addrs { + ri := decoded[a] + next := a + int64(ri.length) + switch ri.class { + case "ret": + if _, ok := decoded[next]; ok { + leaders[next] = true + } + case "jmp": + if ri.hasTgt { + if _, ok := decoded[ri.target]; ok { + leaders[ri.target] = true + } + } + if _, ok := decoded[next]; ok { + leaders[next] = true + } + case "cond", "call": + if ri.hasTgt { + if _, ok := decoded[ri.target]; ok { + leaders[ri.target] = true + } + } + if _, ok := decoded[next]; ok { + leaders[next] = true + } + } + } + + var blocks []*Block + var cur *Block + flush := func() { + if cur != nil { + blocks = append(blocks, cur) + cur = nil + } + } + for idx, a := range addrs { + ri := decoded[a] + if leaders[a] { + flush() + cur = &Block{Start: a} + } + if cur == nil { + cur = &Block{Start: a} + } + cur.Instrs = append(cur.Instrs, ri.m) + next := a + int64(ri.length) + cur.End = next + + // Decide whether this instruction terminates the block. + isLast := idx == len(addrs)-1 + nextIsLeader := false + if !isLast { + if leaders[addrs[idx+1]] { + nextIsLeader = true + } + } + terminator := ri.class == "ret" || ri.class == "jmp" || ri.class == "cond" + if terminator || nextIsLeader || isLast { + cur.Kind = blockKind(ri.class) + switch ri.class { + case "ret": + // no successors + case "jmp": + if ri.hasTgt && inRange(ri.target) { + cur.Succs = []int64{ri.target} + } + case "cond": + if ri.hasTgt && inRange(ri.target) { + cur.Succs = append(cur.Succs, ri.target) + } + if _, ok := decoded[next]; ok { + cur.Succs = append(cur.Succs, next) + } + default: + if _, ok := decoded[next]; ok { + cur.Succs = []int64{next} + } + } + flush() + } + } + flush() + return blocks +} + +func blockKind(class string) string { + switch class { + case "ret", "jmp", "cond", "call": + return class + default: + return "fallthrough" + } +} + +// decodeAt decodes a single instruction at address addr (buf starts at base). +func decodeAt(buf []byte, addr, base int64, arch string) *rawInstr { + off := int(addr - base) + if off < 0 || off >= len(buf) { + return nil + } + end := off + 16 // max x86 instruction length; fixed-width arches need <= 4 + if end > len(buf) { + end = len(buf) + } + res := Disassemble(buf[off:end], addr, arch, 1) + if len(res) == 0 { + return nil + } + m := res[0] + ri := &rawInstr{addr: addr, m: m} + ri.length = byteCount(m["bytes"].(string)) + if ri.length <= 0 { + ri.length = 1 + } + ri.mnem, _ = m["mnemonic"].(string) + ops, _ := m["operands"].(string) + ri.class = classify(ri.mnem) + if t, ok := directTarget(ops); ok { + ri.target, ri.hasTgt = t, true + } + if dataReferencing(ri.class) { + if d, ok := dataTarget(ops); ok { + ri.data, ri.hasDat = d, true + } + } + return ri +} + +func byteCount(s string) int { + if s == "" { + return 0 + } + return len(strings.Fields(s)) +} + +func classify(mnem string) string { + switch mnem { + case "ret", "retf", "iret": + return "ret" + case "jmp", "jmpf", "b", "br": + return "jmp" + case "call", "callf", "bl", "blr", "jal", "jalr": + return "call" + case "cbz", "cbnz", "tbz", "tbnz", "loop", "loope", "loopne", "jecxz", "jrcxz": + return "cond" + } + if strings.HasPrefix(mnem, "b.") { // AArch64 conditional branch + return "cond" + } + if strings.HasPrefix(mnem, "j") { // x86 Jcc (jmp already handled above) + return "cond" + } + if strings.HasPrefix(mnem, "beq") || strings.HasPrefix(mnem, "bne") { // MIPS + return "cond" + } + return "normal" +} + +func dataReferencing(class string) bool { + // RIP-relative resolved targets show up as "; 0x.." comments on any + // instruction; only mine them on non-control-flow ops as data xrefs. + return class == "normal" || class == "call" +} + +// directTarget extracts a branch/call target when the final operand is a bare +// absolute address (a direct, statically known target). Register/memory +// indirect operands return false. +func directTarget(operands string) (int64, bool) { + if i := strings.Index(operands, ";"); i >= 0 { + operands = operands[:i] + } + if c := strings.LastIndex(operands, ","); c >= 0 { + operands = operands[c+1:] + } + operands = strings.TrimSpace(operands) + return parseHexAddr(operands) +} + +// dataTarget extracts a resolved address from a "; 0x.." trailing comment +// (e.g. an x86 RIP-relative memory operand). +func dataTarget(operands string) (int64, bool) { + i := strings.Index(operands, ";") + if i < 0 { + return 0, false + } + return parseHexAddr(strings.TrimSpace(operands[i+1:])) +} + +func parseHexAddr(s string) (int64, bool) { + if !strings.HasPrefix(s, "0x") { + return 0, false + } + v, err := strconv.ParseUint(s[2:], 16, 64) + if err != nil { + return 0, false + } + return int64(v), true +} diff --git a/internal/leviathan/disasm/image.go b/internal/leviathan/disasm/image.go new file mode 100644 index 0000000..a9991ce --- /dev/null +++ b/internal/leviathan/disasm/image.go @@ -0,0 +1,74 @@ +package disasm + +import ( + "github.com/andydixon/nexus/internal/leviathan/document" + "github.com/andydixon/nexus/internal/leviathan/parsers" +) + +// ImageAnalysis is a metadata-seeded analysis across file-backed executable +// regions in an ELF or PE image. +type ImageAnalysis struct { + Info *parsers.ExecutableInfo `json:"info"` + Programs []*Program `json:"programs"` + Truncated bool `json:"truncated"` +} + +// DiscoverExecutable analyzes executable regions using header entries, +// symbols and exports as roots. Budgets apply across the whole image. +func DiscoverExecutable(d *document.Document, maxBytes int64, maxFunctions, maxInstr int) (*ImageAnalysis, error) { + info, err := parsers.InspectExecutable(d) + if err != nil { + return nil, err + } + if maxBytes <= 0 || maxBytes > 32<<20 { + maxBytes = 8 << 20 + } + if maxFunctions <= 0 || maxFunctions > 4096 { + maxFunctions = 512 + } + if maxInstr <= 0 || maxInstr > 1_000_000 { + maxInstr = 131_072 + } + analysis := &ImageAnalysis{Info: info} + remainingBytes := maxBytes + remainingFunctions := maxFunctions + remainingInstr := maxInstr + for _, region := range info.Regions { + if remainingBytes <= 0 || remainingFunctions <= 0 || remainingInstr <= 0 { + analysis.Truncated = true + break + } + length := region.Size + if length > 4<<20 { + length = 4 << 20 + analysis.Truncated = true + } + if length > remainingBytes { + length = remainingBytes + analysis.Truncated = true + } + if length <= 0 { + continue + } + buf, err := d.Range(region.Offset, length) + if err != nil { + return nil, err + } + seeds := make([]Seed, 0) + for _, seed := range info.Seeds { + if seed.Offset >= region.Offset && seed.Offset < region.Offset+length { + seeds = append(seeds, Seed{Name: seed.Name, Entry: seed.Offset}) + } + } + if len(seeds) == 0 { + seeds = append(seeds, Seed{Name: region.Name, Entry: region.Offset}) + } + program := DiscoverFunctionsFromSeeds(buf, region.Offset, info.Arch, seeds, remainingFunctions, remainingInstr) + analysis.Programs = append(analysis.Programs, program) + remainingBytes -= length + remainingFunctions -= len(program.Functions) + remainingInstr -= program.Instructions + analysis.Truncated = analysis.Truncated || program.Truncated + } + return analysis, nil +} diff --git a/internal/leviathan/disasm/mips.go b/internal/leviathan/disasm/mips.go new file mode 100644 index 0000000..6f1f6a9 --- /dev/null +++ b/internal/leviathan/disasm/mips.go @@ -0,0 +1,144 @@ +package disasm + +import ( + "encoding/binary" + "fmt" +) + +func diMIPS32(buf []byte, base int64, little bool, maxInstr int) []map[string]any { + out := []map[string]any{} + for i := 0; i+4 <= len(buf) && len(out) < maxInstr; i += 4 { + var w uint32 + if little { + w = binary.LittleEndian.Uint32(buf[i : i+4]) + } else { + w = binary.BigEndian.Uint32(buf[i : i+4]) + } + mn, ops, conf := decodeMIPS32(w, base+int64(i)) + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+4]), + "mnemonic": mn, + "operands": ops, + "confident": conf, + }) + } + for i := len(buf) - len(buf)%4; i < len(buf) && len(out) < maxInstr; i++ { + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+1]), + "mnemonic": "db", + "operands": hexCommaList(buf[i : i+1]), + "confident": false, + }) + } + return out +} + +func decodeMIPS32(w uint32, pc int64) (string, string, bool) { + if w == 0 { + return "nop", "", true + } + op := (w >> 26) & 0x3f + rs := (w >> 21) & 0x1f + rt := (w >> 16) & 0x1f + rd := (w >> 11) & 0x1f + shamt := (w >> 6) & 0x1f + funct := w & 0x3f + imm := uint16(w) + simm := int16(imm) + target26 := w & 0x03ffffff + + switch op { + case 0x00: + switch funct { + case 0x00: + return "sll", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true + case 0x02: + return "srl", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true + case 0x03: + return "sra", fmt.Sprintf("%s, %s, %d", mipsReg(rd), mipsReg(rt), shamt), true + case 0x08: + return "jr", mipsReg(rs), true + case 0x09: + return "jalr", fmt.Sprintf("%s, %s", mipsReg(rd), mipsReg(rs)), true + case 0x0c: + return "syscall", fmt.Sprintf("0x%x", (w>>6)&0xfffff), true + case 0x20: + return "add", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x21: + return "addu", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x22: + return "sub", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x23: + return "subu", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x24: + return "and", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x25: + return "or", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x26: + return "xor", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x27: + return "nor", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + case 0x2a: + return "slt", fmt.Sprintf("%s, %s, %s", mipsReg(rd), mipsReg(rs), mipsReg(rt)), true + } + case 0x02: + return "j", fmt.Sprintf("0x%x", mipsJumpTarget(pc, target26)), true + case 0x03: + return "jal", fmt.Sprintf("0x%x", mipsJumpTarget(pc, target26)), true + case 0x04: + return "beq", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rs), mipsReg(rt), mipsBranchTarget(pc, simm)), true + case 0x05: + return "bne", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rs), mipsReg(rt), mipsBranchTarget(pc, simm)), true + case 0x08: + return "addi", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true + case 0x09: + return "addiu", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true + case 0x0a: + return "slti", fmt.Sprintf("%s, %s, %d", mipsReg(rt), mipsReg(rs), simm), true + case 0x0c: + return "andi", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true + case 0x0d: + return "ori", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true + case 0x0e: + return "xori", fmt.Sprintf("%s, %s, 0x%x", mipsReg(rt), mipsReg(rs), imm), true + case 0x0f: + return "lui", fmt.Sprintf("%s, 0x%x", mipsReg(rt), imm), true + case 0x20: + return "lb", mipsMem(rt, rs, simm), true + case 0x21: + return "lh", mipsMem(rt, rs, simm), true + case 0x23: + return "lw", mipsMem(rt, rs, simm), true + case 0x28: + return "sb", mipsMem(rt, rs, simm), true + case 0x29: + return "sh", mipsMem(rt, rs, simm), true + case 0x2b: + return "sw", mipsMem(rt, rs, simm), true + } + return ".word", fmt.Sprintf("0x%08x", w), false +} + +func mipsJumpTarget(pc int64, target uint32) int64 { + upper := uint64(pc+4) & 0xfffffffff0000000 + return int64(upper | uint64(target<<2)) +} + +func mipsBranchTarget(pc int64, imm int16) int64 { + return pc + 4 + int64(imm)*4 +} + +func mipsMem(rt, rs uint32, off int16) string { + return fmt.Sprintf("%s, %d(%s)", mipsReg(rt), off, mipsReg(rs)) +} + +func mipsReg(r uint32) string { + return [...]string{ + "$zero", "$at", "$v0", "$v1", "$a0", "$a1", "$a2", "$a3", + "$t0", "$t1", "$t2", "$t3", "$t4", "$t5", "$t6", "$t7", + "$s0", "$s1", "$s2", "$s3", "$s4", "$s5", "$s6", "$s7", + "$t8", "$t9", "$k0", "$k1", "$gp", "$sp", "$fp", "$ra", + }[r&0x1f] +} diff --git a/internal/leviathan/disasm/thumb.go b/internal/leviathan/disasm/thumb.go new file mode 100644 index 0000000..5fee9e9 --- /dev/null +++ b/internal/leviathan/disasm/thumb.go @@ -0,0 +1,117 @@ +package disasm + +import ( + "encoding/binary" + "fmt" +) + +func diThumb16(buf []byte, base int64, maxInstr int) []map[string]any { + out := []map[string]any{} + for i := 0; i+2 <= len(buf) && len(out) < maxInstr; i += 2 { + h := binary.LittleEndian.Uint16(buf[i : i+2]) + mn, ops, conf := decodeThumb16(h, base+int64(i)) + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+2]), + "mnemonic": mn, + "operands": ops, + "confident": conf, + }) + } + for i := len(buf) - len(buf)%2; i < len(buf) && len(out) < maxInstr; i++ { + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+1]), + "mnemonic": "db", + "operands": hexCommaList(buf[i : i+1]), + "confident": false, + }) + } + return out +} + +func decodeThumb16(h uint16, pc int64) (string, string, bool) { + switch { + case h == 0xbf00: + return "nop", "", true + case h&0xff87 == 0x4700: + return "bx", thumbHighReg((h >> 3) & 0xf), true + case h&0xff87 == 0x4780: + return "blx", thumbHighReg((h >> 3) & 0xf), true + case h&0xf800 == 0xe000: + imm := signExtend64(uint64(h&0x07ff)<<1, 12) + return "b", fmt.Sprintf("0x%x", pc+4+imm), true + case h&0xf000 == 0xd000 && h&0x0f00 != 0x0f00: + cond := uint32((h >> 8) & 0xf) + imm := signExtend64(uint64(h&0x00ff)<<1, 9) + return "b" + armCondSuffix(cond), fmt.Sprintf("0x%x", pc+4+imm), true + case h&0xff00 == 0xdf00: + return "svc", fmt.Sprintf("#0x%x", h&0x00ff), true + case h&0xfe00 == 0xb400: + mask := uint32(h & 0x00ff) + if h&0x0100 != 0 { + mask |= 1 << 14 + } + return "push", armRegList(mask), true + case h&0xfe00 == 0xbc00: + mask := uint32(h & 0x00ff) + if h&0x0100 != 0 { + mask |= 1 << 15 + } + return "pop", armRegList(mask), true + case h&0xf800 == 0x2000: + rd := uint32((h >> 8) & 0x7) + return "movs", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true + case h&0xf800 == 0x2800: + rn := uint32((h >> 8) & 0x7) + return "cmp", fmt.Sprintf("%s, #0x%x", armReg(rn), h&0x00ff), true + case h&0xf800 == 0x3000: + rd := uint32((h >> 8) & 0x7) + return "adds", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true + case h&0xf800 == 0x3800: + rd := uint32((h >> 8) & 0x7) + return "subs", fmt.Sprintf("%s, #0x%x", armReg(rd), h&0x00ff), true + case h&0xfc00 == 0x4000: + op := (h >> 6) & 0xf + rs := uint32((h >> 3) & 0x7) + rd := uint32(h & 0x7) + names := [...]string{"ands", "eors", "lsls", "lsrs", "asrs", "adcs", "sbcs", "rors", "tst", "rsbs", "cmp", "cmn", "orrs", "muls", "bics", "mvns"} + mn := names[op] + if op == 8 || op == 10 || op == 11 { + return mn, fmt.Sprintf("%s, %s", armReg(rd), armReg(rs)), true + } + return mn, fmt.Sprintf("%s, %s", armReg(rd), armReg(rs)), true + case h&0xfe00 == 0x1800: + rm := uint32((h >> 6) & 0x7) + rn := uint32((h >> 3) & 0x7) + rd := uint32(h & 0x7) + mn := "adds" + if h&0x0200 != 0 { + mn = "subs" + } + return mn, fmt.Sprintf("%s, %s, %s", armReg(rd), armReg(rn), armReg(rm)), true + case h&0xe000 == 0x6000: + imm := uint32((h>>6)&0x1f) * 4 + rn := uint32((h >> 3) & 0x7) + rd := uint32(h & 0x7) + mn := "str" + if h&0x0800 != 0 { + mn = "ldr" + } + return mn, fmt.Sprintf("%s, [%s, #0x%x]", armReg(rd), armReg(rn), imm), true + } + return ".hword", fmt.Sprintf("0x%04x", h), false +} + +func thumbHighReg(r uint16) string { + if r == 13 { + return "sp" + } + if r == 14 { + return "lr" + } + if r == 15 { + return "pc" + } + return fmt.Sprintf("r%d", r) +} diff --git a/internal/leviathan/disasm/x86.go b/internal/leviathan/disasm/x86.go new file mode 100644 index 0000000..90fb21d --- /dev/null +++ b/internal/leviathan/disasm/x86.go @@ -0,0 +1,91 @@ +// Package disasm provides a honest, limited heuristic disassembler. +// It does NOT aim to be a full x86/ARM decoder. It recognises a +// subset of common opcodes so the UI can show plausible disassembly +// for inspection, and is explicit in the output when instructions +// are unknown. +package disasm + +import "fmt" + +// Instr is one decoded instruction. +type Instr struct { + Offset int64 + Bytes []byte + Mnemonic string + Operands string + Confident bool +} + +// Disassemble returns a slice of Instr for buf starting at `base`. +// arch is one of "x86", "x86_64", "arm", "thumb", "aarch64", "mips". +// For unsupported arches we emit a byte stream with "db" entries. +func Disassemble(buf []byte, base int64, arch string, maxInstr int) []map[string]any { + if maxInstr <= 0 { + maxInstr = 4096 + } + switch arch { + case "x86", "x86_64": + return diX86(buf, base, arch == "x86_64", maxInstr) + case "aarch64", "arm64": + return diAArch64(buf, base, maxInstr) + case "arm", "arm32": + return diARM32(buf, base, maxInstr) + case "thumb", "thumb16", "t16": + return diThumb16(buf, base, maxInstr) + case "mips", "mipsbe", "mips32": + return diMIPS32(buf, base, false, maxInstr) + case "mipsle", "mipsel": + return diMIPS32(buf, base, true, maxInstr) + } + return diRaw(buf, base, arch, maxInstr) +} + +// diRaw emits `db` lines 16 bytes at a time for unsupported arches. +func diRaw(buf []byte, base int64, arch string, maxInstr int) []map[string]any { + out := []map[string]any{} + step := 16 + for i := 0; i < len(buf) && len(out) < maxInstr; i += step { + end := i + step + if end > len(buf) { + end = len(buf) + } + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i:end]), + "mnemonic": "db", + "operands": hexCommaList(buf[i:end]), + "confident": false, + "note": arch + " decode not implemented; raw bytes only", + }) + } + return out +} + +// diX86 decodes x86 / x86-64 instructions. The full decoder lives in +// x86_decode.go; this dispatcher only forwards to it. +func diX86(buf []byte, base int64, is64 bool, maxInstr int) []map[string]any { + return decodeX86(buf, base, is64, maxInstr) +} + +func hexStr(b []byte) string { + const h = "0123456789abcdef" + out := make([]byte, 0, len(b)*3) + for i, c := range b { + if i > 0 { + out = append(out, ' ') + } + out = append(out, h[c>>4], h[c&0x0f]) + } + return string(out) +} + +func hexCommaList(b []byte) string { + out := "" + for i, c := range b { + if i > 0 { + out += "," + } + out += fmt.Sprintf("0x%02x", c) + } + return out +} diff --git a/internal/leviathan/disasm/x86_decode.go b/internal/leviathan/disasm/x86_decode.go new file mode 100644 index 0000000..7fc23c6 --- /dev/null +++ b/internal/leviathan/disasm/x86_decode.go @@ -0,0 +1,1172 @@ +package disasm + +import "fmt" + +// x86_decode.go is a from-scratch x86 / x86-64 instruction decoder. Unlike +// the previous heuristic, it tracks legacy prefixes, the REX prefix, and +// fully decodes ModR/M + SIB + displacement + immediate operands so that +// instruction *lengths* are correct (which keeps a linear sweep in sync) and +// operands are rendered in Intel syntax. +// +// Coverage: the complete general-purpose integer ISA (ALU groups, MOV/LEA, +// shifts/rotates, the F6/F7/FE/FF groups, string ops, stack ops, all near +// branches), the two-byte 0F map (Jcc/SETcc/CMOVcc, MOVZX/MOVSX, IMUL, bit +// tests, CMPXCHG/XADD, BSWAP, SHLD/SHRD, system ops) and the common SSE/SSE2 +// moves and arithmetic. VEX/EVEX (AVX), x87 detail and a few rare encodings +// fall back to a one-byte `db` resync, exactly like objdump's "(bad)". + +type x86dec struct { + buf []byte + pos int + base int64 + is64 bool + + opSize int // 16/32/64 effective operand size + addrSize int // 16/32/64 effective address size + has66 bool + seg string + rep string // "rep" / "repne" / "" + lock bool + + rexW, rexR, rexX, rexB bool + hasRex bool + + trunc bool // ran past end of buffer mid-instruction + hasRip bool // a RIP-relative memory operand was decoded + ripDisp int64 // its displacement (resolved once length is known) +} + +func decodeX86(buf []byte, base int64, is64 bool, maxInstr int) []map[string]any { + out := []map[string]any{} + i := 0 + for i < len(buf) && len(out) < maxInstr { + d := &x86dec{buf: buf, pos: i, base: base, is64: is64} + mn, ops, conf := d.decode() + var length int + if d.trunc || mn == "" { + length = 1 + mn = "db" + ops = fmt.Sprintf("0x%02x", buf[i]) + conf = false + } else { + length = d.pos - i + if length <= 0 { + length = 1 + } + if i+length > len(buf) { + length = len(buf) - i + } + if d.hasRip { + abs := base + int64(i+length) + d.ripDisp + ops += fmt.Sprintf(" ; 0x%x", abs) + } + } + out = append(out, map[string]any{ + "offset": base + int64(i), + "bytes": hexStr(buf[i : i+length]), + "mnemonic": mn, + "operands": ops, + "confident": conf, + }) + i += length + } + return out +} + +// ---- byte readers (set trunc on overrun, never panic) ---- + +func (d *x86dec) u8() uint8 { + if d.pos >= len(d.buf) { + d.trunc = true + return 0 + } + b := d.buf[d.pos] + d.pos++ + return b +} + +func (d *x86dec) u16() uint16 { return uint16(d.u8()) | uint16(d.u8())<<8 } +func (d *x86dec) u32() uint32 { + return uint32(d.u8()) | uint32(d.u8())<<8 | uint32(d.u8())<<16 | uint32(d.u8())<<24 +} +func (d *x86dec) u64() uint64 { return uint64(d.u32()) | uint64(d.u32())<<32 } + +func (d *x86dec) peek() (uint8, bool) { + if d.pos >= len(d.buf) { + return 0, false + } + return d.buf[d.pos], true +} + +// ---- prefix + opcode dispatch ---- + +func (d *x86dec) decode() (string, string, bool) { + d.opSize = 32 + if d.is64 { + d.addrSize = 64 + } else { + d.addrSize = 32 + } +prefixes: + for { + b, ok := d.peek() + if !ok { + return "", "", false + } + switch b { + case 0x66: + d.has66 = true + d.opSize = 16 + d.pos++ + continue + case 0x67: + if d.is64 { + d.addrSize = 32 + } else { + d.addrSize = 16 + } + d.pos++ + continue + case 0xF0: + d.lock = true + d.pos++ + continue + case 0xF2: + d.rep = "repne" + d.pos++ + continue + case 0xF3: + d.rep = "rep" + d.pos++ + continue + case 0x2E: + d.seg = "cs" + d.pos++ + continue + case 0x36: + d.seg = "ss" + d.pos++ + continue + case 0x3E: + d.seg = "ds" + d.pos++ + continue + case 0x26: + d.seg = "es" + d.pos++ + continue + case 0x64: + d.seg = "fs" + d.pos++ + continue + case 0x65: + d.seg = "gs" + d.pos++ + continue + } + if d.is64 && b >= 0x40 && b <= 0x4f { + d.hasRex = true + d.rexW = b&0x08 != 0 + d.rexR = b&0x04 != 0 + d.rexX = b&0x02 != 0 + d.rexB = b&0x01 != 0 + d.pos++ + } + break prefixes + } + if d.rexW { + d.opSize = 64 + } + + op := d.u8() + if d.trunc { + return "", "", false + } + if op == 0x0f { + return d.decode0F() + } + return d.decode1(op) +} + +// ---- register naming ---- + +var reg8 = [...]string{"al", "cl", "dl", "bl", "ah", "ch", "dh", "bh"} +var reg8rex = [...]string{"al", "cl", "dl", "bl", "spl", "bpl", "sil", "dil", + "r8b", "r9b", "r10b", "r11b", "r12b", "r13b", "r14b", "r15b"} +var reg16 = [...]string{"ax", "cx", "dx", "bx", "sp", "bp", "si", "di", + "r8w", "r9w", "r10w", "r11w", "r12w", "r13w", "r14w", "r15w"} +var reg32 = [...]string{"eax", "ecx", "edx", "ebx", "esp", "ebp", "esi", "edi", + "r8d", "r9d", "r10d", "r11d", "r12d", "r13d", "r14d", "r15d"} +var reg64 = [...]string{"rax", "rcx", "rdx", "rbx", "rsp", "rbp", "rsi", "rdi", + "r8", "r9", "r10", "r11", "r12", "r13", "r14", "r15"} + +func (d *x86dec) regName(size, n int) string { + switch size { + case 8: + if d.hasRex { + return reg8rex[n&15] + } + return reg8[n&7] + case 16: + return reg16[n&15] + case 64: + return reg64[n&15] + case 128: + return fmt.Sprintf("xmm%d", n&15) + default: + return reg32[n&15] + } +} + +func accReg(size int) string { + switch size { + case 8: + return "al" + case 16: + return "ax" + case 64: + return "rax" + default: + return "eax" + } +} + +func ptrSize(size int) string { + switch size { + case 8: + return "byte ptr" + case 16: + return "word ptr" + case 64: + return "qword ptr" + case 128: + return "xmmword ptr" + default: + return "dword ptr" + } +} + +// ---- ModR/M + SIB ---- + +// modrm reads the ModR/M byte (and any SIB/displacement) and returns the +// extended reg field plus the textual r/m operand sized to rmSize. +func (d *x86dec) modrm(rmSize int) (int, string) { + return d.modrmEx(rmSize, true) +} + +func (d *x86dec) modrmEx(rmSize int, ptr bool) (int, string) { + m := d.u8() + mod := int(m >> 6) + reg := int((m >> 3) & 7) + if d.rexR { + reg += 8 + } + rm := int(m & 7) + if mod == 3 { + r := rm + if d.rexB { + r += 8 + } + return reg, d.regName(rmSize, r) + } + if d.addrSize == 16 { + return reg, d.mem16(mod, rm, rmSize, ptr) + } + return reg, d.mem32(mod, rm, rmSize, ptr) +} + +func (d *x86dec) mem32(mod, rm, opsize int, ptr bool) string { + var base, index string + scale := 1 + dispSize := 0 + ripRel := false + + switch { + case rm == 4: + sib := d.u8() + scale = 1 << ((sib >> 6) & 3) + idx := int((sib >> 3) & 7) + baseRaw := int(sib & 7) + if !(idx == 4 && !d.rexX) { + ir := idx + if d.rexX { + ir += 8 + } + index = d.regName(d.addrSize, ir) + } + if baseRaw == 5 && mod == 0 { + dispSize = 32 + } else { + br := baseRaw + if d.rexB { + br += 8 + } + base = d.regName(d.addrSize, br) + } + case rm == 5 && mod == 0: + dispSize = 32 + if d.is64 { + ripRel = true + } + default: + br := rm + if d.rexB { + br += 8 + } + base = d.regName(d.addrSize, br) + } + + switch mod { + case 1: + if dispSize == 0 { + dispSize = 8 + } + case 2: + if dispSize == 0 { + dispSize = 32 + } + } + + var disp int64 + hasDisp := dispSize != 0 + switch dispSize { + case 8: + disp = int64(int8(d.u8())) + case 32: + disp = int64(int32(d.u32())) + } + return d.formatMem(opsize, base, index, scale, disp, hasDisp, ripRel, ptr) +} + +var amode16 = [...]string{"bx+si", "bx+di", "bp+si", "bp+di", "si", "di", "bp", "bx"} + +func (d *x86dec) mem16(mod, rm, opsize int, ptr bool) string { + base := amode16[rm] + dispSize := 0 + if rm == 6 && mod == 0 { + base = "" + dispSize = 16 + } + switch mod { + case 1: + dispSize = 8 + case 2: + dispSize = 16 + } + var disp int64 + hasDisp := dispSize != 0 + switch dispSize { + case 8: + disp = int64(int8(d.u8())) + case 16: + disp = int64(int16(d.u16())) + } + return d.formatMem(opsize, base, "", 1, disp, hasDisp, false, ptr) +} + +func (d *x86dec) formatMem(opsize int, base, index string, scale int, disp int64, hasDisp, ripRel, ptr bool) string { + var inner string + if ripRel { + d.hasRip = true + d.ripDisp = disp + inner = "rip" + signHex(disp) + } else { + first := true + if base != "" { + inner += base + first = false + } + if index != "" { + if !first { + inner += "+" + } + inner += index + if scale != 1 { + inner += fmt.Sprintf("*%d", scale) + } + first = false + } + if hasDisp { + if first { + inner += fmt.Sprintf("0x%x", uint64(disp)&addrMask(d.addrSize)) + } else { + inner += signHex(disp) + } + } else if first { + inner += "0x0" + } + } + seg := "" + if d.seg != "" { + seg = d.seg + ":" + } + if ptr { + return ptrSize(opsize) + " " + seg + "[" + inner + "]" + } + return seg + "[" + inner + "]" +} + +func addrMask(size int) uint64 { + switch size { + case 16: + return 0xffff + case 32: + return 0xffffffff + default: + return ^uint64(0) + } +} + +func signHex(v int64) string { + if v < 0 { + return fmt.Sprintf("-0x%x", -v) + } + return fmt.Sprintf("+0x%x", v) +} + +// ---- immediates as text ---- + +func (d *x86dec) immZ() string { + if d.opSize == 16 { + return fmt.Sprintf("0x%x", d.u16()) + } + return fmt.Sprintf("0x%x", d.u32()) +} + +func (d *x86dec) immB() string { return fmt.Sprintf("0x%x", d.u8()) } +func (d *x86dec) immBs() string { return fmt.Sprintf("0x%x", uint64(int64(int8(d.u8())))) } + +// ---- condition codes ---- + +var ccName = [...]string{"o", "no", "b", "ae", "e", "ne", "be", "a", + "s", "ns", "p", "np", "l", "ge", "le", "g"} + +func (d *x86dec) rel8() string { + rel := int64(int8(d.u8())) + return fmt.Sprintf("0x%x", d.base+int64(d.pos)+rel) +} + +func (d *x86dec) rel32() string { + rel := int64(int32(d.u32())) + return fmt.Sprintf("0x%x", d.base+int64(d.pos)+rel) +} + +// stackSize is the operand size for implicit-stack ops (push/pop/call/jmp), +// which default to 64-bit in long mode. +func (d *x86dec) stackSize() int { + if d.is64 { + if d.has66 { + return 16 + } + return 64 + } + return d.opSize +} + +var aluName = [...]string{"add", "or", "adc", "sbb", "and", "sub", "xor", "cmp"} +var shiftName = [...]string{"rol", "ror", "rcl", "rcr", "shl", "shr", "shl", "sar"} + +// ---- one-byte opcode map ---- + +func (d *x86dec) decode1(op uint8) (string, string, bool) { + // ALU group: 0x00-0x3F in the standard 8-wide pattern. + if op <= 0x3f && (op&7) <= 5 { + name := aluName[(op>>3)&7] + switch op & 7 { + case 0: + reg, rm := d.modrm(8) + return name, rm + ", " + d.regName(8, reg), true + case 1: + reg, rm := d.modrm(d.opSize) + return name, rm + ", " + d.regName(d.opSize, reg), true + case 2: + reg, rm := d.modrm(8) + return name, d.regName(8, reg) + ", " + rm, true + case 3: + reg, rm := d.modrm(d.opSize) + return name, d.regName(d.opSize, reg) + ", " + rm, true + case 4: + return name, "al, " + d.immB(), true + case 5: + return name, accReg(d.opSize) + ", " + d.immZ(), true + } + } + + switch { + case op == 0x0f: + return d.decode0F() + + // INC/DEC reg (32-bit only; REX in 64-bit is handled as a prefix). + case op >= 0x40 && op <= 0x4f && !d.is64: + name := "inc" + if op >= 0x48 { + name = "dec" + } + return name, d.regName(d.opSize, int(op&7)), true + + case op >= 0x50 && op <= 0x57: + r := int(op - 0x50) + if d.rexB { + r += 8 + } + return "push", d.regName(d.stackSize(), r), true + case op >= 0x58 && op <= 0x5f: + r := int(op - 0x58) + if d.rexB { + r += 8 + } + return "pop", d.regName(d.stackSize(), r), true + + case op == 0x63: // MOVSXD (64) / ARPL (32) + if d.is64 { + reg, rm := d.modrm(32) + return "movsxd", d.regName(64, reg) + ", " + rm, true + } + reg, rm := d.modrm(16) + return "arpl", rm + ", " + d.regName(16, reg), true + + case op == 0x68: + return "push", d.immZ(), true + case op == 0x6a: + return "push", d.immBs(), true + case op == 0x69: + reg, rm := d.modrm(d.opSize) + return "imul", d.regName(d.opSize, reg) + ", " + rm + ", " + d.immZ(), true + case op == 0x6b: + reg, rm := d.modrm(d.opSize) + return "imul", d.regName(d.opSize, reg) + ", " + rm + ", " + d.immBs(), true + + case op >= 0x70 && op <= 0x7f: + return "j" + ccName[op-0x70], d.rel8(), true + + case op == 0x80: // GRP1 Eb, ib + reg, rm := d.modrm(8) + return aluName[reg&7], rm + ", " + d.immB(), true + case op == 0x81: // GRP1 Ev, iz + reg, rm := d.modrm(d.opSize) + return aluName[reg&7], rm + ", " + d.immZ(), true + case op == 0x82: // alias of 0x80 + reg, rm := d.modrm(8) + return aluName[reg&7], rm + ", " + d.immB(), true + case op == 0x83: // GRP1 Ev, ib (sign-extended) + reg, rm := d.modrm(d.opSize) + return aluName[reg&7], rm + ", " + d.immBs(), true + + case op == 0x84: + reg, rm := d.modrm(8) + return "test", rm + ", " + d.regName(8, reg), true + case op == 0x85: + reg, rm := d.modrm(d.opSize) + return "test", rm + ", " + d.regName(d.opSize, reg), true + case op == 0x86: + reg, rm := d.modrm(8) + return "xchg", rm + ", " + d.regName(8, reg), true + case op == 0x87: + reg, rm := d.modrm(d.opSize) + return "xchg", rm + ", " + d.regName(d.opSize, reg), true + + case op == 0x88: + reg, rm := d.modrm(8) + return "mov", rm + ", " + d.regName(8, reg), true + case op == 0x89: + reg, rm := d.modrm(d.opSize) + return "mov", rm + ", " + d.regName(d.opSize, reg), true + case op == 0x8a: + reg, rm := d.modrm(8) + return "mov", d.regName(8, reg) + ", " + rm, true + case op == 0x8b: + reg, rm := d.modrm(d.opSize) + return "mov", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0x8c: // mov Ew, Sreg + reg, rm := d.modrm(16) + return "mov", rm + ", " + segReg(reg), true + case op == 0x8d: // lea + reg, rm := d.modrmEx(d.opSize, false) + return "lea", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0x8e: // mov Sreg, Ew + reg, rm := d.modrm(16) + return "mov", segReg(reg) + ", " + rm, true + case op == 0x8f: // GRP1A pop Ev + _, rm := d.modrm(d.stackSize()) + return "pop", rm, true + + case op == 0x90: + if d.rep == "rep" { + return "pause", "", true + } + if !d.rexB { + return "nop", "", true + } + fallthrough + case op >= 0x91 && op <= 0x97: + r := int(op - 0x90) + if d.rexB { + r += 8 + } + return "xchg", accReg(d.opSize) + ", " + d.regName(d.opSize, r), true + case op == 0x98: + return map[int]string{16: "cbw", 32: "cwde", 64: "cdqe"}[d.opSize], "", true + case op == 0x99: + return map[int]string{16: "cwd", 32: "cdq", 64: "cqo"}[d.opSize], "", true + case op == 0x9b: + return "fwait", "", true + case op == 0x9c: + return "pushf", "", true + case op == 0x9d: + return "popf", "", true + case op == 0x9e: + return "sahf", "", true + case op == 0x9f: + return "lahf", "", true + + case op == 0xa0: + return "mov", "al, " + d.moffs(), true + case op == 0xa1: + return "mov", accReg(d.opSize) + ", " + d.moffs(), true + case op == 0xa2: + return "mov", d.moffs() + ", al", true + case op == 0xa3: + return "mov", d.moffs() + ", " + accReg(d.opSize), true + case op == 0xa4: + return d.repPfx() + "movsb", "", true + case op == 0xa5: + return d.repPfx() + "movs" + sizeSuffix(d.opSize), "", true + case op == 0xa6: + return d.repPfx() + "cmpsb", "", true + case op == 0xa7: + return d.repPfx() + "cmps" + sizeSuffix(d.opSize), "", true + case op == 0xa8: + return "test", "al, " + d.immB(), true + case op == 0xa9: + return "test", accReg(d.opSize) + ", " + d.immZ(), true + case op == 0xaa: + return d.repPfx() + "stosb", "", true + case op == 0xab: + return d.repPfx() + "stos" + sizeSuffix(d.opSize), "", true + case op == 0xac: + return d.repPfx() + "lodsb", "", true + case op == 0xad: + return d.repPfx() + "lods" + sizeSuffix(d.opSize), "", true + case op == 0xae: + return d.repPfx() + "scasb", "", true + case op == 0xaf: + return d.repPfx() + "scas" + sizeSuffix(d.opSize), "", true + + case op >= 0xb0 && op <= 0xb7: + r := int(op - 0xb0) + if d.rexB { + r += 8 + } + return "mov", d.regName(8, r) + ", " + d.immB(), true + case op >= 0xb8 && op <= 0xbf: + r := int(op - 0xb8) + if d.rexB { + r += 8 + } + if d.opSize == 64 { + return "movabs", d.regName(64, r) + fmt.Sprintf(", 0x%x", d.u64()), true + } + return "mov", d.regName(d.opSize, r) + ", " + d.immZ(), true + + case op == 0xc0: // GRP2 Eb, ib + reg, rm := d.modrm(8) + return shiftName[reg&7], rm + ", " + d.immB(), true + case op == 0xc1: // GRP2 Ev, ib + reg, rm := d.modrm(d.opSize) + return shiftName[reg&7], rm + ", " + d.immB(), true + case op == 0xc2: + return "ret", fmt.Sprintf("0x%x", d.u16()), true + case op == 0xc3: + return "ret", "", true + case op == 0xc6: // GRP11 mov Eb, ib + _, rm := d.modrm(8) + return "mov", rm + ", " + d.immB(), true + case op == 0xc7: // GRP11 mov Ev, iz + _, rm := d.modrm(d.opSize) + return "mov", rm + ", " + d.immZ(), true + case op == 0xc8: + iw := d.u16() + ib := d.u8() + return "enter", fmt.Sprintf("0x%x, 0x%x", iw, ib), true + case op == 0xc9: + return "leave", "", true + case op == 0xca: + return "retf", fmt.Sprintf("0x%x", d.u16()), true + case op == 0xcb: + return "retf", "", true + case op == 0xcc: + return "int3", "", true + case op == 0xcd: + return "int", d.immB(), true + case op == 0xce: + return "into", "", true + case op == 0xcf: + return "iret", "", true + + case op == 0xd0: // GRP2 Eb, 1 + reg, rm := d.modrm(8) + return shiftName[reg&7], rm + ", 1", true + case op == 0xd1: // GRP2 Ev, 1 + reg, rm := d.modrm(d.opSize) + return shiftName[reg&7], rm + ", 1", true + case op == 0xd2: // GRP2 Eb, cl + reg, rm := d.modrm(8) + return shiftName[reg&7], rm + ", cl", true + case op == 0xd3: // GRP2 Ev, cl + reg, rm := d.modrm(d.opSize) + return shiftName[reg&7], rm + ", cl", true + case op == 0xd7: + return "xlat", "", true + case op >= 0xd8 && op <= 0xdf: // x87 — consume ModR/M for length, name generic + _, rm := d.modrmEx(32, true) + return "fpu", rm, false + + case op == 0xe0: + return "loopne", d.rel8(), true + case op == 0xe1: + return "loope", d.rel8(), true + case op == 0xe2: + return "loop", d.rel8(), true + case op == 0xe3: + if d.addrSize == 32 { + return "jecxz", d.rel8(), true + } + return "jrcxz", d.rel8(), true + case op == 0xe4: + return "in", "al, " + d.immB(), true + case op == 0xe5: + return "in", accReg(d.opSize) + ", " + d.immB(), true + case op == 0xe6: + return "out", d.immB() + ", al", true + case op == 0xe7: + return "out", d.immB() + ", " + accReg(d.opSize), true + case op == 0xe8: + return "call", d.rel32(), true + case op == 0xe9: + return "jmp", d.rel32(), true + case op == 0xeb: + return "jmp", d.rel8(), true + case op == 0xec: + return "in", "al, dx", true + case op == 0xed: + return "in", accReg(d.opSize) + ", dx", true + case op == 0xee: + return "out", "dx, al", true + case op == 0xef: + return "out", "dx, " + accReg(d.opSize), true + + case op == 0xf1: + return "int1", "", true + case op == 0xf4: + return "hlt", "", true + case op == 0xf5: + return "cmc", "", true + case op == 0xf6: + return d.grp3(8) + case op == 0xf7: + return d.grp3(d.opSize) + case op == 0xf8: + return "clc", "", true + case op == 0xf9: + return "stc", "", true + case op == 0xfa: + return "cli", "", true + case op == 0xfb: + return "sti", "", true + case op == 0xfc: + return "cld", "", true + case op == 0xfd: + return "std", "", true + case op == 0xfe: // GRP4 + reg, rm := d.modrm(8) + if reg&7 == 0 { + return "inc", rm, true + } + return "dec", rm, true + case op == 0xff: // GRP5 + return d.grp5() + } + return "", "", false +} + +func (d *x86dec) repPfx() string { + if d.rep == "rep" { + return "rep " + } + if d.rep == "repne" { + return "repne " + } + return "" +} + +func sizeSuffix(size int) string { + switch size { + case 16: + return "w" + case 64: + return "q" + default: + return "d" + } +} + +func segReg(n int) string { + names := [...]string{"es", "cs", "ss", "ds", "fs", "gs", "?", "?"} + return names[n&7] +} + +func (d *x86dec) moffs() string { + var off uint64 + switch d.addrSize { + case 16: + off = uint64(d.u16()) + case 32: + off = uint64(d.u32()) + default: + off = d.u64() + } + seg := "" + if d.seg != "" { + seg = d.seg + ":" + } + return fmt.Sprintf("%s[0x%x]", seg, off) +} + +func (d *x86dec) grp3(size int) (string, string, bool) { + reg, rm := d.modrm(size) + switch reg & 7 { + case 0, 1: + if size == 8 { + return "test", rm + ", " + d.immB(), true + } + return "test", rm + ", " + d.immZ(), true + case 2: + return "not", rm, true + case 3: + return "neg", rm, true + case 4: + return "mul", rm, true + case 5: + return "imul", rm, true + case 6: + return "div", rm, true + default: + return "idiv", rm, true + } +} + +func (d *x86dec) grp5() (string, string, bool) { + // Peek reg field without committing size, then decode appropriately. + m, ok := d.peek() + if !ok { + return "", "", false + } + reg := int((m >> 3) & 7) + switch reg { + case 0: + _, rm := d.modrm(d.opSize) + return "inc", rm, true + case 1: + _, rm := d.modrm(d.opSize) + return "dec", rm, true + case 2: + _, rm := d.modrm(d.stackSize()) + return "call", rm, true + case 3: + _, rm := d.modrm(d.opSize) + return "callf", rm, true + case 4: + _, rm := d.modrm(d.stackSize()) + return "jmp", rm, true + case 5: + _, rm := d.modrm(d.opSize) + return "jmpf", rm, true + case 6: + _, rm := d.modrm(d.stackSize()) + return "push", rm, true + default: + _, rm := d.modrm(d.opSize) + return "(bad)", rm, false + } +} + +// ---- two-byte 0F map ---- + +func (d *x86dec) decode0F() (string, string, bool) { + op := d.u8() + if d.trunc { + return "", "", false + } + switch { + case op == 0x05: + return "syscall", "", true + case op == 0x06: + return "clts", "", true + case op == 0x07: + return "sysret", "", true + case op == 0x08: + return "invd", "", true + case op == 0x09: + return "wbinvd", "", true + case op == 0x0b: + return "ud2", "", true + case op == 0x1f: // multi-byte NOP + _, rm := d.modrm(d.opSize) + return "nop", rm, true + case op == 0x1e && d.rep == "rep": // ENDBR32/ENDBR64 (f3 0f 1e fa/fb) + m := d.u8() + if m == 0xfa { + return "endbr64", "", true + } + if m == 0xfb { + return "endbr32", "", true + } + return "nop", "", true + case op >= 0x18 && op <= 0x1e: // hints / prefetch + _, rm := d.modrmEx(8, true) + return "nop", rm, true + case op == 0x31: + return "rdtsc", "", true + case op == 0x34: + return "sysenter", "", true + case op == 0x35: + return "sysexit", "", true + case op == 0xa2: + return "cpuid", "", true + case op == 0x77: + return "emms", "", true + case op == 0xa0: + return "push", "fs", true + case op == 0xa1: + return "pop", "fs", true + case op == 0xa8: + return "push", "gs", true + case op == 0xa9: + return "pop", "gs", true + + case op >= 0x40 && op <= 0x4f: // CMOVcc + reg, rm := d.modrm(d.opSize) + return "cmov" + ccName[op-0x40], d.regName(d.opSize, reg) + ", " + rm, true + case op >= 0x80 && op <= 0x8f: // Jcc rel32 + return "j" + ccName[op-0x80], d.rel32(), true + case op >= 0x90 && op <= 0x9f: // SETcc Eb + _, rm := d.modrm(8) + return "set" + ccName[op-0x90], rm, true + + case op == 0xa3: + reg, rm := d.modrm(d.opSize) + return "bt", rm + ", " + d.regName(d.opSize, reg), true + case op == 0xab: + reg, rm := d.modrm(d.opSize) + return "bts", rm + ", " + d.regName(d.opSize, reg), true + case op == 0xb3: + reg, rm := d.modrm(d.opSize) + return "btr", rm + ", " + d.regName(d.opSize, reg), true + case op == 0xbb: + reg, rm := d.modrm(d.opSize) + return "btc", rm + ", " + d.regName(d.opSize, reg), true + case op == 0xba: // GRP8 Ev, ib + reg, rm := d.modrm(d.opSize) + names := map[int]string{4: "bt", 5: "bts", 6: "btr", 7: "btc"} + if n, okn := names[reg&7]; okn { + return n, rm + ", " + d.immB(), true + } + return "(bad)", rm, false + + case op == 0xa4: + reg, rm := d.modrm(d.opSize) + return "shld", rm + ", " + d.regName(d.opSize, reg) + ", " + d.immB(), true + case op == 0xa5: + reg, rm := d.modrm(d.opSize) + return "shld", rm + ", " + d.regName(d.opSize, reg) + ", cl", true + case op == 0xac: + reg, rm := d.modrm(d.opSize) + return "shrd", rm + ", " + d.regName(d.opSize, reg) + ", " + d.immB(), true + case op == 0xad: + reg, rm := d.modrm(d.opSize) + return "shrd", rm + ", " + d.regName(d.opSize, reg) + ", cl", true + case op == 0xaf: + reg, rm := d.modrm(d.opSize) + return "imul", d.regName(d.opSize, reg) + ", " + rm, true + + case op == 0xb0: + reg, rm := d.modrm(8) + return "cmpxchg", rm + ", " + d.regName(8, reg), true + case op == 0xb1: + reg, rm := d.modrm(d.opSize) + return "cmpxchg", rm + ", " + d.regName(d.opSize, reg), true + case op == 0xb6: + reg, rm := d.modrm(8) + return "movzx", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xb7: + reg, rm := d.modrm(16) + return "movzx", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xbe: + reg, rm := d.modrm(8) + return "movsx", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xbf: + reg, rm := d.modrm(16) + return "movsx", d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xbc: + reg, rm := d.modrm(d.opSize) + name := "bsf" + if d.rep == "rep" { + name = "tzcnt" + } + return name, d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xbd: + reg, rm := d.modrm(d.opSize) + name := "bsr" + if d.rep == "rep" { + name = "lzcnt" + } + return name, d.regName(d.opSize, reg) + ", " + rm, true + case op == 0xc0: + reg, rm := d.modrm(8) + return "xadd", rm + ", " + d.regName(8, reg), true + case op == 0xc1: + reg, rm := d.modrm(d.opSize) + return "xadd", rm + ", " + d.regName(d.opSize, reg), true + case op >= 0xc8 && op <= 0xcf: + r := int(op - 0xc8) + if d.rexB { + r += 8 + } + return "bswap", d.regName(d.opSize, r), true + } + // Common SSE / SSE2 (XMM) subset. + if mn, ops, ok := d.sse0F(op); ok { + return mn, ops, true + } + return "", "", false +} + +// sse0F decodes the common SSE/SSE2 instructions. Mnemonics depend on the +// mandatory prefix (none / 0x66 / 0xF3 / 0xF2). +func (d *x86dec) sse0F(op uint8) (string, string, bool) { + pfx := "none" + switch { + case d.has66: + pfx = "66" + case d.rep == "rep": + pfx = "f3" + case d.rep == "repne": + pfx = "f2" + } + + xmmRM := func(name string, store bool) (string, string, bool) { + reg, rm := d.modrm(128) + r := d.regName(128, reg) + if store { + return name, rm + ", " + r, true + } + return name, r + ", " + rm, true + } + + switch op { + case 0x10: + return xmmRM(map[string]string{"none": "movups", "66": "movupd", "f3": "movss", "f2": "movsd"}[pfx], false) + case 0x11: + return xmmRM(map[string]string{"none": "movups", "66": "movupd", "f3": "movss", "f2": "movsd"}[pfx], true) + case 0x28: + return xmmRM(map[string]string{"none": "movaps", "66": "movapd"}[pfx2(pfx)], false) + case 0x29: + return xmmRM(map[string]string{"none": "movaps", "66": "movapd"}[pfx2(pfx)], true) + case 0x2e: + return xmmRM(map[string]string{"none": "ucomiss", "66": "ucomisd"}[pfx2(pfx)], false) + case 0x2f: + return xmmRM(map[string]string{"none": "comiss", "66": "comisd"}[pfx2(pfx)], false) + case 0x51: + return xmmRM(sseScalar("sqrt", pfx), false) + case 0x54: + return xmmRM(map[string]string{"none": "andps", "66": "andpd"}[pfx2(pfx)], false) + case 0x57: + return xmmRM(map[string]string{"none": "xorps", "66": "xorpd"}[pfx2(pfx)], false) + case 0x58: + return xmmRM(sseScalar("add", pfx), false) + case 0x59: + return xmmRM(sseScalar("mul", pfx), false) + case 0x5c: + return xmmRM(sseScalar("sub", pfx), false) + case 0x5e: + return xmmRM(sseScalar("div", pfx), false) + case 0x5d: + return xmmRM(sseScalar("min", pfx), false) + case 0x5f: + return xmmRM(sseScalar("max", pfx), false) + case 0x6e: // movd/movq xmm, r/m + sz := 32 + name := "movd" + if d.opSize == 64 { + sz, name = 64, "movq" + } + reg, rm := d.modrm(sz) + return name, d.regName(128, reg) + ", " + rm, true + case 0x7e: + if pfx == "f3" { // movq xmm, xmm/m64 + return xmmRM("movq", false) + } + sz := 32 + name := "movd" + if d.opSize == 64 { + sz, name = 64, "movq" + } + reg, rm := d.modrm(sz) + return name, rm + ", " + d.regName(128, reg), true + case 0x6f: + return xmmRM(map[string]string{"66": "movdqa", "f3": "movdqu"}[pfx], false) + case 0x7f: + return xmmRM(map[string]string{"66": "movdqa", "f3": "movdqu"}[pfx], true) + case 0xd6: + return xmmRM("movq", true) + case 0xef: + return xmmRM("pxor", false) + case 0x76: + return xmmRM("pcmpeqd", false) + case 0x74: + return xmmRM("pcmpeqb", false) + case 0xfc: + return xmmRM("paddb", false) + case 0xfe: + return xmmRM("paddd", false) + case 0x12: + return xmmRM(map[string]string{"none": "movlps", "66": "movlpd"}[pfx2(pfx)], false) + case 0x13: + return xmmRM(map[string]string{"none": "movlps", "66": "movlpd"}[pfx2(pfx)], true) + case 0x16: + return xmmRM(map[string]string{"none": "movhps", "66": "movhpd"}[pfx2(pfx)], false) + case 0x17: + return xmmRM(map[string]string{"none": "movhps", "66": "movhpd"}[pfx2(pfx)], true) + case 0x14: + return xmmRM(map[string]string{"none": "unpcklps", "66": "unpcklpd"}[pfx2(pfx)], false) + case 0x2a: + reg, rm := d.modrm(d.opSize) + return sseScalar("cvtsi2", pfx), d.regName(128, reg) + ", " + rm, true + case 0x2c, 0x2d: + reg, rm := d.modrm(128) + return "cvt", d.regName(d.opSize, reg) + ", " + rm, true + } + return "", "", false +} + +// pfx2 collapses f3/f2 to "none" for instructions that only vary between the +// no-prefix and 0x66 forms. +func pfx2(p string) string { + if p == "66" { + return "66" + } + return "none" +} + +func sseScalar(base, pfx string) string { + switch pfx { + case "f3": + return base + "ss" + case "f2": + return base + "sd" + case "66": + return base + "pd" + default: + return base + "ps" + } +} diff --git a/internal/leviathan/document/document.go b/internal/leviathan/document/document.go new file mode 100644 index 0000000..d43797d --- /dev/null +++ b/internal/leviathan/document/document.go @@ -0,0 +1,439 @@ +// Package document implements the chunked binary document model. +// +// The document holds three conceptual layers per slot: +// +// source — immutable original bytes, loaded once. +// patch — sparse overlay of modified bytes keyed by offset. +// working — logical view = source with patch applied on read. +// +// Reads happen through windowed Range() calls so neither the editor +// nor analysis services need to materialise the whole file. +package document + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" +) + +// ChunkSize is the granularity at which source bytes are exposed to +// analysis services. It is independent of the view row width. +const ChunkSize = 64 * 1024 + +// Document represents one loaded artefact. A Session may hold many. +type Document struct { + mu sync.RWMutex + name string + source []byte // original immutable content + path string // optional immutable file-backed source + size int64 + patch map[int64]byte + history []EditOp + redo []EditOp + cursor int +} + +// EditOp records a single byte-level mutation for undo/redo. +type EditOp struct { + Offset int64 + Old byte + New byte + Kind string // "set", "fill", "xor" +} + +// New creates a Document from raw source bytes. The slice is not +// copied; callers must not mutate it after handoff. +func New(name string, src []byte) *Document { + return &Document{ + name: name, + source: src, + size: int64(len(src)), + patch: make(map[int64]byte), + } +} + +// Open creates a Document backed by an immutable file on disk. Source +// bytes are read on demand through Range rather than loaded eagerly. +func Open(path string) (*Document, error) { + st, err := os.Stat(path) + if err != nil { + return nil, err + } + if st.IsDir() { + return nil, errors.New("document path is a directory") + } + return &Document{ + name: filepath.Base(path), + path: path, + size: st.Size(), + patch: make(map[int64]byte), + }, nil +} + +// Name returns the document's display name. +func (d *Document) Name() string { + d.mu.RLock() + defer d.mu.RUnlock() + return d.name +} + +// Size returns the logical document length in bytes. +func (d *Document) Size() int64 { + d.mu.RLock() + defer d.mu.RUnlock() + return d.size +} + +// At returns the logical byte at offset (patch takes precedence). +func (d *Document) At(offset int64) (byte, error) { + d.mu.RLock() + defer d.mu.RUnlock() + if offset < 0 || offset >= d.size { + return 0, errors.New("offset out of range") + } + if b, ok := d.patch[offset]; ok { + return b, nil + } + return d.sourceByteLocked(offset) +} + +// Range copies [start, start+length) into dst. It applies the patch +// overlay. Callers get a fresh slice they are free to retain. +func (d *Document) Range(start, length int64) ([]byte, error) { + d.mu.RLock() + defer d.mu.RUnlock() + n := d.size + if start < 0 || start > n { + return nil, errors.New("range start out of bounds") + } + if length < 0 { + return nil, errors.New("negative range") + } + if length > n-start { + length = n - start + } + out, err := d.sourceRangeLocked(start, length) + if err != nil { + return nil, err + } + if len(d.patch) > 0 { + end := start + length + for off, b := range d.patch { + if off >= start && off < end { + out[off-start] = b + } + } + } + return out, nil +} + +// IsDirty reports whether any byte has been patched. +func (d *Document) IsDirty() bool { + d.mu.RLock() + defer d.mu.RUnlock() + return len(d.patch) > 0 +} + +// Set patches a single byte. Returns the previous value. +func (d *Document) Set(offset int64, b byte) (byte, error) { + d.mu.Lock() + defer d.mu.Unlock() + if offset < 0 || offset >= d.size { + return 0, errors.New("offset out of range") + } + orig, err := d.sourceByteLocked(offset) + if err != nil { + return 0, err + } + var old byte + if v, ok := d.patch[offset]; ok { + old = v + } else { + old = orig + } + if b == orig { + delete(d.patch, offset) + } else { + d.patch[offset] = b + } + d.history = append(d.history, EditOp{Offset: offset, Old: old, New: b, Kind: "set"}) + d.redo = nil + return old, nil +} + +// Fill patches [start,start+length) with b. +func (d *Document) Fill(start, length int64, b byte) error { + d.mu.Lock() + defer d.mu.Unlock() + if err := d.validateRangeLocked("fill", start, length); err != nil { + return err + } + orig, err := d.sourceRangeLocked(start, length) + if err != nil { + return err + } + for i := int64(0); i < length; i++ { + off := start + i + origB := orig[i] + var old byte + if v, ok := d.patch[off]; ok { + old = v + } else { + old = origB + } + if b == origB { + delete(d.patch, off) + } else { + d.patch[off] = b + } + d.history = append(d.history, EditOp{Offset: off, Old: old, New: b, Kind: "fill"}) + } + d.redo = nil + return nil +} + +// XOR patches [start,start+length) with a repeating key. +func (d *Document) XOR(start, length int64, key []byte) error { + if len(key) == 0 { + return errors.New("empty key") + } + d.mu.Lock() + defer d.mu.Unlock() + if err := d.validateRangeLocked("xor", start, length); err != nil { + return err + } + orig, err := d.sourceRangeLocked(start, length) + if err != nil { + return err + } + for i := int64(0); i < length; i++ { + off := start + i + origB := orig[i] + var cur byte + if v, ok := d.patch[off]; ok { + cur = v + } else { + cur = origB + } + newB := cur ^ key[i%int64(len(key))] + if newB == origB { + delete(d.patch, off) + } else { + d.patch[off] = newB + } + d.history = append(d.history, EditOp{Offset: off, Old: cur, New: newB, Kind: "xor"}) + } + d.redo = nil + return nil +} + +// Copy copies [src,src+length) onto [dst,dst+length) as a patch. +func (d *Document) Copy(src, length, dst int64) error { + d.mu.Lock() + defer d.mu.Unlock() + if err := d.validateRangeLocked("copy source", src, length); err != nil { + return err + } + if err := d.validateRangeLocked("copy destination", dst, length); err != nil { + return err + } + buf, err := d.workingRangeLocked(src, length) + if err != nil { + return err + } + dstOrig, err := d.sourceRangeLocked(dst, length) + if err != nil { + return err + } + for i := int64(0); i < length; i++ { + off := dst + i + origB := dstOrig[i] + var cur byte + if v, ok := d.patch[off]; ok { + cur = v + } else { + cur = origB + } + b := buf[i] + if b == origB { + delete(d.patch, off) + } else { + d.patch[off] = b + } + d.history = append(d.history, EditOp{Offset: off, Old: cur, New: b, Kind: "fill"}) + } + d.redo = nil + return nil +} + +// Undo reverts the last edit. Returns false if there is no history. +func (d *Document) Undo() bool { + d.mu.Lock() + defer d.mu.Unlock() + if len(d.history) == 0 { + return false + } + op := d.history[len(d.history)-1] + d.history = d.history[:len(d.history)-1] + orig, err := d.sourceByteLocked(op.Offset) + if err != nil { + return false + } + if op.Old == orig { + delete(d.patch, op.Offset) + } else { + d.patch[op.Offset] = op.Old + } + d.redo = append(d.redo, op) + return true +} + +// Redo reapplies the last undone edit. Returns false if there is no redo. +func (d *Document) Redo() bool { + d.mu.Lock() + defer d.mu.Unlock() + if len(d.redo) == 0 { + return false + } + op := d.redo[len(d.redo)-1] + d.redo = d.redo[:len(d.redo)-1] + orig, err := d.sourceByteLocked(op.Offset) + if err != nil { + return false + } + if op.New == orig { + delete(d.patch, op.Offset) + } else { + d.patch[op.Offset] = op.New + } + d.history = append(d.history, op) + return true +} + +// Revert drops all patches and history. +func (d *Document) Revert() { + d.mu.Lock() + defer d.mu.Unlock() + d.patch = make(map[int64]byte) + d.history = nil + d.redo = nil +} + +// Patches returns a snapshot of the patch overlay. +func (d *Document) Patches() map[int64]byte { + d.mu.RLock() + defer d.mu.RUnlock() + out := make(map[int64]byte, len(d.patch)) + for k, v := range d.patch { + out[k] = v + } + return out +} + +// HistoryLen returns the number of recorded edits. +func (d *Document) HistoryLen() int { + d.mu.RLock() + defer d.mu.RUnlock() + return len(d.history) +} + +// RedoLen returns the number of redoable edits. +func (d *Document) RedoLen() int { + d.mu.RLock() + defer d.mu.RUnlock() + return len(d.redo) +} + +// ForEachChunk yields chunks covering [start,start+length) with the +// working-copy bytes visible. It is suitable for analysis passes that +// want bounded memory use. +func (d *Document) ForEachChunk(start, length int64, fn func(off int64, chunk []byte) error) error { + d.mu.RLock() + n := d.size + d.mu.RUnlock() + if start < 0 { + start = 0 + } + end := start + length + if length < 0 || end > n { + end = n + } + off := start + for off < end { + chunkEnd := off + ChunkSize + if chunkEnd > end { + chunkEnd = end + } + buf, err := d.Range(off, chunkEnd-off) + if err != nil { + return err + } + if err := fn(off, buf); err != nil { + return err + } + off = chunkEnd + } + return nil +} + +func (d *Document) sourceByteLocked(offset int64) (byte, error) { + buf, err := d.sourceRangeLocked(offset, 1) + if err != nil { + return 0, err + } + if len(buf) == 0 { + return 0, errors.New("offset out of range") + } + return buf[0], nil +} + +func (d *Document) workingRangeLocked(start, length int64) ([]byte, error) { + buf, err := d.sourceRangeLocked(start, length) + if err != nil { + return nil, err + } + for off, b := range d.patch { + if off >= start && off < start+length { + buf[off-start] = b + } + } + return buf, nil +} + +func (d *Document) sourceRangeLocked(start, length int64) ([]byte, error) { + if err := d.validateRangeLocked("range", start, length); err != nil { + return nil, err + } + if length == 0 { + return []byte{}, nil + } + if d.path == "" { + out := make([]byte, length) + copy(out, d.source[start:start+length]) + return out, nil + } + f, err := os.Open(d.path) + if err != nil { + return nil, err + } + defer f.Close() + out := make([]byte, length) + _, err = f.ReadAt(out, start) + if errors.Is(err, io.EOF) { + err = nil + } + if err != nil { + return nil, err + } + return out, nil +} + +func (d *Document) validateRangeLocked(label string, start, length int64) error { + if start < 0 || start > d.size || length < 0 || length > d.size-start { + return fmt.Errorf("%s range invalid", label) + } + return nil +} diff --git a/internal/leviathan/document/session.go b/internal/leviathan/document/session.go new file mode 100644 index 0000000..c0c0a86 --- /dev/null +++ b/internal/leviathan/document/session.go @@ -0,0 +1,62 @@ +package document + +import ( + "errors" + "sync" +) + +// Slot identifies one of the documents in a session. +type Slot string + +const ( + SlotOriginal Slot = "original" + SlotCompare Slot = "compare" +) + +// Session tracks loaded documents and addresses them by slot. +// The original slot is the active working document; compare is +// used by the diff tooling. +type Session struct { + mu sync.RWMutex + documents map[Slot]*Document +} + +// NewSession returns an empty session. +func NewSession() *Session { + return &Session{documents: make(map[Slot]*Document)} +} + +// Load replaces the document in the given slot. +func (s *Session) Load(slot Slot, name string, src []byte) *Document { + s.mu.Lock() + defer s.mu.Unlock() + d := New(name, src) + s.documents[slot] = d + return d +} + +// Get returns the document for a slot, or an error if not loaded. +func (s *Session) Get(slot Slot) (*Document, error) { + s.mu.RLock() + defer s.mu.RUnlock() + d, ok := s.documents[slot] + if !ok { + return nil, errors.New("slot not loaded: " + string(slot)) + } + return d, nil +} + +// Has reports whether a slot has a document loaded. +func (s *Session) Has(slot Slot) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, ok := s.documents[slot] + return ok +} + +// Drop removes a slot. +func (s *Session) Drop(slot Slot) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.documents, slot) +} diff --git a/internal/leviathan/parsers/asn1.go b/internal/leviathan/parsers/asn1.go new file mode 100644 index 0000000..635025e --- /dev/null +++ b/internal/leviathan/parsers/asn1.go @@ -0,0 +1,321 @@ +package parsers + +import ( + "encoding/asn1" + "encoding/pem" + "errors" + "fmt" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ParseASN1 produces a field tree for the DER TLV structure starting +// at `at`, spanning at most `length` bytes. Nested SEQUENCE / SET +// structures are decoded recursively; primitive content is rendered +// as best-effort. +func ParseASN1(d *document.Document, at, length int64) (*Field, error) { + size := d.Size() + if length < 0 || at+length > size { + length = size - at + } + if length <= 0 { + return nil, errors.New("asn1: empty range") + } + buf, err := d.Range(at, length) + if err != nil { + return nil, err + } + // If the buffer looks like PEM, decode first. + if pemBlock, _ := pem.Decode(buf); pemBlock != nil { + root := &Field{Name: "PEM " + pemBlock.Type, Type: "container", Offset: at, Length: length} + inner, err := decodeASN1(pemBlock.Bytes, at) + if err == nil { + root.Children = append(root.Children, inner) + } else { + root.Note = "pem decoded but asn1 failed: " + err.Error() + } + return root, nil + } + return decodeASN1(buf, at) +} + +// decodeASN1 walks buf recursively, treating it as one or more DER TLVs. +func decodeASN1(buf []byte, base int64) (*Field, error) { + root := &Field{Name: "ASN.1", Type: "container", Offset: base, Length: int64(len(buf))} + off := 0 + for off < len(buf) { + raw, consumed, err := readTLV(buf[off:]) + if err != nil { + if off == 0 { + return nil, err + } + break + } + node := asn1Node(raw, base+int64(off)) + root.Children = append(root.Children, node) + off += consumed + } + if len(root.Children) == 0 { + return nil, errors.New("asn1: no TLV decoded") + } + if len(root.Children) == 1 { + return root.Children[0], nil + } + return root, nil +} + +// tlv is one DER TLV we've split out of the byte stream. +type tlv struct { + Class int // 0=universal, 1=application, 2=context, 3=private + Constructed bool + Tag int + Content []byte // the V of the TLV + HeaderLen int // length of T+L bytes +} + +func readTLV(buf []byte) (tlv, int, error) { + if len(buf) < 2 { + return tlv{}, 0, errors.New("short TLV") + } + b0 := buf[0] + t := tlv{ + Class: int(b0 >> 6), + Constructed: b0&0x20 != 0, + } + tagVal := int(b0 & 0x1f) + i := 1 + if tagVal == 0x1f { + // multi-byte tag + tagVal = 0 + for i < len(buf) { + c := buf[i] + i++ + tagVal = (tagVal << 7) | int(c&0x7f) + if c&0x80 == 0 { + break + } + } + } + t.Tag = tagVal + if i >= len(buf) { + return tlv{}, 0, errors.New("truncated length") + } + ln := int(buf[i]) + i++ + if ln >= 0x80 { + n := ln & 0x7f + if n == 0 || n > 8 || i+n > len(buf) { + return tlv{}, 0, errors.New("bad length") + } + ln = 0 + for k := 0; k < n; k++ { + ln = (ln << 8) | int(buf[i+k]) + } + i += n + } + if i+ln > len(buf) { + return tlv{}, 0, errors.New("length exceeds buffer") + } + t.HeaderLen = i + t.Content = buf[i : i+ln] + return t, i + ln, nil +} + +func asn1Node(t tlv, off int64) *Field { + name, kind := asn1TagName(t) + total := int64(t.HeaderLen + len(t.Content)) + node := &Field{Name: name, Type: kind, Offset: off, Length: total} + if t.Class == 2 { + node.Name = fmt.Sprintf("[%d]", t.Tag) + } + if t.Constructed { + // Recurse into children. + inner := t.Content + childOff := off + int64(t.HeaderLen) + i := 0 + for i < len(inner) { + child, consumed, err := readTLV(inner[i:]) + if err != nil { + break + } + node.Children = append(node.Children, asn1Node(child, childOff+int64(i))) + i += consumed + } + } else { + node.Value = renderPrimitive(t) + } + return node +} + +func asn1TagName(t tlv) (string, string) { + if t.Class != 0 { + prefix := []string{"", "APP ", "CTX ", "PRIV "}[t.Class] + return fmt.Sprintf("%s%d", prefix, t.Tag), "tag" + } + names := map[int]string{ + 1: "BOOLEAN", + 2: "INTEGER", + 3: "BIT STRING", + 4: "OCTET STRING", + 5: "NULL", + 6: "OBJECT IDENTIFIER", + 10: "ENUMERATED", + 12: "UTF8String", + 16: "SEQUENCE", + 17: "SET", + 19: "PrintableString", + 20: "T61String", + 22: "IA5String", + 23: "UTCTime", + 24: "GeneralizedTime", + } + if n, ok := names[t.Tag]; ok { + return n, strings.ToLower(n) + } + return fmt.Sprintf("tag %d", t.Tag), "tag" +} + +func renderPrimitive(t tlv) string { + switch t.Tag { + case 1: + if len(t.Content) == 1 { + if t.Content[0] == 0 { + return "false" + } + return "true" + } + case 2: + return fmt.Sprintf("%s (%d bytes)", bigIntString(t.Content), len(t.Content)) + case 3: + if len(t.Content) == 0 { + return "empty" + } + return fmt.Sprintf("unused=%d bits=%x", t.Content[0], t.Content[1:]) + case 4: + if isPrintable(t.Content) { + return fmt.Sprintf("%q (%d bytes)", string(t.Content), len(t.Content)) + } + return fmt.Sprintf("%x (%d bytes)", t.Content, len(t.Content)) + case 5: + return "null" + case 6: + oid, err := parseOID(t.Content) + if err != nil { + return err.Error() + } + return oid + oidNote(oid) + case 12, 19, 20, 22: + return fmt.Sprintf("%q", string(t.Content)) + case 23, 24: + return fmt.Sprintf("%q (time)", string(t.Content)) + } + if isPrintable(t.Content) { + return fmt.Sprintf("%q", string(t.Content)) + } + if len(t.Content) > 32 { + return fmt.Sprintf("%x… (%d bytes)", t.Content[:32], len(t.Content)) + } + return fmt.Sprintf("%x", t.Content) +} + +func bigIntString(b []byte) string { + if len(b) == 0 { + return "0" + } + if len(b) <= 8 { + var v int64 + negative := b[0]&0x80 != 0 + for _, c := range b { + v = (v << 8) | int64(c) + } + if negative { + // sign-extend + v -= 1 << uint(8*len(b)) + } + return fmt.Sprintf("%d", v) + } + // large — hex is more useful than a decimal approximation. + return fmt.Sprintf("0x%x", b) +} + +func parseOID(b []byte) (string, error) { + // Use Go's asn1 parser for correctness. + var oid asn1.ObjectIdentifier + rest, err := asn1.Unmarshal(append([]byte{0x06, byte(len(b))}, b...), &oid) + if err != nil || len(rest) != 0 { + // Fallback manual decode. + return manualOID(b), nil + } + return oid.String(), nil +} + +func manualOID(b []byte) string { + if len(b) == 0 { + return "" + } + parts := []string{} + first := int(b[0]) + parts = append(parts, fmt.Sprintf("%d.%d", first/40, first%40)) + v := 0 + for _, c := range b[1:] { + v = (v << 7) | int(c&0x7f) + if c&0x80 == 0 { + parts = append(parts, fmt.Sprintf("%d", v)) + v = 0 + } + } + return strings.Join(parts, ".") +} + +// Known-OID table for human-readable hints in cert dumps. +var oidNames = map[string]string{ + "1.2.840.113549.1.1.1": " rsaEncryption", + "1.2.840.113549.1.1.11": " sha256WithRSAEncryption", + "1.2.840.113549.1.1.5": " sha1WithRSAEncryption", + "1.2.840.113549.1.1.13": " sha512WithRSAEncryption", + "1.2.840.10045.2.1": " ecPublicKey", + "1.2.840.10045.4.3.2": " ecdsaWithSHA256", + "1.3.101.112": " Ed25519", + "2.5.4.3": " commonName", + "2.5.4.6": " countryName", + "2.5.4.7": " localityName", + "2.5.4.8": " stateOrProvinceName", + "2.5.4.10": " organizationName", + "2.5.4.11": " organizationalUnitName", + "2.5.4.5": " serialNumber", + "1.2.840.113549.1.9.1": " emailAddress", + "2.5.29.14": " subjectKeyIdentifier", + "2.5.29.15": " keyUsage", + "2.5.29.17": " subjectAltName", + "2.5.29.19": " basicConstraints", + "2.5.29.31": " cRLDistributionPoints", + "2.5.29.32": " certificatePolicies", + "2.5.29.35": " authorityKeyIdentifier", + "2.5.29.37": " extKeyUsage", + "1.3.6.1.5.5.7.3.1": " serverAuth", + "1.3.6.1.5.5.7.3.2": " clientAuth", + "1.3.6.1.5.5.7.1.1": " authorityInfoAccess", +} + +func oidNote(oid string) string { + if n, ok := oidNames[oid]; ok { + return n + } + return "" +} + +func isPrintable(b []byte) bool { + if len(b) == 0 { + return false + } + for _, c := range b { + if c == 0x09 || c == 0x0a || c == 0x0d { + continue + } + if c < 0x20 || c > 0x7e { + return false + } + } + return true +} diff --git a/internal/leviathan/parsers/cert.go b/internal/leviathan/parsers/cert.go new file mode 100644 index 0000000..18291d2 --- /dev/null +++ b/internal/leviathan/parsers/cert.go @@ -0,0 +1,196 @@ +package parsers + +import ( + "crypto/sha256" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ParseCerts scans the whole document for PEM-encoded certificates +// and for DER-encoded X.509 certificates appearing at arbitrary +// offsets (heuristic: SEQUENCE 0x30 0x82 , then tbsCertificate +// SEQUENCE, then version INTEGER). Returns a list of decoded certs. +func ParseCerts(d *document.Document) (*Field, error) { + size := d.Size() + root := &Field{Name: "Certificates", Type: "container", Offset: 0, Length: size} + + // Read whole doc — bounded by caller's judgment. For huge firmware + // blobs this might be expensive but certs are small; most real + // cases are 32*1024*1024 { + return nil, errors.New("certs scan skipped: file > 32 MiB") + } + all, err := d.Range(0, size) + if err != nil { + return nil, err + } + + // PEM pass. + rest := all + pemCount := 0 + var pemOff int64 = 0 + for { + block, next := pem.Decode(rest) + if block == nil { + break + } + // Approximate offset: the header "-----BEGIN" will be somewhere in `rest`. + idx := indexOfHeader(rest) + if idx < 0 { + break + } + start := pemOff + int64(idx) + end := pemOff + int64(len(rest)-len(next)) + pemCount++ + child := &Field{ + Name: fmt.Sprintf("PEM %s #%d", block.Type, pemCount), + Type: "pem", + Offset: start, + Length: end - start, + } + if block.Type == "CERTIFICATE" { + if cert, err := x509.ParseCertificate(block.Bytes); err == nil { + populateCert(child, cert) + } else { + child.Note = "x509 parse failed: " + err.Error() + } + } + root.Children = append(root.Children, child) + pemOff += int64(len(rest) - len(next)) + rest = next + } + + // DER pass — scan for SEQUENCE 0x30 0x82 xx yy (definite long-form, 2 length bytes). + // + // Each candidate runs a full x509.ParseCertificate over a blob up to 64 KiB. + // Without guards a buffer dense with "30 82" byte pairs (random/firmware + // data, or a hostile file) would attempt that parse at nearly every offset + // — O(n) expensive parses, enough to freeze the UI. Two guards keep it + // bounded: a cheap structural pre-filter (a DER certificate body begins + // with the tbsCertificate SEQUENCE, 0x30) rejects almost all false + // candidates without parsing, and a hard cap on parse attempts bounds even + // adversarial input that defeats the pre-filter. + const maxDERAttempts = 1 << 16 + derCount := 0 + derAttempts := 0 + for i := 0; i+4 < len(all); i++ { + if all[i] != 0x30 || all[i+1] != 0x82 { + continue + } + if all[i+4] != 0x30 { + continue + } + totalLen := int(all[i+2])<<8 | int(all[i+3]) + if totalLen < 100 || i+4+totalLen > len(all) { + continue + } + if derAttempts++; derAttempts > maxDERAttempts { + break + } + blob := all[i : i+4+totalLen] + cert, err := x509.ParseCertificate(blob) + if err != nil { + continue + } + derCount++ + child := &Field{ + Name: fmt.Sprintf("DER cert #%d", derCount), + Type: "der", + Offset: int64(i), + Length: int64(4 + totalLen), + } + populateCert(child, cert) + root.Children = append(root.Children, child) + i += 4 + totalLen - 1 + } + + if len(root.Children) == 0 { + return nil, errors.New("no certificates found") + } + root.Value = fmt.Sprintf("PEM=%d DER=%d", pemCount, derCount) + return root, nil +} + +func populateCert(f *Field, c *x509.Certificate) { + f.Value = fmt.Sprintf("subject=%q issuer=%q serial=%x", + c.Subject.String(), c.Issuer.String(), c.SerialNumber) + add := func(name, value string) { + f.Children = append(f.Children, &Field{Name: name, Type: "field", Offset: f.Offset, Length: 0, Value: value}) + } + add("subject", c.Subject.String()) + add("issuer", c.Issuer.String()) + add("serial", fmt.Sprintf("%x", c.SerialNumber)) + add("valid_from", c.NotBefore.UTC().Format("2006-01-02 15:04:05 UTC")) + add("valid_until", c.NotAfter.UTC().Format("2006-01-02 15:04:05 UTC")) + add("pubkey_alg", c.PublicKeyAlgorithm.String()) + add("sig_alg", c.SignatureAlgorithm.String()) + if len(c.DNSNames) > 0 { + add("dns_names", strings.Join(c.DNSNames, ", ")) + } + if len(c.IPAddresses) > 0 { + ips := make([]string, 0, len(c.IPAddresses)) + for _, ip := range c.IPAddresses { + ips = append(ips, ip.String()) + } + add("ip_addresses", strings.Join(ips, ", ")) + } + if len(c.EmailAddresses) > 0 { + add("emails", strings.Join(c.EmailAddresses, ", ")) + } + if c.IsCA { + add("ca", "true") + } + if c.KeyUsage != 0 { + add("key_usage", fmt.Sprintf("0x%x", int(c.KeyUsage))) + } + if len(c.ExtKeyUsage) > 0 { + parts := make([]string, 0, len(c.ExtKeyUsage)) + for _, u := range c.ExtKeyUsage { + parts = append(parts, extKeyUsageName(u)) + } + add("ext_key_usage", strings.Join(parts, ", ")) + } + // SHA-256 fingerprint of DER + add("sha256_fingerprint", sha256Hex(c.Raw)) +} + +func extKeyUsageName(u x509.ExtKeyUsage) string { + switch u { + case x509.ExtKeyUsageAny: + return "any" + case x509.ExtKeyUsageServerAuth: + return "serverAuth" + case x509.ExtKeyUsageClientAuth: + return "clientAuth" + case x509.ExtKeyUsageCodeSigning: + return "codeSigning" + case x509.ExtKeyUsageEmailProtection: + return "emailProtection" + case x509.ExtKeyUsageTimeStamping: + return "timeStamping" + case x509.ExtKeyUsageOCSPSigning: + return "OCSPSigning" + } + return fmt.Sprintf("%d", int(u)) +} + +func sha256Hex(b []byte) string { + h := sha256.Sum256(b) + return fmt.Sprintf("%x", h[:]) +} + +// indexOfHeader returns the byte offset of the next PEM BEGIN line. +func indexOfHeader(b []byte) int { + const tag = "-----BEGIN" + for i := 0; i+len(tag) <= len(b); i++ { + if string(b[i:i+len(tag)]) == tag { + return i + } + } + return -1 +} diff --git a/internal/leviathan/parsers/executable.go b/internal/leviathan/parsers/executable.go new file mode 100644 index 0000000..44ac1f3 --- /dev/null +++ b/internal/leviathan/parsers/executable.go @@ -0,0 +1,412 @@ +package parsers + +import ( + "encoding/binary" + "errors" + "fmt" + "sort" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ExecutableRegion is a file-backed executable code range. +type ExecutableRegion struct { + Name string `json:"name"` + Offset int64 `json:"offset"` + Size int64 `json:"size"` + Address uint64 `json:"address"` +} + +// FunctionSeed is a named function entry mapped to a file offset. +type FunctionSeed struct { + Name string `json:"name"` + Offset int64 `json:"offset"` + Address uint64 `json:"address"` + Size uint64 `json:"size"` + Source string `json:"source"` +} + +// ExecutableInfo contains disassembly-oriented metadata for an ELF or PE. +type ExecutableInfo struct { + Format string `json:"format"` + Arch string `json:"arch"` + Entry int64 `json:"entry"` + Regions []ExecutableRegion `json:"regions"` + Seeds []FunctionSeed `json:"seeds"` +} + +// InspectExecutable extracts executable ranges, entry point and named function +// seeds without scraping the display-oriented parser tree. +func InspectExecutable(d *document.Document) (*ExecutableInfo, error) { + head, err := d.Range(0, min64(64, d.Size())) + if err != nil { + return nil, err + } + if len(head) >= 4 && string(head[:4]) == "\x7fELF" { + return inspectELFExecutable(d, head) + } + if len(head) >= 2 && string(head[:2]) == "MZ" { + return inspectPEExecutable(d) + } + return nil, errors.New("not a supported ELF or PE executable") +} + +func inspectELFExecutable(d *document.Document, hdr []byte) (*ExecutableInfo, error) { + if len(hdr) < 52 || (hdr[4] != 1 && hdr[4] != 2) || (hdr[5] != 1 && hdr[5] != 2) { + return nil, errors.New("invalid ELF header") + } + class, data := hdr[4], hdr[5] + bo := elfByteOrder(data) + var machine uint16 + var entry, phoff, shoff uint64 + var phentsize, phnum, shentsize, shnum uint16 + if class == 2 { + if len(hdr) < 64 { + return nil, errors.New("ELF64 header too short") + } + machine = bo.Uint16(hdr[18:20]) + entry = bo.Uint64(hdr[24:32]) + phoff = bo.Uint64(hdr[32:40]) + shoff = bo.Uint64(hdr[40:48]) + phentsize, phnum = bo.Uint16(hdr[54:56]), bo.Uint16(hdr[56:58]) + shentsize, shnum = bo.Uint16(hdr[58:60]), bo.Uint16(hdr[60:62]) + } else { + machine = bo.Uint16(hdr[18:20]) + entry = uint64(bo.Uint32(hdr[24:28])) + phoff = uint64(bo.Uint32(hdr[28:32])) + shoff = uint64(bo.Uint32(hdr[32:36])) + phentsize, phnum = bo.Uint16(hdr[42:44]), bo.Uint16(hdr[44:46]) + shentsize, shnum = bo.Uint16(hdr[46:48]), bo.Uint16(hdr[48:50]) + } + sections, _ := executableELFSections(d, bo, class, shoff, shentsize, shnum) + info := &ExecutableInfo{Format: "elf", Arch: elfDisasmArch(machine, data), Entry: -1} + if info.Arch == "" { + return nil, fmt.Errorf("unsupported ELF machine 0x%x", machine) + } + for i, section := range sections { + if section.flags&0x4 != 0 && validFileRange(d, section.fileOff, section.secSize) { + info.Regions = append(info.Regions, ExecutableRegion{ + Name: fmt.Sprintf("section[%d]", i), Offset: int64(section.fileOff), + Size: int64(section.secSize), Address: section.addr, + }) + } + } + if len(info.Regions) == 0 { + info.Regions = executableELFSegments(d, bo, class, phoff, phentsize, phnum) + } + if off, ok := elfAddressToOffset(sections, entry); ok { + info.Entry = off + info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: off, Address: entry, Source: "header"}) + } else if off, ok := regionAddressToOffset(info.Regions, entry); ok { + info.Entry = off + info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: off, Address: entry, Source: "header"}) + } + for _, symtab := range sections { + if symtab.typ == 2 || symtab.typ == 11 { + info.Seeds = append(info.Seeds, executableELFFunctions(d, bo, class, sections, symtab)...) + } + } + finishExecutableInfo(info) + return info, nil +} + +func executableELFSegments(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entrySize, count uint16) []ExecutableRegion { + if off == 0 || entrySize == 0 || count == 0 || count > 8192 { + return nil + } + regions := make([]ExecutableRegion, 0) + for i := uint16(0); i < count; i++ { + buf, err := d.Range(int64(off)+int64(i)*int64(entrySize), int64(entrySize)) + if err != nil { + break + } + var typ, flags uint32 + var fileOff, address, fileSize uint64 + if class == 2 { + if len(buf) < 56 { + break + } + typ, flags = bo.Uint32(buf[0:4]), bo.Uint32(buf[4:8]) + fileOff, address, fileSize = bo.Uint64(buf[8:16]), bo.Uint64(buf[16:24]), bo.Uint64(buf[32:40]) + } else { + if len(buf) < 32 { + break + } + typ, flags = bo.Uint32(buf[0:4]), bo.Uint32(buf[24:28]) + fileOff, address, fileSize = uint64(bo.Uint32(buf[4:8])), uint64(bo.Uint32(buf[8:12])), uint64(bo.Uint32(buf[16:20])) + } + if typ == 1 && flags&0x1 != 0 && validFileRange(d, fileOff, fileSize) { + regions = append(regions, ExecutableRegion{ + Name: fmt.Sprintf("PT_LOAD[%d]", i), Offset: int64(fileOff), Size: int64(fileSize), Address: address, + }) + } + } + return regions +} + +func executableELFSections(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entrySize, count uint16) ([]elfSectionInfo, error) { + if off == 0 || entrySize == 0 || count == 0 || count > 8192 { + return nil, errors.New("ELF has no usable section table") + } + sections := make([]elfSectionInfo, 0, count) + for i := uint16(0); i < count; i++ { + buf, err := d.Range(int64(off)+int64(i)*int64(entrySize), int64(entrySize)) + if err != nil { + return nil, err + } + var section elfSectionInfo + if class == 2 { + if len(buf) < 64 { + return nil, errors.New("truncated ELF64 section") + } + section.typ, section.flags = bo.Uint32(buf[4:8]), bo.Uint64(buf[8:16]) + section.addr, section.fileOff, section.secSize = bo.Uint64(buf[16:24]), bo.Uint64(buf[24:32]), bo.Uint64(buf[32:40]) + section.link, section.entrySize = bo.Uint32(buf[40:44]), bo.Uint64(buf[56:64]) + } else { + if len(buf) < 40 { + return nil, errors.New("truncated ELF32 section") + } + section.typ, section.flags = bo.Uint32(buf[4:8]), uint64(bo.Uint32(buf[8:12])) + section.addr = uint64(bo.Uint32(buf[12:16])) + section.fileOff, section.secSize = uint64(bo.Uint32(buf[16:20])), uint64(bo.Uint32(buf[20:24])) + section.link, section.entrySize = bo.Uint32(buf[24:28]), uint64(bo.Uint32(buf[36:40])) + } + sections = append(sections, section) + } + return sections, nil +} + +func executableELFFunctions(d *document.Document, bo binary.ByteOrder, class byte, sections []elfSectionInfo, symtab elfSectionInfo) []FunctionSeed { + entrySize := symtab.entrySize + if entrySize == 0 { + entrySize = 16 + if class == 2 { + entrySize = 24 + } + } + if !validFileRange(d, symtab.fileOff, symtab.secSize) || int(symtab.link) >= len(sections) { + return nil + } + strsec := sections[symtab.link] + if !validFileRange(d, strsec.fileOff, strsec.secSize) || strsec.secSize > 8<<20 { + return nil + } + strtab, err := d.Range(int64(strsec.fileOff), int64(strsec.secSize)) + if err != nil { + return nil + } + count := int(symtab.secSize / entrySize) + if count > 16384 { + count = 16384 + } + out := make([]FunctionSeed, 0) + for i := 0; i < count; i++ { + buf, err := d.Range(int64(symtab.fileOff)+int64(i)*int64(entrySize), int64(entrySize)) + if err != nil || len(buf) < int(entrySize) { + break + } + var nameOff uint32 + var info byte + var sectionIndex uint16 + var value, size uint64 + if class == 2 { + nameOff, info, sectionIndex = bo.Uint32(buf[0:4]), buf[4], bo.Uint16(buf[6:8]) + value, size = bo.Uint64(buf[8:16]), bo.Uint64(buf[16:24]) + } else { + nameOff, value, size = bo.Uint32(buf[0:4]), uint64(bo.Uint32(buf[4:8])), uint64(bo.Uint32(buf[8:12])) + info, sectionIndex = buf[12], bo.Uint16(buf[14:16]) + } + if info&0xf != 2 || sectionIndex == 0 || value == 0 { + continue + } + name := elfStringAt(strtab, nameOff) + offset, ok := elfAddressToOffset(sections, value) + if name == "" || !ok { + continue + } + source := "symtab" + if symtab.typ == 11 { + source = "dynsym" + } + out = append(out, FunctionSeed{Name: name, Offset: offset, Address: value, Size: size, Source: source}) + } + return out +} + +func inspectPEExecutable(d *document.Document) (*ExecutableInfo, error) { + sections, dirs, is64, err := peLayout(d) + if err != nil { + return nil, err + } + hdr, _ := d.Range(0, 64) + lfanew := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40])) + peHdr, _ := d.Range(lfanew, 24) + machine := binary.LittleEndian.Uint16(peHdr[4:6]) + optSize := int64(binary.LittleEndian.Uint16(peHdr[20:22])) + opt, err := d.Range(lfanew+24, optSize) + if err != nil || len(opt) < 20 { + return nil, errors.New("PE optional header too short") + } + entryRVA := binary.LittleEndian.Uint32(opt[16:20]) + info := &ExecutableInfo{Format: "pe", Arch: peDisasmArch(machine, is64), Entry: -1} + if info.Arch == "" { + return nil, fmt.Errorf("unsupported PE machine 0x%x", machine) + } + for _, section := range sections { + if section.Characteristics&0x20000000 != 0 && validFileRange(d, uint64(section.RawOffset), uint64(section.RawSize)) { + info.Regions = append(info.Regions, ExecutableRegion{ + Name: section.Name, Offset: int64(section.RawOffset), Size: int64(section.RawSize), Address: uint64(section.VirtualAddress), + }) + } + } + if off, ok := peRVAToOffset(sections, entryRVA); ok { + info.Entry = int64(off) + info.Seeds = append(info.Seeds, FunctionSeed{Name: "entry", Offset: int64(off), Address: uint64(entryRVA), Source: "header"}) + } + if len(dirs) > 0 && dirs[0].RVA != 0 { + info.Seeds = append(info.Seeds, executablePEExports(d, sections, dirs[0])...) + } + finishExecutableInfo(info) + return info, nil +} + +func executablePEExports(d *document.Document, sections []peSectionInfo, dir peDataDirectory) []FunctionSeed { + off, ok := peRVAToOffset(sections, dir.RVA) + if !ok { + return nil + } + buf, err := d.Range(int64(off), 40) + if err != nil || len(buf) < 40 { + return nil + } + functionCount, nameCount := binary.LittleEndian.Uint32(buf[20:24]), binary.LittleEndian.Uint32(buf[24:28]) + functionsOff, funcsOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[28:32])) + namesOff, namesOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[32:36])) + ordinalsOff, ordOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(buf[36:40])) + if !funcsOK || !namesOK || !ordOK { + return nil + } + if nameCount > 16384 { + nameCount = 16384 + } + out := make([]FunctionSeed, 0, nameCount) + for i := uint32(0); i < nameCount; i++ { + namePtr, nameErr := d.Range(int64(namesOff)+int64(i)*4, 4) + ordBuf, ordErr := d.Range(int64(ordinalsOff)+int64(i)*2, 2) + if nameErr != nil || ordErr != nil || len(namePtr) < 4 || len(ordBuf) < 2 { + break + } + index := uint32(binary.LittleEndian.Uint16(ordBuf)) + if index >= functionCount { + continue + } + funcBuf, err := d.Range(int64(functionsOff)+int64(index)*4, 4) + if err != nil || len(funcBuf) < 4 { + break + } + funcRVA := binary.LittleEndian.Uint32(funcBuf) + if funcRVA >= dir.RVA && funcRVA < dir.RVA+dir.Size { + continue + } + nameOff, nameOK := peRVAToOffset(sections, binary.LittleEndian.Uint32(namePtr)) + funcOff, funcOK := peRVAToOffset(sections, funcRVA) + if !nameOK || !funcOK { + continue + } + name := readCString(d, int64(nameOff), 512) + if name != "" { + out = append(out, FunctionSeed{Name: name, Offset: int64(funcOff), Address: uint64(funcRVA), Source: "export"}) + } + } + return out +} + +func validFileRange(d *document.Document, offset, size uint64) bool { + return size > 0 && offset <= uint64(d.Size()) && size <= uint64(d.Size())-offset +} + +func elfAddressToOffset(sections []elfSectionInfo, address uint64) (int64, bool) { + for _, section := range sections { + if section.secSize > 0 && address >= section.addr && address < section.addr+section.secSize { + return int64(section.fileOff + address - section.addr), true + } + } + return 0, false +} + +func regionAddressToOffset(regions []ExecutableRegion, address uint64) (int64, bool) { + for _, region := range regions { + if address >= region.Address && address < region.Address+uint64(region.Size) { + return region.Offset + int64(address-region.Address), true + } + } + return 0, false +} + +func elfDisasmArch(machine uint16, data byte) string { + switch machine { + case 3: + return "x86" + case 8: + if data == 1 { + return "mipsle" + } + return "mips" + case 40: + return "arm32" + case 62: + return "x86_64" + case 183: + return "aarch64" + } + return "" +} + +func peDisasmArch(machine uint16, _ bool) string { + switch machine { + case 0x14c: + return "x86" + case 0x8664: + return "x86_64" + case 0x1c0, 0x1c4: + return "arm32" + case 0xaa64: + return "aarch64" + case 0x166, 0x266, 0x366, 0x466: + return "mipsle" + } + return "" +} + +func finishExecutableInfo(info *ExecutableInfo) { + sort.Slice(info.Regions, func(i, j int) bool { return info.Regions[i].Offset < info.Regions[j].Offset }) + sort.Slice(info.Seeds, func(i, j int) bool { + if info.Seeds[i].Offset != info.Seeds[j].Offset { + return info.Seeds[i].Offset < info.Seeds[j].Offset + } + return info.Seeds[i].Name < info.Seeds[j].Name + }) + unique := info.Seeds[:0] + for _, seed := range info.Seeds { + if seed.Offset < 0 || !offsetInRegions(info.Regions, seed.Offset) { + continue + } + if len(unique) > 0 && unique[len(unique)-1].Offset == seed.Offset { + if unique[len(unique)-1].Name == "entry" && seed.Name != "entry" { + unique[len(unique)-1] = seed + } + continue + } + unique = append(unique, seed) + } + info.Seeds = unique +} + +func offsetInRegions(regions []ExecutableRegion, offset int64) bool { + for _, region := range regions { + if offset >= region.Offset && offset < region.Offset+region.Size { + return true + } + } + return false +} diff --git a/internal/leviathan/parsers/formats.go b/internal/leviathan/parsers/formats.go new file mode 100644 index 0000000..17fd2dd --- /dev/null +++ b/internal/leviathan/parsers/formats.go @@ -0,0 +1,2182 @@ +// Package parsers provides lightweight structure parsers for common +// binary formats used in Leviathan's structures / templates panel. +// Each parser returns a tree of Field values anchored at absolute +// offsets into the document. +package parsers + +import ( + "encoding/binary" + "errors" + "fmt" + "strings" + "unicode/utf16" + "unicode/utf8" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Field is one node of the parsed structure. +type Field struct { + Name string + Type string + Offset int64 + Length int64 + Value string + Note string + Children []*Field +} + +// ToMap serialises a Field tree to JS-friendly maps. +func (f *Field) ToMap() map[string]any { + m := map[string]any{ + "name": f.Name, + "type": f.Type, + "offset": f.Offset, + "length": f.Length, + "value": f.Value, + } + if f.Note != "" { + m["note"] = f.Note + } + if len(f.Children) > 0 { + kids := make([]map[string]any, len(f.Children)) + for i, c := range f.Children { + kids[i] = c.ToMap() + } + m["children"] = kids + } + return m +} + +// DetectAndParse picks the first matching parser for the document. +// It returns the parser name used and the tree. +func DetectAndParse(d *document.Document) (string, *Field, error) { + // Read enough to cover the tar ustar magic at offset 257. + head, err := d.Range(0, min64(265, d.Size())) + if err != nil { + return "", nil, err + } + switch { + case len(head) >= 8 && string(head[:8]) == "\x89PNG\r\n\x1a\n": + f, err := ParsePNG(d) + return "png", f, err + case len(head) >= 3 && head[0] == 0xff && head[1] == 0xd8 && head[2] == 0xff: + f, err := ParseJPEG(d) + return "jpeg", f, err + case len(head) >= 6 && (string(head[:6]) == "GIF87a" || string(head[:6]) == "GIF89a"): + f, err := ParseGIF(d) + return "gif", f, err + case len(head) >= 2 && string(head[:2]) == "BM": + f, err := ParseBMP(d) + return "bmp", f, err + case len(head) >= 4 && string(head[:4]) == "PK\x03\x04": + f, err := ParseZIP(d) + return "zip", f, err + case len(head) >= 4 && string(head[:4]) == "\x7fELF": + f, err := ParseELF(d) + return "elf", f, err + case len(head) >= 2 && string(head[:2]) == "MZ": + f, err := ParsePE(d) + return "pe", f, err + case len(head) >= 4 && string(head[:4]) == "RIFF": + f, err := ParseRIFF(d) + return "riff", f, err + case len(head) >= 16 && string(head[:16]) == "SQLite format 3\x00": + f, err := ParseSQLite(d) + return "sqlite", f, err + case len(head) >= 8 && string(head[:8]) == string(cfbMagic): + f, err := ParseCFB(d) + return "cfb", f, err + case len(head) >= 3 && head[0] == 0x1f && head[1] == 0x8b && head[2] == 8: + f, err := ParseGZIP(d) + return "gzip", f, err + case len(head) >= 4 && string(head[:4]) == "\x0a\x0d\x0d\x0a": + f, err := ParsePCAPNG(d) + return "pcapng", f, err + case len(head) >= 4 && isPCAPMagic(head): + f, err := ParsePCAP(d) + return "pcap", f, err + case len(head) >= 4 && (string(head[:4]) == "II\x2a\x00" || string(head[:4]) == "MM\x00\x2a"): + f, err := ParseTIFF(d) + return "tiff", f, err + case len(head) >= 265 && (string(head[257:262]) == "ustar"): + f, err := ParseTAR(d) + return "tar", f, err + } + // Mach-O (including fat/universal) — detection needs more than 8 bytes. + if IsMachO(head, d.Size()) { + f, err := ParseMachO(d) + return "macho", f, err + } + return "", nil, errors.New("no parser matched") +} + +func isPCAPMagic(head []byte) bool { + m := binary.LittleEndian.Uint32(head[:4]) + return m == 0xa1b2c3d4 || m == 0xd4c3b2a1 || m == 0xa1b23c4d || m == 0x4d3cb2a1 +} + +// ParsePNG walks chunks sequentially and returns a Field tree. +func ParsePNG(d *document.Document) (*Field, error) { + root := &Field{Name: "PNG", Type: "container", Offset: 0} + hdr, err := d.Range(0, 8) + if err != nil { + return nil, err + } + root.Children = append(root.Children, &Field{ + Name: "signature", Type: "bytes", Offset: 0, Length: 8, + Value: fmt.Sprintf("% x", hdr), + }) + off := int64(8) + size := d.Size() + for off+8 <= size { + hdr, err := d.Range(off, 8) + if err != nil { + return root, err + } + ln := int64(binary.BigEndian.Uint32(hdr[0:4])) + typ := string(hdr[4:8]) + chunk := &Field{ + Name: typ + "@" + fmt.Sprintf("0x%x", off), + Type: "chunk", + Offset: off, + Length: 12 + ln, + Value: fmt.Sprintf("length=%d", ln), + } + chunk.Children = []*Field{ + {Name: "length", Type: "u32be", Offset: off, Length: 4, Value: fmt.Sprintf("%d", ln)}, + {Name: "type", Type: "ascii", Offset: off + 4, Length: 4, Value: typ}, + {Name: "data", Type: "bytes", Offset: off + 8, Length: ln, Value: fmt.Sprintf("%d bytes", ln)}, + {Name: "crc", Type: "u32be", Offset: off + 8 + ln, Length: 4, Value: "crc32"}, + } + root.Children = append(root.Children, chunk) + off += 12 + ln + if typ == "IEND" { + break + } + } + root.Length = off + return root, nil +} + +// ParseJPEG walks marker segments. +func ParseJPEG(d *document.Document) (*Field, error) { + root := &Field{Name: "JPEG", Type: "container", Offset: 0} + off := int64(0) + size := d.Size() + for off < size { + b, err := d.Range(off, 2) + if err != nil || len(b) < 2 { + break + } + if b[0] != 0xff { + return root, errors.New("lost sync") + } + marker := b[1] + mark := &Field{ + Name: jpegMarkerName(marker), + Type: "marker", + Offset: off, + Value: fmt.Sprintf("0x%02x", marker), + } + off += 2 + if marker == 0xd8 || marker == 0xd9 || marker == 0x00 || (marker >= 0xd0 && marker <= 0xd7) { + mark.Length = 2 + root.Children = append(root.Children, mark) + if marker == 0xd9 { + break + } + continue + } + lb, err := d.Range(off, 2) + if err != nil || len(lb) < 2 { + break + } + ln := int64(binary.BigEndian.Uint16(lb)) + mark.Length = 2 + ln + mark.Children = []*Field{ + {Name: "length", Type: "u16be", Offset: off, Length: 2, Value: fmt.Sprintf("%d", ln)}, + {Name: "data", Type: "bytes", Offset: off + 2, Length: ln - 2, Value: fmt.Sprintf("%d bytes", ln-2)}, + } + root.Children = append(root.Children, mark) + off += ln + if marker == 0xda { // SOS + // scan until next marker + for off < size { + bb, err := d.Range(off, 2) + if err != nil || len(bb) < 2 { + break + } + if bb[0] == 0xff && bb[1] != 0x00 && !(bb[1] >= 0xd0 && bb[1] <= 0xd7) { + break + } + off++ + } + } + } + root.Length = off + return root, nil +} + +func jpegMarkerName(m byte) string { + switch m { + case 0xd8: + return "SOI" + case 0xd9: + return "EOI" + case 0xda: + return "SOS" + case 0xdb: + return "DQT" + case 0xc4: + return "DHT" + case 0xe0: + return "APP0" + case 0xe1: + return "APP1" + case 0xfe: + return "COM" + } + if m >= 0xc0 && m <= 0xcf { + return fmt.Sprintf("SOF%d", int(m-0xc0)) + } + if m >= 0xe0 && m <= 0xef { + return fmt.Sprintf("APP%d", int(m-0xe0)) + } + return fmt.Sprintf("MRK%02x", m) +} + +// ParseGIF walks the logical screen descriptor, global colour table, +// extension blocks, image descriptors and trailer. +func ParseGIF(d *document.Document) (*Field, error) { + size := d.Size() + if size < 13 { + return nil, errors.New("gif too short") + } + hdr, err := d.Range(0, 13) + if err != nil { + return nil, err + } + version := string(hdr[0:6]) + if version != "GIF87a" && version != "GIF89a" { + return nil, errors.New("bad gif signature") + } + width := binary.LittleEndian.Uint16(hdr[6:8]) + height := binary.LittleEndian.Uint16(hdr[8:10]) + packed := hdr[10] + gctFlag := packed&0x80 != 0 + colorRes := ((packed >> 4) & 0x07) + 1 + sortFlag := packed&0x08 != 0 + gctEntries := 0 + if gctFlag { + gctEntries = 1 << ((packed & 0x07) + 1) + } + root := &Field{ + Name: "GIF", + Type: "container", + Offset: 0, + Value: fmt.Sprintf("%s %dx%d", version, width, height), + Children: []*Field{ + {Name: "signature", Type: "ascii", Offset: 0, Length: 6, Value: version}, + {Name: "width", Type: "u16le", Offset: 6, Length: 2, Value: fmt.Sprintf("%d", width)}, + {Name: "height", Type: "u16le", Offset: 8, Length: 2, Value: fmt.Sprintf("%d", height)}, + {Name: "packed", Type: "u8", Offset: 10, Length: 1, Value: fmt.Sprintf("gct=%v colorRes=%d sort=%v entries=%d", gctFlag, colorRes, sortFlag, gctEntries)}, + {Name: "backgroundColorIndex", Type: "u8", Offset: 11, Length: 1, Value: fmt.Sprintf("%d", hdr[11])}, + {Name: "pixelAspectRatio", Type: "u8", Offset: 12, Length: 1, Value: fmt.Sprintf("%d", hdr[12])}, + }, + } + off := int64(13) + if gctEntries > 0 { + ln := int64(gctEntries * 3) + if off+ln > size { + return root, errors.New("gif global colour table exceeds file") + } + root.Children = append(root.Children, &Field{Name: "globalColorTable", Type: "palette", Offset: off, Length: ln, Value: fmt.Sprintf("%d RGB entries", gctEntries)}) + off += ln + } + for off < size { + tag, err := d.Range(off, 1) + if err != nil || len(tag) < 1 { + break + } + switch tag[0] { + case 0x3b: + root.Children = append(root.Children, &Field{Name: "trailer", Type: "byte", Offset: off, Length: 1, Value: "0x3b"}) + off++ + root.Length = off + return root, nil + case 0x21: + block, next, err := parseGIFExtension(d, off, size) + root.Children = append(root.Children, block) + if err != nil { + root.Length = next + return root, err + } + off = next + case 0x2c: + block, next, err := parseGIFImage(d, off, size) + root.Children = append(root.Children, block) + if err != nil { + root.Length = next + return root, err + } + off = next + default: + root.Children = append(root.Children, &Field{Name: "unknown", Type: "byte", Offset: off, Length: 1, Value: fmt.Sprintf("0x%02x", tag[0])}) + root.Length = off + 1 + return root, fmt.Errorf("unknown gif block 0x%02x at 0x%x", tag[0], off) + } + } + root.Length = off + return root, nil +} + +func parseGIFExtension(d *document.Document, off, size int64) (*Field, int64, error) { + hdr, err := d.Range(off, 2) + if err != nil || len(hdr) < 2 { + return &Field{Name: "extension", Type: "block", Offset: off}, off, errors.New("short gif extension") + } + label := hdr[1] + name := gifExtensionName(label) + next, subBlocks, err := skipGIFSubBlocks(d, off+2, size) + f := &Field{ + Name: name, + Type: "extension", + Offset: off, + Length: next - off, + Value: fmt.Sprintf("label=0x%02x subBlocks=%d", label, subBlocks), + Children: []*Field{ + {Name: "introducer", Type: "u8", Offset: off, Length: 1, Value: "0x21"}, + {Name: "label", Type: "u8", Offset: off + 1, Length: 1, Value: fmt.Sprintf("0x%02x", label)}, + }, + } + return f, next, err +} + +func parseGIFImage(d *document.Document, off, size int64) (*Field, int64, error) { + hdr, err := d.Range(off, 10) + if err != nil || len(hdr) < 10 { + return &Field{Name: "image", Type: "image", Offset: off}, off, errors.New("short gif image descriptor") + } + left := binary.LittleEndian.Uint16(hdr[1:3]) + top := binary.LittleEndian.Uint16(hdr[3:5]) + width := binary.LittleEndian.Uint16(hdr[5:7]) + height := binary.LittleEndian.Uint16(hdr[7:9]) + packed := hdr[9] + lctEntries := 0 + if packed&0x80 != 0 { + lctEntries = 1 << ((packed & 0x07) + 1) + } + next := off + 10 + int64(lctEntries*3) + if next >= size { + return nil, next, errors.New("gif image data missing") + } + lzw, err := d.Range(next, 1) + if err != nil || len(lzw) < 1 { + return nil, next, errors.New("gif lzw minimum code size missing") + } + dataStart := next + 1 + next, subBlocks, err := skipGIFSubBlocks(d, dataStart, size) + f := &Field{ + Name: fmt.Sprintf("image@0x%x", off), + Type: "image", + Offset: off, + Length: next - off, + Value: fmt.Sprintf("%dx%d at %d,%d subBlocks=%d", width, height, left, top, subBlocks), + Children: []*Field{ + {Name: "left", Type: "u16le", Offset: off + 1, Length: 2, Value: fmt.Sprintf("%d", left)}, + {Name: "top", Type: "u16le", Offset: off + 3, Length: 2, Value: fmt.Sprintf("%d", top)}, + {Name: "width", Type: "u16le", Offset: off + 5, Length: 2, Value: fmt.Sprintf("%d", width)}, + {Name: "height", Type: "u16le", Offset: off + 7, Length: 2, Value: fmt.Sprintf("%d", height)}, + {Name: "packed", Type: "u8", Offset: off + 9, Length: 1, Value: fmt.Sprintf("lctEntries=%d interlace=%v", lctEntries, packed&0x40 != 0)}, + {Name: "lzwMinimumCodeSize", Type: "u8", Offset: dataStart - 1, Length: 1, Value: fmt.Sprintf("%d", lzw[0])}, + {Name: "imageData", Type: "subblocks", Offset: dataStart, Length: next - dataStart, Value: fmt.Sprintf("%d sub-blocks", subBlocks)}, + }, + } + if lctEntries > 0 { + f.Children = append(f.Children, &Field{Name: "localColorTable", Type: "palette", Offset: off + 10, Length: int64(lctEntries * 3), Value: fmt.Sprintf("%d RGB entries", lctEntries)}) + } + return f, next, err +} + +func skipGIFSubBlocks(d *document.Document, off, size int64) (int64, int, error) { + count := 0 + for off < size { + b, err := d.Range(off, 1) + if err != nil || len(b) < 1 { + return off, count, errors.New("short gif sub-block") + } + off++ + ln := int64(b[0]) + if ln == 0 { + return off, count, nil + } + if off+ln > size { + return off, count, errors.New("gif sub-block exceeds file") + } + off += ln + count++ + } + return off, count, errors.New("unterminated gif sub-block chain") +} + +func gifExtensionName(label byte) string { + switch label { + case 0xf9: + return "graphicControlExtension" + case 0xfe: + return "commentExtension" + case 0xff: + return "applicationExtension" + case 0x01: + return "plainTextExtension" + } + return fmt.Sprintf("extension%02x", label) +} + +// ParseBMP reads the BMP file header and DIB header metadata. +func ParseBMP(d *document.Document) (*Field, error) { + size := d.Size() + if size < 26 { + return nil, errors.New("bmp too short") + } + hdr, err := d.Range(0, min64(size, 138)) + if err != nil { + return nil, err + } + if len(hdr) < 26 || string(hdr[0:2]) != "BM" { + return nil, errors.New("bad bmp signature") + } + fileSize := binary.LittleEndian.Uint32(hdr[2:6]) + pixelOff := binary.LittleEndian.Uint32(hdr[10:14]) + dibSize := binary.LittleEndian.Uint32(hdr[14:18]) + root := &Field{ + Name: "BMP", + Type: "container", + Offset: 0, + Length: int64(fileSize), + Value: fmt.Sprintf("fileSize=%d pixelOffset=0x%x dib=%s", fileSize, pixelOff, bmpDIBName(dibSize)), + Children: []*Field{ + {Name: "signature", Type: "ascii", Offset: 0, Length: 2, Value: "BM"}, + {Name: "fileSize", Type: "u32le", Offset: 2, Length: 4, Value: fmt.Sprintf("%d", fileSize)}, + {Name: "pixelDataOffset", Type: "u32le", Offset: 10, Length: 4, Value: fmt.Sprintf("0x%x", pixelOff)}, + {Name: "dibHeaderSize", Type: "u32le", Offset: 14, Length: 4, Value: fmt.Sprintf("%d (%s)", dibSize, bmpDIBName(dibSize))}, + }, + } + if dibSize == 12 { + if len(hdr) < 26 { + return root, nil + } + width := binary.LittleEndian.Uint16(hdr[18:20]) + height := binary.LittleEndian.Uint16(hdr[20:22]) + planes := binary.LittleEndian.Uint16(hdr[22:24]) + bpp := binary.LittleEndian.Uint16(hdr[24:26]) + root.Children = append(root.Children, + &Field{Name: "width", Type: "u16le", Offset: 18, Length: 2, Value: fmt.Sprintf("%d", width)}, + &Field{Name: "height", Type: "u16le", Offset: 20, Length: 2, Value: fmt.Sprintf("%d", height)}, + &Field{Name: "planes", Type: "u16le", Offset: 22, Length: 2, Value: fmt.Sprintf("%d", planes)}, + &Field{Name: "bitsPerPixel", Type: "u16le", Offset: 24, Length: 2, Value: fmt.Sprintf("%d", bpp)}, + ) + } else if dibSize >= 40 && len(hdr) >= 54 { + width := int32(binary.LittleEndian.Uint32(hdr[18:22])) + height := int32(binary.LittleEndian.Uint32(hdr[22:26])) + planes := binary.LittleEndian.Uint16(hdr[26:28]) + bpp := binary.LittleEndian.Uint16(hdr[28:30]) + compression := binary.LittleEndian.Uint32(hdr[30:34]) + imageSize := binary.LittleEndian.Uint32(hdr[34:38]) + xppm := int32(binary.LittleEndian.Uint32(hdr[38:42])) + yppm := int32(binary.LittleEndian.Uint32(hdr[42:46])) + coloursUsed := binary.LittleEndian.Uint32(hdr[46:50]) + root.Children = append(root.Children, + &Field{Name: "width", Type: "i32le", Offset: 18, Length: 4, Value: fmt.Sprintf("%d", width)}, + &Field{Name: "height", Type: "i32le", Offset: 22, Length: 4, Value: fmt.Sprintf("%d", height)}, + &Field{Name: "planes", Type: "u16le", Offset: 26, Length: 2, Value: fmt.Sprintf("%d", planes)}, + &Field{Name: "bitsPerPixel", Type: "u16le", Offset: 28, Length: 2, Value: fmt.Sprintf("%d", bpp)}, + &Field{Name: "compression", Type: "u32le", Offset: 30, Length: 4, Value: fmt.Sprintf("%d (%s)", compression, bmpCompressionName(compression))}, + &Field{Name: "imageSize", Type: "u32le", Offset: 34, Length: 4, Value: fmt.Sprintf("%d", imageSize)}, + &Field{Name: "xPixelsPerMeter", Type: "i32le", Offset: 38, Length: 4, Value: fmt.Sprintf("%d", xppm)}, + &Field{Name: "yPixelsPerMeter", Type: "i32le", Offset: 42, Length: 4, Value: fmt.Sprintf("%d", yppm)}, + &Field{Name: "coloursUsed", Type: "u32le", Offset: 46, Length: 4, Value: fmt.Sprintf("%d", coloursUsed)}, + ) + } + if int64(pixelOff) < size { + root.Children = append(root.Children, &Field{Name: "pixelData", Type: "bytes", Offset: int64(pixelOff), Length: size - int64(pixelOff), Value: fmt.Sprintf("%d bytes", size-int64(pixelOff))}) + } + return root, nil +} + +func bmpDIBName(size uint32) string { + switch size { + case 12: + return "BITMAPCOREHEADER" + case 40: + return "BITMAPINFOHEADER" + case 52: + return "BITMAPV2INFOHEADER" + case 56: + return "BITMAPV3INFOHEADER" + case 108: + return "BITMAPV4HEADER" + case 124: + return "BITMAPV5HEADER" + } + return "unknown" +} + +func bmpCompressionName(v uint32) string { + switch v { + case 0: + return "BI_RGB" + case 1: + return "BI_RLE8" + case 2: + return "BI_RLE4" + case 3: + return "BI_BITFIELDS" + case 4: + return "BI_JPEG" + case 5: + return "BI_PNG" + } + return "unknown" +} + +// ParseZIP walks local file headers by seeking forward. +func ParseZIP(d *document.Document) (*Field, error) { + root := &Field{Name: "ZIP", Type: "container", Offset: 0, Length: d.Size()} + size := d.Size() + + // 1) Local file headers, walked from the front. + off := int64(0) + entries := 0 + for off+30 <= size && entries < 5000 { + sig, err := d.Range(off, 4) + if err != nil || string(sig) != "PK\x03\x04" { + break // reached the central directory / EOCD region + } + hdr, err := d.Range(off, 30) + if err != nil { + break + } + flags := binary.LittleEndian.Uint16(hdr[6:8]) + method := binary.LittleEndian.Uint16(hdr[8:10]) + dosTime := binary.LittleEndian.Uint16(hdr[10:12]) + dosDate := binary.LittleEndian.Uint16(hdr[12:14]) + crc := binary.LittleEndian.Uint32(hdr[14:18]) + compSize := int64(binary.LittleEndian.Uint32(hdr[18:22])) + uncompSize := int64(binary.LittleEndian.Uint32(hdr[22:26])) + nameLen := int64(binary.LittleEndian.Uint16(hdr[26:28])) + extraLen := int64(binary.LittleEndian.Uint16(hdr[28:30])) + nameB, _ := d.Range(off+30, nameLen) + e := &Field{ + Name: string(nameB), + Type: "entry", + Offset: off, + Length: 30 + nameLen + extraLen + compSize, + Value: fmt.Sprintf("%s comp=%d uncomp=%d", zipMethod(method), compSize, uncompSize), + Children: []*Field{ + {Name: "signature", Type: "bytes", Offset: off, Length: 4, Value: "PK\\x03\\x04"}, + {Name: "version_needed", Type: "u16le", Offset: off + 4, Length: 2, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint16(hdr[4:6]))}, + {Name: "flags", Type: "u16le", Offset: off + 6, Length: 2, Value: zipFlags(flags)}, + {Name: "method", Type: "u16le", Offset: off + 8, Length: 2, Value: zipMethod(method)}, + {Name: "mod_time", Type: "dostime", Offset: off + 10, Length: 4, Value: zipDOSDateTime(dosDate, dosTime)}, + {Name: "crc32", Type: "u32le", Offset: off + 14, Length: 4, Value: fmt.Sprintf("0x%08x", crc)}, + {Name: "compressed_size", Type: "u32le", Offset: off + 18, Length: 4, Value: fmt.Sprintf("%d", compSize)}, + {Name: "uncompressed_size", Type: "u32le", Offset: off + 22, Length: 4, Value: fmt.Sprintf("%d", uncompSize)}, + {Name: "name", Type: "ascii", Offset: off + 30, Length: nameLen, Value: string(nameB)}, + {Name: "data", Type: "bytes", Offset: off + 30 + nameLen + extraLen, Length: compSize, Value: fmt.Sprintf("%d bytes", compSize)}, + }, + } + root.Children = append(root.Children, e) + off += 30 + nameLen + extraLen + compSize + entries++ + } + + // 2) Central directory + end-of-central-directory, located from the back. + if cd := zipCentralDirectory(d); cd != nil { + root.Children = append(root.Children, cd...) + } + return root, nil +} + +// zipCentralDirectory finds the EOCD record, decodes it, and walks the central +// directory it points at. Returns the EOCD field plus a "central_directory" +// container, or nil if no EOCD is present. +func zipCentralDirectory(d *document.Document) []*Field { + size := d.Size() + // EOCD is at most 22 + 65535 bytes from the end; scan a bounded tail. + scan := int64(22 + 65535) + if scan > size { + scan = size + } + tail, err := d.Range(size-scan, scan) + if err != nil { + return nil + } + eocdRel := int64(-1) + for i := len(tail) - 22; i >= 0; i-- { + if tail[i] == 'P' && tail[i+1] == 'K' && tail[i+2] == 0x05 && tail[i+3] == 0x06 { + eocdRel = int64(i) + break + } + } + if eocdRel < 0 { + return nil + } + eocdOff := size - scan + eocdRel + e, _ := d.Range(eocdOff, 22) + if len(e) < 22 { + return nil + } + total := binary.LittleEndian.Uint16(e[10:12]) + cdSize := int64(binary.LittleEndian.Uint32(e[12:16])) + cdOff := int64(binary.LittleEndian.Uint32(e[16:20])) + commentLen := int64(binary.LittleEndian.Uint16(e[20:22])) + eocd := &Field{ + Name: "end_of_central_directory", Type: "eocd", Offset: eocdOff, Length: 22 + commentLen, + Value: fmt.Sprintf("%d entries, cd@0x%x size=%d", total, cdOff, cdSize), + Children: []*Field{ + {Name: "total_entries", Type: "u16le", Offset: eocdOff + 10, Length: 2, Value: fmt.Sprintf("%d", total)}, + {Name: "cd_size", Type: "u32le", Offset: eocdOff + 12, Length: 4, Value: fmt.Sprintf("%d", cdSize)}, + {Name: "cd_offset", Type: "u32le", Offset: eocdOff + 16, Length: 4, Value: fmt.Sprintf("0x%x", cdOff)}, + {Name: "comment_length", Type: "u16le", Offset: eocdOff + 20, Length: 2, Value: fmt.Sprintf("%d", commentLen)}, + }, + } + out := []*Field{} + + // ZIP64: a 0xffffffff offset or 0xffff count signals a ZIP64 EOCD locator. + if cdOff == 0xffffffff || total == 0xffff { + if eocdOff-20 >= 0 { + loc, _ := d.Range(eocdOff-20, 20) + if len(loc) == 20 && string(loc[0:4]) == "PK\x06\x07" { + z64Off := int64(binary.LittleEndian.Uint64(loc[8:16])) + eocd.Children = append(eocd.Children, &Field{Name: "zip64_locator", Type: "bytes", Offset: eocdOff - 20, Length: 20, Value: fmt.Sprintf("zip64 EOCD @0x%x", z64Off)}) + if z64, _ := d.Range(z64Off, 56); len(z64) == 56 && string(z64[0:4]) == "PK\x06\x06" { + cdOff = int64(binary.LittleEndian.Uint64(z64[48:56])) + cdSize = int64(binary.LittleEndian.Uint64(z64[40:48])) + out = append(out, &Field{Name: "zip64_eocd", Type: "zip64", Offset: z64Off, Length: 56, Value: fmt.Sprintf("entries=%d cd@0x%x", binary.LittleEndian.Uint64(z64[32:40]), cdOff)}) + } + } + } + } + + // Walk central-directory file headers. + if cdOff >= 0 && cdOff < size { + cd := &Field{Name: "central_directory", Type: "directory", Offset: cdOff, Length: cdSize} + off := cdOff + end := size + if cdSize > 0 && cdOff+cdSize <= size { + end = cdOff + cdSize + } + for n := 0; off+46 <= end && n < 5000; n++ { + h, err := d.Range(off, 46) + if err != nil || string(h[0:4]) != "PK\x01\x02" { + break + } + method := binary.LittleEndian.Uint16(h[10:12]) + crc := binary.LittleEndian.Uint32(h[16:20]) + compSize := int64(binary.LittleEndian.Uint32(h[20:24])) + nameLen := int64(binary.LittleEndian.Uint16(h[28:30])) + extraLen := int64(binary.LittleEndian.Uint16(h[30:32])) + commLen := int64(binary.LittleEndian.Uint16(h[32:34])) + lho := binary.LittleEndian.Uint32(h[42:46]) + nameB, _ := d.Range(off+46, nameLen) + cd.Children = append(cd.Children, &Field{ + Name: string(nameB), Type: "cd_entry", Offset: off, Length: 46 + nameLen + extraLen + commLen, + Value: fmt.Sprintf("%s crc=0x%08x comp=%d lfh@0x%x", zipMethod(method), crc, compSize, lho), + }) + off += 46 + nameLen + extraLen + commLen + } + out = append(out, cd) + } + out = append(out, eocd) + return out +} + +func zipMethod(m uint16) string { + names := map[uint16]string{0: "store", 1: "shrink", 6: "implode", 8: "deflate", 9: "deflate64", 12: "bzip2", 14: "lzma", 95: "xz", 96: "jpeg", 97: "wavpack", 98: "ppmd", 99: "aes"} + if n, ok := names[m]; ok { + return fmt.Sprintf("%d (%s)", m, n) + } + return fmt.Sprintf("%d", m) +} + +func zipFlags(f uint16) string { + var on []string + if f&0x0001 != 0 { + on = append(on, "encrypted") + } + if f&0x0008 != 0 { + on = append(on, "data-descriptor") + } + if f&0x0800 != 0 { + on = append(on, "utf8") + } + if len(on) == 0 { + return fmt.Sprintf("0x%04x", f) + } + return fmt.Sprintf("0x%04x (%s)", f, strings.Join(on, "|")) +} + +// zipDOSDateTime decodes the MS-DOS packed date/time used by ZIP entries. +func zipDOSDateTime(date, t uint16) string { + if date == 0 && t == 0 { + return "0" + } + year := 1980 + int(date>>9) + month := int(date>>5) & 0x0f + day := int(date) & 0x1f + hour := int(t >> 11) + minute := int(t>>5) & 0x3f + second := (int(t) & 0x1f) * 2 + return fmt.Sprintf("%04d-%02d-%02d %02d:%02d:%02d", year, month, day, hour, minute, second) +} + +// ParseELF returns header, program header, and section header fields. +func ParseELF(d *document.Document) (*Field, error) { + hdr, err := d.Range(0, 64) + if err != nil || len(hdr) < 52 { + return nil, errors.New("elf too short") + } + class := hdr[4] + data := hdr[5] + kind := "32-bit" + if class == 2 { + kind = "64-bit" + } + endian := "little-endian" + if data == 2 { + endian = "big-endian" + } + root := &Field{ + Name: "ELF", + Type: "container", + Offset: 0, + Length: d.Size(), + Value: kind + " " + endian, + Children: []*Field{ + {Name: "ei_mag", Type: "bytes", Offset: 0, Length: 4, Value: "\\x7fELF"}, + {Name: "ei_class", Type: "u8", Offset: 4, Length: 1, Value: kind}, + {Name: "ei_data", Type: "u8", Offset: 5, Length: 1, Value: endian}, + {Name: "ei_version", Type: "u8", Offset: 6, Length: 1, Value: fmt.Sprintf("%d", hdr[6])}, + {Name: "ei_osabi", Type: "u8", Offset: 7, Length: 1, Value: fmt.Sprintf("%d", hdr[7])}, + }, + } + var e_type, e_machine uint16 + var e_entry uint64 + var e_phoff, e_shoff uint64 + var e_phentsize, e_phnum, e_shentsize, e_shnum, e_shstrndx uint16 + bo := elfByteOrder(data) + if class == 2 { + e_type = bo.Uint16(hdr[16:18]) + e_machine = bo.Uint16(hdr[18:20]) + e_entry = bo.Uint64(hdr[24:32]) + e_phoff = bo.Uint64(hdr[32:40]) + e_shoff = bo.Uint64(hdr[40:48]) + e_phentsize = bo.Uint16(hdr[54:56]) + e_phnum = bo.Uint16(hdr[56:58]) + e_shentsize = bo.Uint16(hdr[58:60]) + e_shnum = bo.Uint16(hdr[60:62]) + e_shstrndx = bo.Uint16(hdr[62:64]) + } else { + e_type = bo.Uint16(hdr[16:18]) + e_machine = bo.Uint16(hdr[18:20]) + e_entry = uint64(bo.Uint32(hdr[24:28])) + e_phoff = uint64(bo.Uint32(hdr[28:32])) + e_shoff = uint64(bo.Uint32(hdr[32:36])) + e_phentsize = bo.Uint16(hdr[42:44]) + e_phnum = bo.Uint16(hdr[44:46]) + e_shentsize = bo.Uint16(hdr[46:48]) + e_shnum = bo.Uint16(hdr[48:50]) + e_shstrndx = bo.Uint16(hdr[50:52]) + } + root.Children = append(root.Children, + &Field{Name: "e_type", Value: fmt.Sprintf("%d (%s)", e_type, elfType(e_type))}, + &Field{Name: "e_machine", Value: fmt.Sprintf("%d (%s)", e_machine, elfMachine(e_machine))}, + &Field{Name: "e_entry", Value: fmt.Sprintf("0x%x", e_entry)}, + &Field{Name: "e_phoff", Value: fmt.Sprintf("0x%x", e_phoff)}, + &Field{Name: "e_shoff", Value: fmt.Sprintf("0x%x", e_shoff)}, + &Field{Name: "e_phentsize", Value: fmt.Sprintf("%d", e_phentsize)}, + &Field{Name: "e_phnum", Value: fmt.Sprintf("%d", e_phnum)}, + &Field{Name: "e_shentsize", Value: fmt.Sprintf("%d", e_shentsize)}, + &Field{Name: "e_shnum", Value: fmt.Sprintf("%d", e_shnum)}, + &Field{Name: "e_shstrndx", Value: fmt.Sprintf("%d", e_shstrndx)}, + ) + root.Children = append(root.Children, elfProgramHeaders(d, bo, class, e_phoff, e_phentsize, e_phnum)...) + root.Children = append(root.Children, elfSectionHeaders(d, bo, class, e_shoff, e_shentsize, e_shnum, e_shstrndx)...) + return root, nil +} + +func elfByteOrder(data byte) binary.ByteOrder { + if data == 2 { + return binary.BigEndian + } + return binary.LittleEndian +} + +func elfProgramHeaders(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entsize, count uint16) []*Field { + if off == 0 || entsize == 0 || count == 0 { + return nil + } + size := d.Size() + out := make([]*Field, 0, count) + for i := uint16(0); i < count; i++ { + at := int64(off) + int64(i)*int64(entsize) + if at < 0 || at+int64(entsize) > size { + break + } + buf, err := d.Range(at, int64(entsize)) + if err != nil { + break + } + var typ, flags uint32 + var pOffset, vaddr, filesz, memsz, align uint64 + if class == 2 { + if len(buf) < 56 { + break + } + typ = bo.Uint32(buf[0:4]) + flags = bo.Uint32(buf[4:8]) + pOffset = bo.Uint64(buf[8:16]) + vaddr = bo.Uint64(buf[16:24]) + filesz = bo.Uint64(buf[32:40]) + memsz = bo.Uint64(buf[40:48]) + align = bo.Uint64(buf[48:56]) + } else { + if len(buf) < 32 { + break + } + typ = bo.Uint32(buf[0:4]) + pOffset = uint64(bo.Uint32(buf[4:8])) + vaddr = uint64(bo.Uint32(buf[8:12])) + filesz = uint64(bo.Uint32(buf[16:20])) + memsz = uint64(bo.Uint32(buf[20:24])) + flags = bo.Uint32(buf[24:28]) + align = uint64(bo.Uint32(buf[28:32])) + } + out = append(out, &Field{ + Name: fmt.Sprintf("program[%d]", i), + Type: "program_header", + Offset: at, + Length: int64(entsize), + Value: fmt.Sprintf("type=%s offset=0x%x vaddr=0x%x filesz=%d memsz=%d flags=%s align=0x%x", elfProgramType(typ), pOffset, vaddr, filesz, memsz, elfProgramFlags(flags), align), + }) + } + return out +} + +type elfSectionInfo struct { + nameOff uint32 + typ uint32 + flags, addr uint64 + fileOff, secSize uint64 + link, info uint32 + align, entrySize uint64 + at int64 + length int64 +} + +func elfSectionHeaders(d *document.Document, bo binary.ByteOrder, class byte, off uint64, entsize, count, strIndex uint16) []*Field { + if off == 0 || entsize == 0 || count == 0 { + return nil + } + size := d.Size() + sections := make([]elfSectionInfo, 0, count) + for i := uint16(0); i < count; i++ { + at := int64(off) + int64(i)*int64(entsize) + if at < 0 || at+int64(entsize) > size { + break + } + buf, err := d.Range(at, int64(entsize)) + if err != nil { + break + } + var s elfSectionInfo + s.at = at + s.length = int64(entsize) + if class == 2 { + if len(buf) < 64 { + break + } + s.nameOff = bo.Uint32(buf[0:4]) + s.typ = bo.Uint32(buf[4:8]) + s.flags = bo.Uint64(buf[8:16]) + s.addr = bo.Uint64(buf[16:24]) + s.fileOff = bo.Uint64(buf[24:32]) + s.secSize = bo.Uint64(buf[32:40]) + s.link = bo.Uint32(buf[40:44]) + s.info = bo.Uint32(buf[44:48]) + s.align = bo.Uint64(buf[48:56]) + s.entrySize = bo.Uint64(buf[56:64]) + } else { + if len(buf) < 40 { + break + } + s.nameOff = bo.Uint32(buf[0:4]) + s.typ = bo.Uint32(buf[4:8]) + s.flags = uint64(bo.Uint32(buf[8:12])) + s.addr = uint64(bo.Uint32(buf[12:16])) + s.fileOff = uint64(bo.Uint32(buf[16:20])) + s.secSize = uint64(bo.Uint32(buf[20:24])) + s.link = bo.Uint32(buf[24:28]) + s.info = bo.Uint32(buf[28:32]) + s.align = uint64(bo.Uint32(buf[32:36])) + s.entrySize = uint64(bo.Uint32(buf[36:40])) + } + sections = append(sections, s) + } + var names []byte + if int(strIndex) < len(sections) { + str := sections[strIndex] + if str.fileOff <= uint64(size) && str.fileOff+str.secSize <= uint64(size) && str.secSize <= 1<<20 { + names, _ = d.Range(int64(str.fileOff), int64(str.secSize)) + } + } + sectionNames := make([]string, len(sections)) + for i, s := range sections { + name := elfStringAt(names, s.nameOff) + if name == "" { + name = fmt.Sprintf("section[%d]", i) + } + sectionNames[i] = name + } + out := make([]*Field, 0, len(sections)) + for i, s := range sections { + name := sectionNames[i] + children := []*Field{} + if s.typ == 6 { + children = elfDynamicEntries(d, bo, class, sections, s) + } else if s.typ == 2 || s.typ == 11 { + children = elfSymbolEntries(d, bo, class, sections, s) + } + out = append(out, &Field{ + Name: name, + Type: "section_header", + Offset: s.at, + Length: s.length, + Value: fmt.Sprintf("type=%s offset=0x%x size=%d addr=0x%x flags=%s link=%d info=%d align=0x%x entsize=%d", elfSectionType(s.typ), s.fileOff, s.secSize, s.addr, elfSectionFlags(s.flags), s.link, s.info, s.align, s.entrySize), + Children: children, + }) + } + return out +} + +func elfSymbolEntries(d *document.Document, bo binary.ByteOrder, class byte, sections []elfSectionInfo, symtab elfSectionInfo) []*Field { + entrySize := symtab.entrySize + if entrySize == 0 { + if class == 2 { + entrySize = 24 + } else { + entrySize = 16 + } + } + if entrySize == 0 || symtab.secSize == 0 || symtab.fileOff+symtab.secSize > uint64(d.Size()) { + return nil + } + var strtab []byte + if int(symtab.link) < len(sections) { + linked := sections[symtab.link] + if linked.fileOff+linked.secSize <= uint64(d.Size()) && linked.secSize <= 8<<20 { + strtab, _ = d.Range(int64(linked.fileOff), int64(linked.secSize)) + } + } + count := int(symtab.secSize / entrySize) + if count > 4096 { + count = 4096 + } + out := []*Field{} + for i := 0; i < count; i++ { + at := int64(symtab.fileOff + uint64(i)*entrySize) + buf, err := d.Range(at, int64(entrySize)) + if err != nil || len(buf) < int(entrySize) { + break + } + var nameOff uint32 + var info, other byte + var shndx uint16 + var value, size uint64 + if class == 2 { + nameOff = bo.Uint32(buf[0:4]) + info = buf[4] + other = buf[5] + shndx = bo.Uint16(buf[6:8]) + value = bo.Uint64(buf[8:16]) + size = bo.Uint64(buf[16:24]) + } else { + nameOff = bo.Uint32(buf[0:4]) + value = uint64(bo.Uint32(buf[4:8])) + size = uint64(bo.Uint32(buf[8:12])) + info = buf[12] + other = buf[13] + shndx = bo.Uint16(buf[14:16]) + } + name := elfStringAt(strtab, nameOff) + if name == "" { + name = fmt.Sprintf("", i) + } + out = append(out, &Field{ + Name: fmt.Sprintf("symbol[%d] %s", i, name), + Type: "symbol", + Offset: at, + Length: int64(entrySize), + Value: fmt.Sprintf("name=%s bind=%s type=%s vis=%s shndx=%s value=0x%x size=%d", + name, elfSymbolBind(info>>4), elfSymbolType(info&0xf), elfSymbolVisibility(other&0x3), elfSymbolSection(shndx), value, size), + }) + } + return out +} + +func elfDynamicEntries(d *document.Document, bo binary.ByteOrder, class byte, sections []elfSectionInfo, dyn elfSectionInfo) []*Field { + entrySize := dyn.entrySize + if entrySize == 0 { + if class == 2 { + entrySize = 16 + } else { + entrySize = 8 + } + } + if entrySize == 0 || dyn.secSize == 0 || dyn.fileOff+dyn.secSize > uint64(d.Size()) { + return nil + } + var strtab []byte + if int(dyn.link) < len(sections) { + linked := sections[dyn.link] + if linked.fileOff+linked.secSize <= uint64(d.Size()) && linked.secSize <= 8<<20 { + strtab, _ = d.Range(int64(linked.fileOff), int64(linked.secSize)) + } + } + out := []*Field{} + count := int(dyn.secSize / entrySize) + if count > 4096 { + count = 4096 + } + for i := 0; i < count; i++ { + at := int64(dyn.fileOff + uint64(i)*entrySize) + buf, err := d.Range(at, int64(entrySize)) + if err != nil || len(buf) < int(entrySize) { + break + } + var tag int64 + var val uint64 + if class == 2 { + tag = int64(bo.Uint64(buf[0:8])) + val = bo.Uint64(buf[8:16]) + } else { + tag = int64(int32(bo.Uint32(buf[0:4]))) + val = uint64(bo.Uint32(buf[4:8])) + } + value := fmt.Sprintf("%s=0x%x", elfDynamicTag(tag), val) + if tag == 1 && len(strtab) > 0 { + if s := elfStringAt(strtab, uint32(val)); s != "" { + value = "NEEDED=" + s + } + } + out = append(out, &Field{ + Name: fmt.Sprintf("dynamic[%d]", i), + Type: "dynamic_entry", + Offset: at, + Length: int64(entrySize), + Value: value, + }) + if tag == 0 { + break + } + } + return out +} + +func elfSymbolBind(bind byte) string { + switch bind { + case 0: + return "LOCAL" + case 1: + return "GLOBAL" + case 2: + return "WEAK" + } + return fmt.Sprintf("0x%x", bind) +} + +func elfSymbolType(typ byte) string { + switch typ { + case 0: + return "NOTYPE" + case 1: + return "OBJECT" + case 2: + return "FUNC" + case 3: + return "SECTION" + case 4: + return "FILE" + case 5: + return "COMMON" + case 6: + return "TLS" + } + return fmt.Sprintf("0x%x", typ) +} + +func elfSymbolVisibility(vis byte) string { + switch vis { + case 0: + return "DEFAULT" + case 1: + return "INTERNAL" + case 2: + return "HIDDEN" + case 3: + return "PROTECTED" + } + return fmt.Sprintf("0x%x", vis) +} + +func elfSymbolSection(idx uint16) string { + switch idx { + case 0: + return "UND" + case 0xfff1: + return "ABS" + case 0xfff2: + return "COMMON" + } + return fmt.Sprintf("%d", idx) +} + +func elfDynamicTag(tag int64) string { + switch tag { + case 0: + return "NULL" + case 1: + return "NEEDED" + case 2: + return "PLTRELSZ" + case 3: + return "PLTGOT" + case 4: + return "HASH" + case 5: + return "STRTAB" + case 6: + return "SYMTAB" + case 7: + return "RELA" + case 8: + return "RELASZ" + case 9: + return "RELAENT" + case 10: + return "STRSZ" + case 11: + return "SYMENT" + case 12: + return "INIT" + case 13: + return "FINI" + case 14: + return "SONAME" + case 15: + return "RPATH" + case 16: + return "SYMBOLIC" + case 17: + return "REL" + case 18: + return "RELSZ" + case 19: + return "RELENT" + case 20: + return "PLTREL" + case 21: + return "DEBUG" + case 22: + return "TEXTREL" + case 23: + return "JMPREL" + case 29: + return "RUNPATH" + } + return fmt.Sprintf("0x%x", tag) +} + +func elfStringAt(buf []byte, off uint32) string { + if int(off) >= len(buf) { + return "" + } + end := int(off) + for end < len(buf) && buf[end] != 0 { + end++ + } + return string(buf[off:end]) +} + +func elfProgramType(t uint32) string { + switch t { + case 0: + return "NULL" + case 1: + return "LOAD" + case 2: + return "DYNAMIC" + case 3: + return "INTERP" + case 4: + return "NOTE" + case 5: + return "SHLIB" + case 6: + return "PHDR" + case 7: + return "TLS" + } + return fmt.Sprintf("0x%x", t) +} + +func elfProgramFlags(flags uint32) string { + var out string + if flags&4 != 0 { + out += "R" + } + if flags&2 != 0 { + out += "W" + } + if flags&1 != 0 { + out += "X" + } + if out == "" { + return "-" + } + return out +} + +func elfSectionType(t uint32) string { + switch t { + case 0: + return "NULL" + case 1: + return "PROGBITS" + case 2: + return "SYMTAB" + case 3: + return "STRTAB" + case 4: + return "RELA" + case 5: + return "HASH" + case 6: + return "DYNAMIC" + case 7: + return "NOTE" + case 8: + return "NOBITS" + case 9: + return "REL" + case 11: + return "DYNSYM" + case 14: + return "INIT_ARRAY" + case 15: + return "FINI_ARRAY" + case 17: + return "GROUP" + case 18: + return "SYMTAB_SHNDX" + } + return fmt.Sprintf("0x%x", t) +} + +func elfSectionFlags(flags uint64) string { + var out string + if flags&0x1 != 0 { + out += "W" + } + if flags&0x2 != 0 { + out += "A" + } + if flags&0x4 != 0 { + out += "X" + } + if flags&0x10 != 0 { + out += "M" + } + if flags&0x20 != 0 { + out += "S" + } + if flags&0x40 != 0 { + out += "I" + } + if flags&0x400 != 0 { + out += "T" + } + if out == "" { + return "-" + } + return out +} + +func elfType(t uint16) string { + switch t { + case 1: + return "REL" + case 2: + return "EXEC" + case 3: + return "DYN" + case 4: + return "CORE" + } + return "UNKNOWN" +} + +func elfMachine(m uint16) string { + switch m { + case 0x03: + return "x86" + case 0x28: + return "ARM" + case 0x3e: + return "x86_64" + case 0xb7: + return "AArch64" + case 0x08: + return "MIPS" + case 0xf3: + return "RISC-V" + } + return "other" +} + +// ParsePE walks the DOS header, PE header, and section headers. +func ParsePE(d *document.Document) (*Field, error) { + hdr, err := d.Range(0, 64) + if err != nil || len(hdr) < 64 { + return nil, errors.New("too short") + } + lfanew := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40])) + root := &Field{Name: "PE", Type: "container", Offset: 0} + root.Children = append(root.Children, + &Field{Name: "e_magic", Offset: 0, Length: 2, Value: "MZ"}, + &Field{Name: "e_lfanew", Offset: 0x3c, Length: 4, Value: fmt.Sprintf("0x%x", lfanew)}, + ) + peHdr, err := d.Range(lfanew, 24) + if err != nil || len(peHdr) < 24 || string(peHdr[:4]) != "PE\x00\x00" { + return root, errors.New("no PE signature") + } + machine := binary.LittleEndian.Uint16(peHdr[4:6]) + numSections := binary.LittleEndian.Uint16(peHdr[6:8]) + timestamp := binary.LittleEndian.Uint32(peHdr[8:12]) + sizeOptHdr := binary.LittleEndian.Uint16(peHdr[20:22]) + characteristics := binary.LittleEndian.Uint16(peHdr[22:24]) + root.Children = append(root.Children, + &Field{Name: "Signature", Offset: lfanew, Length: 4, Value: "PE\\x00\\x00"}, + &Field{Name: "Machine", Offset: lfanew + 4, Length: 2, Value: fmt.Sprintf("0x%04x (%s)", machine, peMachine(machine))}, + &Field{Name: "NumberOfSections", Offset: lfanew + 6, Length: 2, Value: fmt.Sprintf("%d", numSections)}, + &Field{Name: "TimeDateStamp", Offset: lfanew + 8, Length: 4, Value: fmt.Sprintf("%d", timestamp)}, + &Field{Name: "SizeOfOptionalHeader", Offset: lfanew + 20, Length: 2, Value: fmt.Sprintf("%d", sizeOptHdr)}, + &Field{Name: "Characteristics", Offset: lfanew + 22, Length: 2, Value: fmt.Sprintf("0x%04x", characteristics)}, + ) + sections := make([]peSectionInfo, 0, numSections) + optOff := lfanew + 24 + var dataDirs []peDataDirectory + peIs64 := false + if sizeOptHdr > 0 { + opt, err := d.Range(optOff, int64(sizeOptHdr)) + if err == nil && len(opt) >= 2 { + optional, dirs := parsePEOptionalHeader(opt, optOff) + root.Children = append(root.Children, optional) + dataDirs = dirs + peIs64 = binary.LittleEndian.Uint16(opt[0:2]) == 0x20b + } + } + sectOff := lfanew + 24 + int64(sizeOptHdr) + for i := 0; i < int(numSections); i++ { + sh, err := d.Range(sectOff, 40) + if err != nil || len(sh) < 40 { + break + } + name := string(trimZero(sh[0:8])) + vsize := binary.LittleEndian.Uint32(sh[8:12]) + vaddr := binary.LittleEndian.Uint32(sh[12:16]) + rsize := binary.LittleEndian.Uint32(sh[16:20]) + raddr := binary.LittleEndian.Uint32(sh[20:24]) + flags := binary.LittleEndian.Uint32(sh[36:40]) + sec := &Field{ + Name: name, Type: "section", Offset: sectOff, Length: 40, + Value: fmt.Sprintf("vaddr=0x%x vsize=%d raw=0x%x rsize=%d flags=0x%x", vaddr, vsize, raddr, rsize, flags), + } + root.Children = append(root.Children, sec) + sections = append(sections, peSectionInfo{Name: name, VirtualAddress: vaddr, VirtualSize: vsize, RawOffset: raddr, RawSize: rsize, Characteristics: flags}) + sectOff += 40 + } + if len(dataDirs) > 0 && dataDirs[0].RVA != 0 && dataDirs[0].Size != 0 { + root.Children = append(root.Children, parsePEExports(d, sections, dataDirs[0])...) + } + if len(dataDirs) > 1 && dataDirs[1].RVA != 0 && dataDirs[1].Size != 0 { + root.Children = append(root.Children, parsePEImports(d, sections, dataDirs[1], peIs64)...) + } + if len(dataDirs) > 2 && dataDirs[2].RVA != 0 && dataDirs[2].Size != 0 { + root.Children = append(root.Children, parsePEResources(d, sections, dataDirs[2])...) + } + return root, nil +} + +type peDataDirectory struct { + Name string + RVA uint32 + Size uint32 +} + +type peSectionInfo struct { + Name string + VirtualAddress uint32 + VirtualSize uint32 + RawOffset uint32 + RawSize uint32 + Characteristics uint32 +} + +func parsePEOptionalHeader(opt []byte, off int64) (*Field, []peDataDirectory) { + magic := binary.LittleEndian.Uint16(opt[0:2]) + is64 := magic == 0x20b + name := "PE32" + if is64 { + name = "PE32+" + } + field := &Field{Name: "optional_header", Type: "optional_header", Offset: off, Length: int64(len(opt)), Value: name} + if len(opt) >= 28 { + field.Children = append(field.Children, + &Field{Name: "Magic", Type: "u16le", Offset: off, Length: 2, Value: fmt.Sprintf("0x%04x (%s)", magic, name)}, + &Field{Name: "AddressOfEntryPoint", Type: "u32le", Offset: off + 16, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[16:20]))}, + &Field{Name: "BaseOfCode", Type: "u32le", Offset: off + 20, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[20:24]))}, + ) + } + if is64 && len(opt) >= 112 { + field.Children = append(field.Children, + &Field{Name: "ImageBase", Type: "u64le", Offset: off + 24, Length: 8, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint64(opt[24:32]))}, + &Field{Name: "SectionAlignment", Type: "u32le", Offset: off + 32, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[32:36]))}, + &Field{Name: "FileAlignment", Type: "u32le", Offset: off + 36, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[36:40]))}, + &Field{Name: "SizeOfImage", Type: "u32le", Offset: off + 56, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[56:60]))}, + &Field{Name: "Subsystem", Type: "u16le", Offset: off + 68, Length: 2, Value: peSubsystem(binary.LittleEndian.Uint16(opt[68:70]))}, + ) + return field, peDataDirectories(opt, off, 112) + } + if !is64 && len(opt) >= 96 { + field.Children = append(field.Children, + &Field{Name: "ImageBase", Type: "u32le", Offset: off + 28, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[28:32]))}, + &Field{Name: "SectionAlignment", Type: "u32le", Offset: off + 32, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[32:36]))}, + &Field{Name: "FileAlignment", Type: "u32le", Offset: off + 36, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[36:40]))}, + &Field{Name: "SizeOfImage", Type: "u32le", Offset: off + 56, Length: 4, Value: fmt.Sprintf("0x%x", binary.LittleEndian.Uint32(opt[56:60]))}, + &Field{Name: "Subsystem", Type: "u16le", Offset: off + 68, Length: 2, Value: peSubsystem(binary.LittleEndian.Uint16(opt[68:70]))}, + ) + return field, peDataDirectories(opt, off, 96) + } + return field, nil +} + +func peDataDirectories(opt []byte, optOff int64, dirOff int) []peDataDirectory { + if len(opt) < dirOff { + return nil + } + names := []string{"export", "import", "resource", "exception", "certificate", "base_reloc", "debug", "architecture", "global_ptr", "tls", "load_config", "bound_import", "iat", "delay_import", "clr_runtime", "reserved"} + count := (len(opt) - dirOff) / 8 + if count > len(names) { + count = len(names) + } + out := make([]peDataDirectory, 0, count) + for i := 0; i < count; i++ { + at := dirOff + i*8 + rva := binary.LittleEndian.Uint32(opt[at : at+4]) + size := binary.LittleEndian.Uint32(opt[at+4 : at+8]) + out = append(out, peDataDirectory{Name: names[i], RVA: rva, Size: size}) + } + return out +} + +func parsePEImports(d *document.Document, sections []peSectionInfo, dir peDataDirectory, is64 bool) []*Field { + off, ok := peRVAToOffset(sections, dir.RVA) + if !ok { + return nil + } + out := []*Field{} + for i := 0; i < 256; i++ { + at := int64(off) + int64(i*20) + buf, err := d.Range(at, 20) + if err != nil || len(buf) < 20 { + break + } + oft := binary.LittleEndian.Uint32(buf[0:4]) + timestamp := binary.LittleEndian.Uint32(buf[4:8]) + forwarder := binary.LittleEndian.Uint32(buf[8:12]) + nameRVA := binary.LittleEndian.Uint32(buf[12:16]) + firstThunk := binary.LittleEndian.Uint32(buf[16:20]) + if oft == 0 && timestamp == 0 && forwarder == 0 && nameRVA == 0 && firstThunk == 0 { + break + } + name := "" + if nameOff, ok := peRVAToOffset(sections, nameRVA); ok { + name = readCString(d, int64(nameOff), 256) + } + if name == "" { + name = fmt.Sprintf("rva_0x%x", nameRVA) + } + out = append(out, &Field{ + Name: fmt.Sprintf("import[%d]", i), + Type: "import_descriptor", + Offset: at, + Length: 20, + Value: fmt.Sprintf("dll=%s oft=0x%x first_thunk=0x%x timestamp=0x%x", name, oft, firstThunk, timestamp), + Children: parsePEImportThunks(d, sections, oft, firstThunk, is64), + }) + } + return out +} + +func parsePEExports(d *document.Document, sections []peSectionInfo, dir peDataDirectory) []*Field { + off, ok := peRVAToOffset(sections, dir.RVA) + if !ok { + return nil + } + buf, err := d.Range(int64(off), 40) + if err != nil || len(buf) < 40 { + return nil + } + nameRVA := binary.LittleEndian.Uint32(buf[12:16]) + ordinalBase := binary.LittleEndian.Uint32(buf[16:20]) + numberOfFunctions := binary.LittleEndian.Uint32(buf[20:24]) + numberOfNames := binary.LittleEndian.Uint32(buf[24:28]) + functionsRVA := binary.LittleEndian.Uint32(buf[28:32]) + namesRVA := binary.LittleEndian.Uint32(buf[32:36]) + ordinalsRVA := binary.LittleEndian.Uint32(buf[36:40]) + module := "" + if nameOff, ok := peRVAToOffset(sections, nameRVA); ok { + module = readCString(d, int64(nameOff), 512) + } + if module == "" { + module = fmt.Sprintf("rva_0x%x", nameRVA) + } + root := &Field{ + Name: "exports", + Type: "export_directory", + Offset: int64(off), + Length: 40, + Value: fmt.Sprintf("module=%s ordinal_base=%d functions=%d names=%d", module, ordinalBase, numberOfFunctions, numberOfNames), + } + if numberOfNames > 4096 { + numberOfNames = 4096 + } + namesOff, namesOK := peRVAToOffset(sections, namesRVA) + ordinalsOff, ordOK := peRVAToOffset(sections, ordinalsRVA) + functionsOff, funcsOK := peRVAToOffset(sections, functionsRVA) + if namesOK && ordOK && funcsOK { + for i := uint32(0); i < numberOfNames; i++ { + namePtrAt := int64(namesOff) + int64(i)*4 + namePtr, err := d.Range(namePtrAt, 4) + if err != nil || len(namePtr) < 4 { + break + } + symNameRVA := binary.LittleEndian.Uint32(namePtr) + symName := "" + if symNameOff, ok := peRVAToOffset(sections, symNameRVA); ok { + symName = readCString(d, int64(symNameOff), 512) + } + if symName == "" { + symName = fmt.Sprintf("rva_0x%x", symNameRVA) + } + ordAt := int64(ordinalsOff) + int64(i)*2 + ordBuf, err := d.Range(ordAt, 2) + if err != nil || len(ordBuf) < 2 { + break + } + index := uint32(binary.LittleEndian.Uint16(ordBuf)) + if index >= numberOfFunctions { + continue + } + funcAt := int64(functionsOff) + int64(index)*4 + funcBuf, err := d.Range(funcAt, 4) + if err != nil || len(funcBuf) < 4 { + break + } + funcRVA := binary.LittleEndian.Uint32(funcBuf) + value := fmt.Sprintf("name=%s ordinal=%d rva=0x%x", symName, ordinalBase+index, funcRVA) + if funcOff, ok := peRVAToOffset(sections, funcRVA); ok { + value += fmt.Sprintf(" file=0x%x", funcOff) + } + root.Children = append(root.Children, &Field{ + Name: fmt.Sprintf("export[%d] %s", i, symName), + Type: "export", + Offset: namePtrAt, + Length: 4, + Value: value, + }) + } + } + return []*Field{root} +} + +func parsePEResources(d *document.Document, sections []peSectionInfo, dir peDataDirectory) []*Field { + baseOff, ok := peRVAToOffset(sections, dir.RVA) + if !ok { + return nil + } + root := &Field{ + Name: "resources", + Type: "resource_directory", + Offset: int64(baseOff), + Length: int64(dir.Size), + Value: fmt.Sprintf("rva=0x%x size=%d", dir.RVA, dir.Size), + } + seen := map[uint32]bool{} + root.Children = parsePEResourceDirectory(d, sections, dir.RVA, baseOff, 0, 0, seen, "") + return []*Field{root} +} + +func parsePEResourceDirectory(d *document.Document, sections []peSectionInfo, baseRVA, baseOff, rel uint32, depth int, seen map[uint32]bool, path string) []*Field { + if depth > 8 || seen[rel] { + return nil + } + seen[rel] = true + defer delete(seen, rel) + + at := int64(baseOff + rel) + buf, err := d.Range(at, 16) + if err != nil || len(buf) < 16 { + return nil + } + named := binary.LittleEndian.Uint16(buf[12:14]) + ids := binary.LittleEndian.Uint16(buf[14:16]) + count := int(named) + int(ids) + if count > 1024 { + count = 1024 + } + header := &Field{ + Name: fmt.Sprintf("directory depth=%d", depth), + Type: "resource_directory_header", + Offset: at, + Length: 16, + Value: fmt.Sprintf("named=%d ids=%d timestamp=0x%x version=%d.%d", named, ids, binary.LittleEndian.Uint32(buf[4:8]), binary.LittleEndian.Uint16(buf[10:12]), binary.LittleEndian.Uint16(buf[8:10])), + } + out := []*Field{header} + for i := 0; i < count; i++ { + entryAt := at + 16 + int64(i)*8 + ent, err := d.Range(entryAt, 8) + if err != nil || len(ent) < 8 { + break + } + nameRaw := binary.LittleEndian.Uint32(ent[0:4]) + targetRaw := binary.LittleEndian.Uint32(ent[4:8]) + name := peResourceName(d, baseOff, nameRaw, depth) + targetRel := targetRaw & 0x7fffffff + nextPath := path + if nextPath == "" { + nextPath = name + } else { + nextPath += "/" + name + } + entry := &Field{ + Name: fmt.Sprintf("entry[%d] %s", i, name), + Type: "resource_entry", + Offset: entryAt, + Length: 8, + Value: fmt.Sprintf("name=0x%x target=0x%x path=%s", nameRaw, targetRaw, nextPath), + } + if targetRaw&0x80000000 != 0 { + entry.Children = parsePEResourceDirectory(d, sections, baseRVA, baseOff, targetRel, depth+1, seen, nextPath) + } else { + entry.Children = parsePEResourceDataEntry(d, sections, baseOff, targetRel, nextPath) + } + out = append(out, entry) + } + return out +} + +func parsePEResourceDataEntry(d *document.Document, sections []peSectionInfo, baseOff, rel uint32, path string) []*Field { + at := int64(baseOff + rel) + buf, err := d.Range(at, 16) + if err != nil || len(buf) < 16 { + return nil + } + dataRVA := binary.LittleEndian.Uint32(buf[0:4]) + size := binary.LittleEndian.Uint32(buf[4:8]) + codePage := binary.LittleEndian.Uint32(buf[8:12]) + dataOff, mapped := peRVAToOffset(sections, dataRVA) + value := fmt.Sprintf("data_rva=0x%x size=%d codepage=%d", dataRVA, size, codePage) + if mapped { + value += fmt.Sprintf(" file=0x%x", dataOff) + } + entry := &Field{Name: "data", Type: "resource_data", Offset: at, Length: 16, Value: value} + if mapped && size > 0 { + entry.Children = append(entry.Children, peResourcePayloadFields(d, int64(dataOff), size, path, codePage)...) + } + return []*Field{entry} +} + +func peResourcePayloadFields(d *document.Document, off int64, size uint32, path string, codePage uint32) []*Field { + if size > 16<<20 { + return []*Field{{Name: "payload", Type: "resource_payload", Offset: off, Length: int64(size), Value: "too large to decode"}} + } + buf, err := d.Range(off, int64(size)) + if err != nil { + return nil + } + out := []*Field{{Name: "payload", Type: "resource_payload", Offset: off, Length: int64(size), Value: fmt.Sprintf("%d bytes", size)}} + parts := strings.Split(path, "/") + typ := "" + if len(parts) > 0 { + typ = parts[0] + } + switch { + case strings.HasPrefix(typ, "RT_VERSION"): + out = append(out, decodePEVersionResource(buf, off)...) + case strings.HasPrefix(typ, "RT_STRING"): + out = append(out, decodePEStringTable(buf, off)...) + case strings.HasPrefix(typ, "RT_MANIFEST"): + out = append(out, &Field{Name: "manifest", Type: "manifest", Offset: off, Length: int64(size), Value: peResourceTextPreview(buf, codePage)}) + default: + if txt := peResourceTextPreview(buf, codePage); txt != "" { + out = append(out, &Field{Name: "text_preview", Type: "text", Offset: off, Length: int64(size), Value: txt}) + } + } + return out +} + +func peResourceName(d *document.Document, baseOff, raw uint32, depth int) string { + if raw&0x80000000 != 0 { + rel := raw & 0x7fffffff + buf, err := d.Range(int64(baseOff+rel), 2) + if err != nil || len(buf) < 2 { + return fmt.Sprintf("name@0x%x", rel) + } + n := int(binary.LittleEndian.Uint16(buf)) + if n <= 0 || n > 512 { + return fmt.Sprintf("name@0x%x", rel) + } + rawName, err := d.Range(int64(baseOff+rel+2), int64(n*2)) + if err != nil || len(rawName) < n*2 { + return fmt.Sprintf("name@0x%x", rel) + } + return decodeUTF16LE(rawName) + } + id := raw & 0xffff + if depth == 0 { + return peResourceTypeName(id) + } + if depth == 2 { + return peLanguageName(id) + } + return fmt.Sprintf("#%d", id) +} + +func peResourceTypeName(id uint32) string { + names := map[uint32]string{ + 1: "RT_CURSOR", 2: "RT_BITMAP", 3: "RT_ICON", 4: "RT_MENU", 5: "RT_DIALOG", 6: "RT_STRING", + 7: "RT_FONTDIR", 8: "RT_FONT", 9: "RT_ACCELERATOR", 10: "RT_RCDATA", 11: "RT_MESSAGETABLE", + 12: "RT_GROUP_CURSOR", 14: "RT_GROUP_ICON", 16: "RT_VERSION", 17: "RT_DLGINCLUDE", + 19: "RT_PLUGPLAY", 20: "RT_VXD", 21: "RT_ANICURSOR", 22: "RT_ANIICON", 23: "RT_HTML", 24: "RT_MANIFEST", + } + if name, ok := names[id]; ok { + return fmt.Sprintf("%s(%d)", name, id) + } + return fmt.Sprintf("#%d", id) +} + +func peLanguageName(id uint32) string { + switch id { + case 0: + return "LANG_NEUTRAL" + case 0x0409: + return "en-US" + case 0x0809: + return "en-GB" + } + return fmt.Sprintf("lang-0x%04x", id) +} + +func decodePEStringTable(buf []byte, off int64) []*Field { + out := []*Field{} + pos := 0 + for i := 0; i < 16 && pos+2 <= len(buf); i++ { + n := int(binary.LittleEndian.Uint16(buf[pos : pos+2])) + pos += 2 + if pos+n*2 > len(buf) { + break + } + if n > 0 { + out = append(out, &Field{Name: fmt.Sprintf("string[%d]", i), Type: "resource_string", Offset: off + int64(pos), Length: int64(n * 2), Value: decodeUTF16LE(buf[pos : pos+n*2])}) + } + pos += n * 2 + } + return out +} + +func decodePEVersionResource(buf []byte, off int64) []*Field { + if len(buf) < 6 { + return nil + } + key, keyEnd := readPEVersionKey(buf, 6) + out := []*Field{{Name: "version_info", Type: "version_info", Offset: off, Length: int64(len(buf)), Value: fmt.Sprintf("key=%s length=%d valueLength=%d type=%d", key, binary.LittleEndian.Uint16(buf[0:2]), binary.LittleEndian.Uint16(buf[2:4]), binary.LittleEndian.Uint16(buf[4:6]))}} + valueOff := align4(keyEnd) + if key == "VS_VERSION_INFO" && valueOff+52 <= len(buf) { + fixed := buf[valueOff : valueOff+52] + if binary.LittleEndian.Uint32(fixed[0:4]) == 0xfeef04bd { + fileMS := binary.LittleEndian.Uint32(fixed[8:12]) + fileLS := binary.LittleEndian.Uint32(fixed[12:16]) + prodMS := binary.LittleEndian.Uint32(fixed[16:20]) + prodLS := binary.LittleEndian.Uint32(fixed[20:24]) + out = append(out, &Field{Name: "fixed_file_info", Type: "version_fixed", Offset: off + int64(valueOff), Length: 52, Value: fmt.Sprintf("file=%d.%d.%d.%d product=%d.%d.%d.%d flags=0x%x type=0x%x subtype=0x%x", + fileMS>>16, fileMS&0xffff, fileLS>>16, fileLS&0xffff, + prodMS>>16, prodMS&0xffff, prodLS>>16, prodLS&0xffff, + binary.LittleEndian.Uint32(fixed[28:32]), binary.LittleEndian.Uint32(fixed[36:40]), binary.LittleEndian.Uint32(fixed[40:44]))}) + } + } + if strings.Contains(string(buf), "StringFileInfo") || strings.Contains(string(buf), "VarFileInfo") { + text := decodeUTF16LEASCII(buf) + for _, label := range []string{"CompanyName", "FileDescription", "FileVersion", "ProductName", "ProductVersion", "OriginalFilename"} { + if strings.Contains(text, label) { + out = append(out, &Field{Name: label, Type: "version_string_hint", Offset: off, Length: int64(len(buf)), Value: label}) + } + } + } + return out +} + +func readPEVersionKey(buf []byte, pos int) (string, int) { + start := pos + for pos+1 < len(buf) { + if buf[pos] == 0 && buf[pos+1] == 0 { + return decodeUTF16LE(buf[start:pos]), pos + 2 + } + pos += 2 + } + return "", pos +} + +func align4(n int) int { + return (n + 3) &^ 3 +} + +func peResourceTextPreview(buf []byte, codePage uint32) string { + if len(buf) == 0 { + return "" + } + if len(buf) >= 2 { + if buf[0] == 0xff && buf[1] == 0xfe { + return cleanResourcePreview(decodeUTF16LE(buf[2:])) + } + if buf[0] == 0xfe && buf[1] == 0xff { + return cleanResourcePreview(decodeUTF16BE(buf[2:])) + } + } + if codePage == 1200 { + return cleanResourcePreview(decodeUTF16LE(buf)) + } + if utf8.Valid(buf) { + return cleanResourcePreview(string(buf)) + } + return "" +} + +func cleanResourcePreview(s string) string { + s = strings.TrimSpace(strings.ReplaceAll(s, "\x00", "")) + if len(s) > 512 { + s = s[:512] + "..." + } + if strings.IndexFunc(s, func(r rune) bool { return r < 0x09 }) >= 0 { + return "" + } + return s +} + +func decodeUTF16LE(buf []byte) string { + u := make([]uint16, 0, len(buf)/2) + for i := 0; i+1 < len(buf); i += 2 { + u = append(u, binary.LittleEndian.Uint16(buf[i:i+2])) + } + return string(utf16.Decode(u)) +} + +func decodeUTF16BE(buf []byte) string { + u := make([]uint16, 0, len(buf)/2) + for i := 0; i+1 < len(buf); i += 2 { + u = append(u, binary.BigEndian.Uint16(buf[i:i+2])) + } + return string(utf16.Decode(u)) +} + +func decodeUTF16LEASCII(buf []byte) string { + var b strings.Builder + for i := 0; i+1 < len(buf); i += 2 { + r := binary.LittleEndian.Uint16(buf[i : i+2]) + if r >= 0x20 && r < 0x7f { + b.WriteByte(byte(r)) + } else { + b.WriteByte(' ') + } + } + return b.String() +} + +func parsePEImportThunks(d *document.Document, sections []peSectionInfo, oft, firstThunk uint32, is64 bool) []*Field { + thunkRVA := oft + if thunkRVA == 0 { + thunkRVA = firstThunk + } + thunkOff, ok := peRVAToOffset(sections, thunkRVA) + if !ok { + return nil + } + width := int64(4) + ordinalMask := uint64(0x80000000) + if is64 { + width = 8 + ordinalMask = 0x8000000000000000 + } + out := []*Field{} + for i := 0; i < 4096; i++ { + at := int64(thunkOff) + int64(i)*width + buf, err := d.Range(at, width) + if err != nil || int64(len(buf)) < width { + break + } + var thunk uint64 + if is64 { + thunk = binary.LittleEndian.Uint64(buf) + } else { + thunk = uint64(binary.LittleEndian.Uint32(buf)) + } + if thunk == 0 { + break + } + value := "" + if thunk&ordinalMask != 0 { + value = fmt.Sprintf("ordinal=%d", thunk&0xffff) + } else if nameOff, ok := peRVAToOffset(sections, uint32(thunk)); ok { + name := readImportByName(d, int64(nameOff)) + if name != "" { + value = "name=" + name + } else { + value = fmt.Sprintf("name_rva=0x%x", thunk) + } + } else { + value = fmt.Sprintf("thunk=0x%x", thunk) + } + out = append(out, &Field{ + Name: fmt.Sprintf("thunk[%d]", i), + Type: "import_thunk", + Offset: at, + Length: width, + Value: value, + }) + } + return out +} + +func readImportByName(d *document.Document, off int64) string { + buf, err := d.Range(off, 258) + if err != nil || len(buf) < 3 { + return "" + } + return readCStringFrom(buf[2:]) +} + +func peRVAToOffset(sections []peSectionInfo, rva uint32) (uint32, bool) { + for _, s := range sections { + span := s.VirtualSize + if s.RawSize > span { + span = s.RawSize + } + if span == 0 { + continue + } + if rva >= s.VirtualAddress && rva < s.VirtualAddress+span { + return s.RawOffset + (rva - s.VirtualAddress), true + } + } + return 0, false +} + +func readCString(d *document.Document, off int64, max int64) string { + buf, err := d.Range(off, max) + if err != nil { + return "" + } + return readCStringFrom(buf) +} + +func readCStringFrom(buf []byte) string { + for i, b := range buf { + if b == 0 { + return string(buf[:i]) + } + } + return string(buf) +} + +func peSubsystem(v uint16) string { + switch v { + case 1: + return "native" + case 2: + return "windows_gui" + case 3: + return "windows_cui" + case 7: + return "posix_cui" + case 9: + return "windows_ce_gui" + case 10: + return "efi_application" + case 11: + return "efi_boot_service_driver" + case 12: + return "efi_runtime_driver" + } + return fmt.Sprintf("0x%x", v) +} + +func peMachine(m uint16) string { + switch m { + case 0x14c: + return "i386" + case 0x8664: + return "x86_64" + case 0x1c0: + return "ARM" + case 0xaa64: + return "ARM64" + case 0x200: + return "IA64" + } + return "?" +} + +func trimZero(b []byte) []byte { + for i, c := range b { + if c == 0 { + return b[:i] + } + } + return b +} + +// ParseRIFF walks RIFF chunks. +func ParseRIFF(d *document.Document) (*Field, error) { + size := d.Size() + if size < 12 { + return nil, errors.New("too short") + } + hdr, err := d.Range(0, 12) + if err != nil { + return nil, err + } + total := int64(binary.LittleEndian.Uint32(hdr[4:8])) + form := string(hdr[8:12]) + end := total + 8 + if end > size { + end = size + } + root := &Field{ + Name: "RIFF", Type: "container", Offset: 0, Length: total + 8, + Value: fmt.Sprintf("form=%s (%s)", form, riffFormName(form)), + } + root.Children = riffChunks(d, 12, end, 0) + return root, nil +} + +// riffChunks walks the chunks in [off,end), recursing into LIST containers and +// decoding the common WAV/AVI chunks. +func riffChunks(d *document.Document, off, end int64, depth int) []*Field { + var out []*Field + for off+8 <= end && depth < 16 { + ch, err := d.Range(off, 8) + if err != nil || len(ch) < 8 { + break + } + id := string(ch[0:4]) + ln := int64(binary.LittleEndian.Uint32(ch[4:8])) + body := off + 8 + chunkEnd := body + ln + if chunkEnd > end { + chunkEnd = end + } + f := &Field{Name: riffChunkLabel(id), Type: "chunk", Offset: off, Length: 8 + ln, Value: fmt.Sprintf("length=%d", ln)} + switch id { + case "LIST", "RIFF": + if ln >= 4 { + lb, _ := d.Range(body, 4) + lt := string(lb) + f.Type = "list" + f.Value = fmt.Sprintf("type=%s (%s)", lt, riffFormName(lt)) + f.Children = riffChunks(d, body+4, chunkEnd, depth+1) + } + case "fmt ": + if fb, e := d.Range(body, min64(16, ln)); e == nil && len(fb) >= 16 { + f.Children = []*Field{ + {Name: "audio_format", Type: "u16le", Offset: body, Length: 2, Value: wavFormat(binary.LittleEndian.Uint16(fb[0:2]))}, + {Name: "channels", Type: "u16le", Offset: body + 2, Length: 2, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint16(fb[2:4]))}, + {Name: "sample_rate", Type: "u32le", Offset: body + 4, Length: 4, Value: fmt.Sprintf("%d Hz", binary.LittleEndian.Uint32(fb[4:8]))}, + {Name: "byte_rate", Type: "u32le", Offset: body + 8, Length: 4, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint32(fb[8:12]))}, + {Name: "block_align", Type: "u16le", Offset: body + 12, Length: 2, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint16(fb[12:14]))}, + {Name: "bits_per_sample", Type: "u16le", Offset: body + 14, Length: 2, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint16(fb[14:16]))}, + } + } + case "avih": + if ab, e := d.Range(body, min64(40, ln)); e == nil && len(ab) >= 16 { + f.Children = []*Field{ + {Name: "microsec_per_frame", Type: "u32le", Offset: body, Length: 4, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint32(ab[0:4]))}, + {Name: "max_bytes_per_sec", Type: "u32le", Offset: body + 4, Length: 4, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint32(ab[4:8]))}, + {Name: "total_frames", Type: "u32le", Offset: body + 16, Length: 4, Value: fmt.Sprintf("%d", binary.LittleEndian.Uint32(ab[16:20]))}, + } + } + } + out = append(out, f) + off = chunkEnd + if ln%2 != 0 { + off++ // chunks are word-aligned + } + } + return out +} + +func riffFormName(f string) string { + switch f { + case "WAVE": + return "audio/wav" + case "AVI ": + return "video/avi" + case "WEBP": + return "image/webp" + case "ANI ": + return "animated cursor" + case "RMID": + return "MIDI" + case "INFO": + return "metadata" + case "movi": + return "movie data" + case "hdrl": + return "header list" + case "strl": + return "stream list" + default: + return "?" + } +} + +func riffChunkLabel(id string) string { + return strings.TrimRight(id, " ") +} + +func wavFormat(f uint16) string { + names := map[uint16]string{1: "PCM", 2: "ADPCM", 3: "IEEE float", 6: "A-law", 7: "mu-law", 0xfffe: "extensible"} + if n, ok := names[f]; ok { + return fmt.Sprintf("%d (%s)", f, n) + } + return fmt.Sprintf("%d", f) +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/internal/leviathan/parsers/formats_more.go b/internal/leviathan/parsers/formats_more.go new file mode 100644 index 0000000..a3652c6 --- /dev/null +++ b/internal/leviathan/parsers/formats_more.go @@ -0,0 +1,662 @@ +// Additional structure parsers that round out the structures panel into a +// forensic-grade set: gzip, tar, SQLite, TIFF, pcap/pcapng and the OLE/CFB +// compound-file container used by legacy Office documents and MSI installers. +// +// Each parser follows the same contract as the parsers in formats.go: it +// returns a *Field tree anchored at absolute document offsets, never panics on +// truncated input, and bounds every loop so a hostile file cannot wedge the UI. +package parsers + +import ( + "encoding/binary" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// leaf builds a terminal Field with no children. +func leaf(name, typ string, off, length int64, value string) *Field { + return &Field{Name: name, Type: typ, Offset: off, Length: length, Value: value} +} + +// ---------- gzip (RFC 1952) ---------- + +// ParseGZIP decodes the gzip member header, optional name/comment/extra fields +// and the trailing CRC-32 / ISIZE footer. +func ParseGZIP(d *document.Document) (*Field, error) { + size := d.Size() + hdr, err := d.Range(0, min64(10, size)) + if err != nil || len(hdr) < 10 { + return nil, errors.New("gzip too short") + } + if hdr[0] != 0x1f || hdr[1] != 0x8b { + return nil, errors.New("not gzip") + } + root := &Field{Name: "GZIP", Type: "container", Offset: 0, Length: size} + method := hdr[2] + methodName := "deflate" + if method != 8 { + methodName = fmt.Sprintf("unknown(%d)", method) + } + flg := hdr[3] + mtime := binary.LittleEndian.Uint32(hdr[4:8]) + root.Children = append(root.Children, + leaf("id", "bytes", 0, 2, "\\x1f\\x8b"), + leaf("method", "u8", 2, 1, methodName), + leaf("flags", "u8", 3, 1, gzipFlags(flg)), + leaf("mtime", "u32le", 4, 4, gzipTime(mtime)), + leaf("extra_flags", "u8", 8, 1, fmt.Sprintf("%d", hdr[8])), + leaf("os", "u8", 9, 1, gzipOS(hdr[9])), + ) + off := int64(10) + // FEXTRA + if flg&0x04 != 0 && off+2 <= size { + xb, _ := d.Range(off, 2) + xlen := int64(binary.LittleEndian.Uint16(xb)) + root.Children = append(root.Children, leaf("extra", "bytes", off, 2+xlen, fmt.Sprintf("%d bytes", xlen))) + off += 2 + xlen + } + // FNAME + if flg&0x08 != 0 { + s, n := readCStringField(d, off, size) + root.Children = append(root.Children, leaf("name", "asciiz", off, n, s)) + off += n + } + // FCOMMENT + if flg&0x10 != 0 { + s, n := readCStringField(d, off, size) + root.Children = append(root.Children, leaf("comment", "asciiz", off, n, s)) + off += n + } + // FHCRC + if flg&0x02 != 0 && off+2 <= size { + hb, _ := d.Range(off, 2) + root.Children = append(root.Children, leaf("header_crc16", "u16le", off, 2, fmt.Sprintf("0x%04x", binary.LittleEndian.Uint16(hb)))) + off += 2 + } + if size >= off+8 { + root.Children = append(root.Children, leaf("compressed_data", "bytes", off, size-off-8, fmt.Sprintf("%d bytes", size-off-8))) + ft, _ := d.Range(size-8, 8) + root.Children = append(root.Children, + leaf("crc32", "u32le", size-8, 4, fmt.Sprintf("0x%08x", binary.LittleEndian.Uint32(ft[0:4]))), + leaf("isize", "u32le", size-4, 4, fmt.Sprintf("%d bytes (mod 2^32)", binary.LittleEndian.Uint32(ft[4:8]))), + ) + } + return root, nil +} + +func gzipFlags(f byte) string { + var on []string + for _, b := range []struct { + mask byte + name string + }{{0x01, "TEXT"}, {0x02, "HCRC"}, {0x04, "EXTRA"}, {0x08, "NAME"}, {0x10, "COMMENT"}} { + if f&b.mask != 0 { + on = append(on, b.name) + } + } + if len(on) == 0 { + return fmt.Sprintf("0x%02x", f) + } + return fmt.Sprintf("0x%02x (%s)", f, strings.Join(on, "|")) +} + +func gzipOS(b byte) string { + names := map[byte]string{0: "FAT", 1: "Amiga", 2: "VMS", 3: "Unix", 6: "HPFS", 7: "Macintosh", 11: "NTFS", 255: "unknown"} + if n, ok := names[b]; ok { + return fmt.Sprintf("%d (%s)", b, n) + } + return fmt.Sprintf("%d", b) +} + +func gzipTime(t uint32) string { + if t == 0 { + return "0 (no timestamp)" + } + return fmt.Sprintf("%d (%s)", t, time.Unix(int64(t), 0).UTC().Format("2006-01-02 15:04:05 UTC")) +} + +// ---------- tar (POSIX ustar) ---------- + +// ParseTAR walks the 512-byte record headers of a ustar/GNU tar archive. +func ParseTAR(d *document.Document) (*Field, error) { + size := d.Size() + root := &Field{Name: "TAR", Type: "container", Offset: 0} + off := int64(0) + entries := 0 + for off+512 <= size && entries < 10000 { + blk, err := d.Range(off, 512) + if err != nil { + break + } + if allZero(blk) { + root.Children = append(root.Children, leaf("end_of_archive", "bytes", off, 512, "zero block")) + off += 512 + break + } + name := trimZeroStr(blk[0:100]) + prefix := trimZeroStr(blk[345:500]) + full := name + if prefix != "" { + full = prefix + "/" + name + } + fsize := octal(blk[124:136]) + mtime := octal(blk[136:148]) + typ := blk[156] + e := &Field{ + Name: full, Type: "entry", Offset: off, + Length: 512 + roundUp(fsize, 512), + Value: fmt.Sprintf("%s size=%d", tarType(typ), fsize), + Children: []*Field{ + leaf("name", "asciiz", off, 100, name), + leaf("mode", "octal", off+100, 8, fmt.Sprintf("0%o", octal(blk[100:108]))), + leaf("uid", "octal", off+108, 8, fmt.Sprintf("%d", octal(blk[108:116]))), + leaf("gid", "octal", off+116, 8, fmt.Sprintf("%d", octal(blk[116:124]))), + leaf("size", "octal", off+124, 12, fmt.Sprintf("%d", fsize)), + leaf("mtime", "octal", off+136, 12, tarTime(mtime)), + leaf("typeflag", "u8", off+156, 1, tarType(typ)), + leaf("magic", "ascii", off+257, 8, trimZeroStr(blk[257:265])), + leaf("uname", "asciiz", off+265, 32, trimZeroStr(blk[265:297])), + leaf("gname", "asciiz", off+297, 32, trimZeroStr(blk[297:329])), + }, + } + if prefix != "" { + e.Children = append(e.Children, leaf("prefix", "asciiz", off+345, 155, prefix)) + } + root.Children = append(root.Children, e) + off += 512 + roundUp(fsize, 512) + entries++ + } + root.Length = off + if len(root.Children) == 0 { + return nil, errors.New("no tar entries") + } + return root, nil +} + +func tarType(t byte) string { + switch t { + case 0, '0': + return "file" + case '1': + return "hardlink" + case '2': + return "symlink" + case '3': + return "char-device" + case '4': + return "block-device" + case '5': + return "directory" + case '6': + return "fifo" + case '7': + return "contiguous" + case 'x': + return "pax-header" + case 'g': + return "pax-global" + case 'L': + return "gnu-longname" + case 'K': + return "gnu-longlink" + default: + return fmt.Sprintf("type-%q", t) + } +} + +func tarTime(t int64) string { + if t == 0 { + return "0" + } + return fmt.Sprintf("%d (%s)", t, time.Unix(t, 0).UTC().Format("2006-01-02 15:04:05 UTC")) +} + +// ---------- SQLite 3 ---------- + +// ParseSQLite decodes the 100-byte SQLite database header. +func ParseSQLite(d *document.Document) (*Field, error) { + hdr, err := d.Range(0, min64(100, d.Size())) + if err != nil || len(hdr) < 100 { + return nil, errors.New("sqlite header too short") + } + if string(hdr[0:16]) != "SQLite format 3\x00" { + return nil, errors.New("not a SQLite database") + } + be := binary.BigEndian + pageSize := int64(be.Uint16(hdr[16:18])) + if pageSize == 1 { + pageSize = 65536 + } + enc := map[uint32]string{1: "UTF-8", 2: "UTF-16le", 3: "UTF-16be"}[be.Uint32(hdr[56:60])] + if enc == "" { + enc = "unknown" + } + root := &Field{ + Name: "SQLite", Type: "container", Offset: 0, Length: d.Size(), + Value: fmt.Sprintf("page=%d pages=%d %s", pageSize, be.Uint32(hdr[28:32]), enc), + Children: []*Field{ + leaf("magic", "ascii", 0, 16, "SQLite format 3"), + leaf("page_size", "u16be", 16, 2, fmt.Sprintf("%d", pageSize)), + leaf("write_version", "u8", 18, 1, sqliteFmtVer(hdr[18])), + leaf("read_version", "u8", 19, 1, sqliteFmtVer(hdr[19])), + leaf("reserved_space", "u8", 20, 1, fmt.Sprintf("%d", hdr[20])), + leaf("change_counter", "u32be", 24, 4, fmt.Sprintf("%d", be.Uint32(hdr[24:28]))), + leaf("page_count", "u32be", 28, 4, fmt.Sprintf("%d", be.Uint32(hdr[28:32]))), + leaf("freelist_head", "u32be", 32, 4, fmt.Sprintf("%d", be.Uint32(hdr[32:36]))), + leaf("freelist_pages", "u32be", 36, 4, fmt.Sprintf("%d", be.Uint32(hdr[36:40]))), + leaf("schema_cookie", "u32be", 40, 4, fmt.Sprintf("%d", be.Uint32(hdr[40:44]))), + leaf("schema_format", "u32be", 44, 4, fmt.Sprintf("%d", be.Uint32(hdr[44:48]))), + leaf("text_encoding", "u32be", 56, 4, enc), + leaf("user_version", "u32be", 60, 4, fmt.Sprintf("%d", be.Uint32(hdr[60:64]))), + leaf("vacuum_mode", "u32be", 64, 4, fmt.Sprintf("%d", be.Uint32(hdr[64:68]))), + leaf("application_id", "u32be", 68, 4, fmt.Sprintf("0x%08x", be.Uint32(hdr[68:72]))), + leaf("version_valid_for", "u32be", 92, 4, fmt.Sprintf("%d", be.Uint32(hdr[92:96]))), + leaf("sqlite_version", "u32be", 96, 4, sqliteVersion(be.Uint32(hdr[96:100]))), + }, + } + return root, nil +} + +func sqliteFmtVer(b byte) string { + switch b { + case 1: + return "1 (legacy/rollback)" + case 2: + return "2 (WAL)" + default: + return fmt.Sprintf("%d", b) + } +} + +func sqliteVersion(v uint32) string { + // SQLITE_VERSION_NUMBER = major*1_000_000 + minor*1000 + patch. + return fmt.Sprintf("%d (%d.%d.%d)", v, v/1000000, (v/1000)%1000, v%1000) +} + +// ---------- TIFF ---------- + +// ParseTIFF decodes the TIFF header and the entries of each image file +// directory (IFD), following the next-IFD chain. +func ParseTIFF(d *document.Document) (*Field, error) { + size := d.Size() + hdr, err := d.Range(0, min64(8, size)) + if err != nil || len(hdr) < 8 { + return nil, errors.New("tiff too short") + } + var bo binary.ByteOrder + switch string(hdr[0:2]) { + case "II": + bo = binary.LittleEndian + case "MM": + bo = binary.BigEndian + default: + return nil, errors.New("not TIFF") + } + if bo.Uint16(hdr[2:4]) != 42 { + return nil, errors.New("bad TIFF magic") + } + order := "little-endian" + if string(hdr[0:2]) == "MM" { + order = "big-endian" + } + root := &Field{ + Name: "TIFF", Type: "container", Offset: 0, Length: size, Value: order, + Children: []*Field{ + leaf("byte_order", "ascii", 0, 2, order), + leaf("magic", "u16", 2, 2, "42"), + leaf("first_ifd", "u32", 4, 4, fmt.Sprintf("0x%x", bo.Uint32(hdr[4:8]))), + }, + } + ifdOff := int64(bo.Uint32(hdr[4:8])) + for n := 0; ifdOff > 0 && ifdOff+2 <= size && n < 64; n++ { + cb, err := d.Range(ifdOff, 2) + if err != nil { + break + } + count := int64(bo.Uint16(cb)) + ifd := &Field{ + Name: fmt.Sprintf("IFD[%d]", n), Type: "directory", Offset: ifdOff, + Length: 2 + count*12 + 4, Value: fmt.Sprintf("%d entries", count), + } + for i := int64(0); i < count && i < 4096; i++ { + eoff := ifdOff + 2 + i*12 + eb, err := d.Range(eoff, 12) + if err != nil || len(eb) < 12 { + break + } + tag := bo.Uint16(eb[0:2]) + typ := bo.Uint16(eb[2:4]) + cnt := bo.Uint32(eb[4:8]) + val := bo.Uint32(eb[8:12]) + ifd.Children = append(ifd.Children, leaf( + fmt.Sprintf("tag 0x%04x %s", tag, tiffTag(tag)), "ifd_entry", eoff, 12, + fmt.Sprintf("type=%s count=%d value=%d", tiffType(typ), cnt, val))) + } + root.Children = append(root.Children, ifd) + nb, err := d.Range(ifdOff+2+count*12, 4) + if err != nil || len(nb) < 4 { + break + } + ifdOff = int64(bo.Uint32(nb)) + } + return root, nil +} + +func tiffType(t uint16) string { + names := map[uint16]string{1: "BYTE", 2: "ASCII", 3: "SHORT", 4: "LONG", 5: "RATIONAL", 6: "SBYTE", 7: "UNDEFINED", 8: "SSHORT", 9: "SLONG", 10: "SRATIONAL", 11: "FLOAT", 12: "DOUBLE"} + if n, ok := names[t]; ok { + return n + } + return fmt.Sprintf("type%d", t) +} + +func tiffTag(t uint16) string { + names := map[uint16]string{ + 256: "ImageWidth", 257: "ImageLength", 258: "BitsPerSample", 259: "Compression", + 262: "PhotometricInterpretation", 273: "StripOffsets", 277: "SamplesPerPixel", + 278: "RowsPerStrip", 279: "StripByteCounts", 282: "XResolution", 283: "YResolution", + 296: "ResolutionUnit", 305: "Software", 306: "DateTime", 315: "Artist", + 271: "Make", 272: "Model", 274: "Orientation", 34665: "ExifIFD", 34853: "GPSIFD", + } + if n, ok := names[t]; ok { + return n + } + return "" +} + +// ---------- pcap / pcapng ---------- + +// ParsePCAP decodes a classic libpcap global header and walks packet records. +func ParsePCAP(d *document.Document) (*Field, error) { + size := d.Size() + hdr, err := d.Range(0, min64(24, size)) + if err != nil || len(hdr) < 24 { + return nil, errors.New("pcap too short") + } + magic := binary.LittleEndian.Uint32(hdr[0:4]) + var bo binary.ByteOrder + nano := false + switch magic { + case 0xa1b2c3d4: + bo = binary.LittleEndian + case 0xd4c3b2a1: + bo = binary.BigEndian + case 0xa1b23c4d: + bo, nano = binary.LittleEndian, true + case 0x4d3cb2a1: + bo, nano = binary.BigEndian, true + default: + return nil, errors.New("not pcap") + } + res := "microsecond" + if nano { + res = "nanosecond" + } + linktype := bo.Uint32(hdr[20:24]) + root := &Field{ + Name: "PCAP", Type: "container", Offset: 0, Length: size, + Value: fmt.Sprintf("%s, %s", res, pcapLinkType(linktype)), + Children: []*Field{ + leaf("magic", "u32", 0, 4, fmt.Sprintf("0x%08x (%s)", magic, res)), + leaf("version", "u16", 4, 4, fmt.Sprintf("%d.%d", bo.Uint16(hdr[4:6]), bo.Uint16(hdr[6:8]))), + leaf("thiszone", "i32", 8, 4, fmt.Sprintf("%d", int32(bo.Uint32(hdr[8:12])))), + leaf("sigfigs", "u32", 12, 4, fmt.Sprintf("%d", bo.Uint32(hdr[12:16]))), + leaf("snaplen", "u32", 16, 4, fmt.Sprintf("%d", bo.Uint32(hdr[16:20]))), + leaf("linktype", "u32", 20, 4, fmt.Sprintf("%d (%s)", linktype, pcapLinkType(linktype))), + }, + } + off := int64(24) + for n := 0; off+16 <= size && n < 100000; n++ { + rb, err := d.Range(off, 16) + if err != nil { + break + } + ts := bo.Uint32(rb[0:4]) + us := bo.Uint32(rb[4:8]) + inclLen := int64(bo.Uint32(rb[8:12])) + origLen := bo.Uint32(rb[12:16]) + root.Children = append(root.Children, leaf( + fmt.Sprintf("packet[%d]", n), "record", off, 16+inclLen, + fmt.Sprintf("ts=%d.%06d incl=%d orig=%d", ts, us, inclLen, origLen))) + off += 16 + inclLen + } + return root, nil +} + +// ParsePCAPNG decodes the pcapng block structure (section header + block walk). +func ParsePCAPNG(d *document.Document) (*Field, error) { + size := d.Size() + hb, err := d.Range(0, min64(12, size)) + if err != nil || len(hb) < 12 { + return nil, errors.New("pcapng too short") + } + if binary.LittleEndian.Uint32(hb[0:4]) != 0x0a0d0d0a { + return nil, errors.New("not pcapng") + } + // Byte-order magic lives in the SHB body at offset 8. + var bo binary.ByteOrder + switch binary.LittleEndian.Uint32(hb[8:12]) { + case 0x1a2b3c4d: + bo = binary.LittleEndian + case 0x4d3c2b1a: + bo = binary.BigEndian + default: + return nil, errors.New("bad pcapng byte-order magic") + } + root := &Field{Name: "PCAPNG", Type: "container", Offset: 0, Length: size} + off := int64(0) + for n := 0; off+12 <= size && n < 100000; n++ { + bh, err := d.Range(off, 12) + if err != nil { + break + } + btype := bo.Uint32(bh[0:4]) + blen := int64(bo.Uint32(bh[4:8])) + if blen < 12 || off+blen > size { + break + } + root.Children = append(root.Children, leaf( + fmt.Sprintf("block[%d] %s", n, pcapngBlock(btype)), "block", off, blen, + fmt.Sprintf("type=0x%08x length=%d", btype, blen))) + off += blen + } + return root, nil +} + +func pcapngBlock(t uint32) string { + switch t { + case 0x0a0d0d0a: + return "SectionHeader" + case 0x00000001: + return "InterfaceDescription" + case 0x00000003: + return "SimplePacket" + case 0x00000004: + return "NameResolution" + case 0x00000005: + return "InterfaceStatistics" + case 0x00000006: + return "EnhancedPacket" + default: + return "block" + } +} + +func pcapLinkType(t uint32) string { + names := map[uint32]string{0: "NULL", 1: "ETHERNET", 6: "TOKEN_RING", 9: "PPP", 101: "RAW", 105: "IEEE802_11", 113: "LINUX_SLL", 127: "IEEE802_11_RADIOTAP", 228: "IPV4", 229: "IPV6", 276: "LINUX_SLL2"} + if n, ok := names[t]; ok { + return n + } + return fmt.Sprintf("linktype%d", t) +} + +// ---------- OLE / Compound File Binary (MS-CFB) ---------- + +var cfbMagic = []byte{0xd0, 0xcf, 0x11, 0xe0, 0xa1, 0xb1, 0x1a, 0xe1} + +// ParseCFB decodes the compound-file header and lists the entries in the first +// directory sector (legacy .doc/.xls/.ppt/.msi container). +func ParseCFB(d *document.Document) (*Field, error) { + size := d.Size() + hdr, err := d.Range(0, min64(512, size)) + if err != nil || len(hdr) < 76 { + return nil, errors.New("cfb too short") + } + if string(hdr[0:8]) != string(cfbMagic) { + return nil, errors.New("not a compound file") + } + le := binary.LittleEndian + major := le.Uint16(hdr[26:28]) + sectorShift := le.Uint16(hdr[30:32]) + sectorSize := int64(1) << sectorShift + firstDir := le.Uint32(hdr[48:52]) + root := &Field{ + Name: "CFB/OLE", Type: "container", Offset: 0, Length: size, + Value: fmt.Sprintf("v%d.%d sector=%d", major, le.Uint16(hdr[24:26]), sectorSize), + Children: []*Field{ + leaf("signature", "bytes", 0, 8, "D0 CF 11 E0 A1 B1 1A E1"), + leaf("minor_version", "u16le", 24, 2, fmt.Sprintf("%d", le.Uint16(hdr[24:26]))), + leaf("major_version", "u16le", 26, 2, fmt.Sprintf("%d", major)), + leaf("byte_order", "u16le", 28, 2, fmt.Sprintf("0x%04x", le.Uint16(hdr[28:30]))), + leaf("sector_shift", "u16le", 30, 2, fmt.Sprintf("%d (%d bytes)", sectorShift, sectorSize)), + leaf("mini_sector_shift", "u16le", 32, 2, fmt.Sprintf("%d (%d bytes)", le.Uint16(hdr[32:34]), int64(1)< size { + return nil + } + dir := &Field{Name: "directory", Type: "directory", Offset: off} + per := sectorSize / 128 + count := int64(0) + for i := int64(0); i < per; i++ { + eoff := off + i*128 + eb, err := d.Range(eoff, 128) + if err != nil || len(eb) < 128 { + break + } + nameLen := int64(binary.LittleEndian.Uint16(eb[64:66])) + objType := eb[66] + if objType == 0 { // unallocated entry + continue + } + name := "" + if nameLen >= 2 && nameLen <= 64 { + name = decodeUTF16LE(eb[0 : nameLen-2]) + } + streamSize := binary.LittleEndian.Uint64(eb[120:128]) + dir.Children = append(dir.Children, leaf( + cfbEntryLabel(name, count), "dir_entry", eoff, 128, + fmt.Sprintf("%s size=%d", cfbObjType(objType), streamSize))) + count++ + } + dir.Length = sectorSize + dir.Value = fmt.Sprintf("%d entries", count) + if count == 0 { + return nil + } + return dir +} + +func cfbEntryLabel(name string, i int64) string { + if name == "" { + return fmt.Sprintf("entry[%d]", i) + } + // Strip the control prefix Office uses on some stream names for display. + clean := strings.Map(func(r rune) rune { + if r < 0x20 { + return -1 + } + return r + }, name) + if clean == "" { + clean = name + } + return fmt.Sprintf("entry[%d] %s", i, clean) +} + +func cfbObjType(t byte) string { + switch t { + case 1: + return "storage" + case 2: + return "stream" + case 5: + return "root" + default: + return fmt.Sprintf("type%d", t) + } +} + +// ---------- shared helpers ---------- + +func readCStringField(d *document.Document, off, size int64) (string, int64) { + var sb strings.Builder + n := int64(0) + for off+n < size && n < 4096 { + b, err := d.At(off + n) + if err != nil { + break + } + n++ + if b == 0 { + break + } + sb.WriteByte(b) + } + return sb.String(), n +} + +func allZero(b []byte) bool { + for _, x := range b { + if x != 0 { + return false + } + } + return true +} + +func trimZeroStr(b []byte) string { + return strings.TrimRight(string(b), "\x00 ") +} + +// octal parses a space/NUL-padded octal ASCII field (tar number encoding). +func octal(b []byte) int64 { + s := strings.TrimSpace(strings.Trim(string(b), "\x00")) + if s == "" { + return 0 + } + v, err := strconv.ParseInt(s, 8, 64) + if err != nil { + return 0 + } + return v +} + +func roundUp(n, to int64) int64 { + if n <= 0 { + return 0 + } + return ((n + to - 1) / to) * to +} diff --git a/internal/leviathan/parsers/macho.go b/internal/leviathan/parsers/macho.go new file mode 100644 index 0000000..6675fb5 --- /dev/null +++ b/internal/leviathan/parsers/macho.go @@ -0,0 +1,408 @@ +package parsers + +import ( + "encoding/binary" + "errors" + "fmt" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Mach-O magic values (32/64, LE/BE) plus fat (universal) magic. +const ( + machoMagic32 = 0xfeedface + machoCigam32 = 0xcefaedfe + machoMagic64 = 0xfeedfacf + machoCigam64 = 0xcffaedfe + machoFatMagic = 0xcafebabe // also Java .class; disambiguate by position + nfat_arch + machoFatCigam = 0xbebafeca + machoFatMagic64 = 0xcafebabf + machoFatCigam64 = 0xbfbafeca +) + +// IsMachO reports whether the given prefix looks like a Mach-O header. +// Fat magic 0xcafebabe is shared with Java class files; we accept it +// only when nfat_arch is plausible and the first arch's cputype points +// somewhere in-document. +func IsMachO(head []byte, size int64) bool { + if len(head) < 8 { + return false + } + m := binary.LittleEndian.Uint32(head[:4]) + switch m { + case machoMagic32, machoMagic64, machoCigam32, machoCigam64: + return true + } + mBE := binary.BigEndian.Uint32(head[:4]) + switch mBE { + case machoFatMagic, machoFatMagic64: + // nfat_arch is next BE u32; plausibly small. + n := binary.BigEndian.Uint32(head[4:8]) + return n > 0 && n < 64 && size >= 8+int64(n)*20 + } + _ = mBE + return false +} + +// ParseMachO returns a field tree for a Mach-O (fat or thin) file. +func ParseMachO(d *document.Document) (*Field, error) { + head, err := d.Range(0, min64(8, d.Size())) + if err != nil { + return nil, err + } + if len(head) < 8 { + return nil, errors.New("file too small for Mach-O") + } + mLE := binary.LittleEndian.Uint32(head[:4]) + mBE := binary.BigEndian.Uint32(head[:4]) + + switch mBE { + case machoFatMagic, machoFatMagic64: + return parseMachOFat(d, mBE == machoFatMagic64) + } + + // Thin header. + switch mLE { + case machoMagic32: + return parseMachOThin(d, 0, binary.LittleEndian, false) + case machoMagic64: + return parseMachOThin(d, 0, binary.LittleEndian, true) + case machoCigam32: + return parseMachOThin(d, 0, binary.BigEndian, false) + case machoCigam64: + return parseMachOThin(d, 0, binary.BigEndian, true) + } + return nil, errors.New("not a Mach-O file") +} + +func parseMachOFat(d *document.Document, is64 bool) (*Field, error) { + root := &Field{Name: "Mach-O fat", Type: "container", Offset: 0} + hdr, err := d.Range(0, 8) + if err != nil { + return nil, err + } + n := int64(binary.BigEndian.Uint32(hdr[4:8])) + root.Children = append(root.Children, &Field{ + Name: "magic", Type: "u32", Offset: 0, Length: 4, + Value: fmt.Sprintf("0x%08x", binary.BigEndian.Uint32(hdr[:4])), + }) + root.Children = append(root.Children, &Field{ + Name: "nfat_arch", Type: "u32", Offset: 4, Length: 4, + Value: fmt.Sprintf("%d", n), + }) + + archSize := int64(20) + if is64 { + archSize = 32 + } + off := int64(8) + for i := int64(0); i < n && off+archSize <= d.Size(); i++ { + arch, err := d.Range(off, archSize) + if err != nil { + break + } + cpuType := binary.BigEndian.Uint32(arch[0:4]) + cpuSub := binary.BigEndian.Uint32(arch[4:8]) + var archOff, archLen int64 + if is64 { + archOff = int64(binary.BigEndian.Uint64(arch[8:16])) + archLen = int64(binary.BigEndian.Uint64(arch[16:24])) + } else { + archOff = int64(binary.BigEndian.Uint32(arch[8:12])) + archLen = int64(binary.BigEndian.Uint32(arch[12:16])) + } + node := &Field{ + Name: fmt.Sprintf("arch[%d]", i), + Type: "fat_arch", + Offset: off, + Length: archSize, + Value: fmt.Sprintf("cpu=%s sub=0x%x offset=%d size=%d", cpuTypeName(cpuType), cpuSub, archOff, archLen), + } + // Parse the nested Mach-O at archOff if it fits. + if archOff > 0 && archOff+8 <= d.Size() { + if nested, err := parseMachOInline(d, archOff); err == nil { + node.Children = append(node.Children, nested) + } + } + root.Children = append(root.Children, node) + off += archSize + } + return root, nil +} + +// parseMachOInline is like ParseMachO but anchored at a given offset +// (used for each arch inside a fat binary). +func parseMachOInline(d *document.Document, at int64) (*Field, error) { + head, err := d.Range(at, 8) + if err != nil || len(head) < 8 { + return nil, errors.New("short arch payload") + } + mLE := binary.LittleEndian.Uint32(head[:4]) + switch mLE { + case machoMagic32: + return parseMachOThin(d, at, binary.LittleEndian, false) + case machoMagic64: + return parseMachOThin(d, at, binary.LittleEndian, true) + case machoCigam32: + return parseMachOThin(d, at, binary.BigEndian, false) + case machoCigam64: + return parseMachOThin(d, at, binary.BigEndian, true) + } + return nil, errors.New("arch payload is not Mach-O") +} + +func parseMachOThin(d *document.Document, base int64, bo binary.ByteOrder, is64 bool) (*Field, error) { + headerLen := int64(28) + if is64 { + headerLen = 32 + } + if base+headerLen > d.Size() { + return nil, errors.New("truncated Mach-O header") + } + hdr, err := d.Range(base, headerLen) + if err != nil { + return nil, err + } + + magic := bo.Uint32(hdr[0:4]) + cpuType := bo.Uint32(hdr[4:8]) + cpuSub := bo.Uint32(hdr[8:12]) + fileType := bo.Uint32(hdr[12:16]) + ncmds := bo.Uint32(hdr[16:20]) + sizeOfCmds := bo.Uint32(hdr[20:24]) + flags := bo.Uint32(hdr[24:28]) + + name := "Mach-O (32)" + if is64 { + name = "Mach-O (64)" + } + root := &Field{Name: name, Type: "container", Offset: base, Length: headerLen + int64(sizeOfCmds)} + root.Children = append(root.Children, + &Field{Name: "magic", Type: "u32", Offset: base, Length: 4, Value: fmt.Sprintf("0x%08x", magic)}, + &Field{Name: "cputype", Type: "u32", Offset: base + 4, Length: 4, Value: cpuTypeName(cpuType)}, + &Field{Name: "cpusubtype", Type: "u32", Offset: base + 8, Length: 4, Value: fmt.Sprintf("0x%x", cpuSub)}, + &Field{Name: "filetype", Type: "u32", Offset: base + 12, Length: 4, Value: machoFileType(fileType)}, + &Field{Name: "ncmds", Type: "u32", Offset: base + 16, Length: 4, Value: fmt.Sprintf("%d", ncmds)}, + &Field{Name: "sizeofcmds", Type: "u32", Offset: base + 20, Length: 4, Value: fmt.Sprintf("%d", sizeOfCmds)}, + &Field{Name: "flags", Type: "u32", Offset: base + 24, Length: 4, Value: fmt.Sprintf("0x%08x", flags)}, + ) + + // Load commands. + cmdOff := base + headerLen + cmdEnd := cmdOff + int64(sizeOfCmds) + if cmdEnd > d.Size() { + cmdEnd = d.Size() + } + cmds := &Field{Name: "load_commands", Type: "list", Offset: cmdOff, Length: int64(sizeOfCmds)} + for i := uint32(0); i < ncmds && cmdOff+8 <= cmdEnd; i++ { + h, err := d.Range(cmdOff, 8) + if err != nil { + break + } + cmd := bo.Uint32(h[0:4]) + sz := int64(bo.Uint32(h[4:8])) + if sz < 8 || cmdOff+sz > cmdEnd { + break + } + buf, _ := d.Range(cmdOff, sz) + node := decodeLoadCommand(base, cmdOff, cmd, sz, buf, bo, is64) + cmds.Children = append(cmds.Children, node) + cmdOff += sz + } + root.Children = append(root.Children, cmds) + return root, nil +} + +func decodeLoadCommand(base, off int64, cmd uint32, sz int64, buf []byte, bo binary.ByteOrder, is64 bool) *Field { + name, mask := loadCommandName(cmd) + node := &Field{ + Name: name, Type: "load_command", Offset: off, Length: sz, + Value: fmt.Sprintf("cmd=0x%08x size=%d", cmd, sz), + } + if mask != 0 { + node.Note = "required" + } + + switch { + case (cmd&^0x80000000) == 0x19 && is64 && sz >= 72: + // LC_SEGMENT_64 + segName := cstring(buf[8:24]) + vmAddr := bo.Uint64(buf[24:32]) + vmSize := bo.Uint64(buf[32:40]) + fileOff := bo.Uint64(buf[40:48]) + fileSize := bo.Uint64(buf[48:56]) + nsects := bo.Uint32(buf[64:68]) + node.Value = fmt.Sprintf("name=%q vm=%#x+%#x file=%#x+%#x nsects=%d", + segName, vmAddr, vmSize, fileOff, fileSize, nsects) + sectOff := int64(72) + for i := uint32(0); i < nsects && sectOff+80 <= sz; i++ { + s := buf[sectOff:] + sn := cstring(s[0:16]) + seg := cstring(s[16:32]) + addr := bo.Uint64(s[32:40]) + size := bo.Uint64(s[40:48]) + foff := bo.Uint32(s[48:52]) + node.Children = append(node.Children, &Field{ + Name: sn, Type: "section64", Offset: off + sectOff, Length: 80, + Value: fmt.Sprintf("seg=%q addr=%#x size=%#x file=%#x", seg, addr, size, foff), + }) + sectOff += 80 + } + case cmd == 0x01 && sz >= 56: + // LC_SEGMENT (32) + segName := cstring(buf[8:24]) + vmAddr := bo.Uint32(buf[24:28]) + vmSize := bo.Uint32(buf[28:32]) + fileOff := bo.Uint32(buf[32:36]) + fileSize := bo.Uint32(buf[36:40]) + nsects := bo.Uint32(buf[48:52]) + node.Value = fmt.Sprintf("name=%q vm=%#x+%#x file=%#x+%#x nsects=%d", + segName, vmAddr, vmSize, fileOff, fileSize, nsects) + case cmd == 0x1b && sz >= 24: + // LC_UUID + u := buf[8:24] + node.Value = fmt.Sprintf("UUID=%x-%x-%x-%x-%x", u[0:4], u[4:6], u[6:8], u[8:10], u[10:16]) + case (cmd&^0x80000000) == 0x0c && sz > 24: + // LC_LOAD_DYLIB / similar + strOff := bo.Uint32(buf[8:12]) + if int(strOff) < len(buf) { + node.Value = fmt.Sprintf("lib=%q", cstring(buf[strOff:])) + } + case cmd == 0x0e && sz > 12: + // LC_LOAD_DYLINKER + strOff := bo.Uint32(buf[8:12]) + if int(strOff) < len(buf) { + node.Value = fmt.Sprintf("dylinker=%q", cstring(buf[strOff:])) + } + case cmd == 0x21 && sz >= 16: + // LC_ENCRYPTION_INFO / _64 + node.Value = "encryption info" + case (cmd &^ 0x80000000) == 0x22: + node.Value = "LC_DYLD_INFO" + case cmd == 0x25: + node.Value = "LC_VERSION_MIN_MACOSX" + case cmd == 0x28 || cmd == 0x29: + node.Value = "LC_MAIN / entry point" + } + return node +} + +func cstring(b []byte) string { + for i, c := range b { + if c == 0 { + return string(b[:i]) + } + } + return string(b) +} + +func cpuTypeName(c uint32) string { + switch c { + case 0x7: + return "x86" + case 0x1000007: + return "x86_64" + case 0xc: + return "arm" + case 0x100000c: + return "arm64" + case 0x12: + return "ppc" + case 0x1000012: + return "ppc64" + } + return fmt.Sprintf("0x%x", c) +} + +func machoFileType(t uint32) string { + switch t { + case 1: + return "OBJECT" + case 2: + return "EXECUTE" + case 3: + return "FVMLIB" + case 4: + return "CORE" + case 5: + return "PRELOAD" + case 6: + return "DYLIB" + case 7: + return "DYLINKER" + case 8: + return "BUNDLE" + case 9: + return "DYLIB_STUB" + case 10: + return "DSYM" + case 11: + return "KEXT_BUNDLE" + } + return fmt.Sprintf("%d", t) +} + +// loadCommandName returns a human name and whether the required bit is set. +func loadCommandName(cmd uint32) (string, uint32) { + req := cmd & 0x80000000 + v := cmd &^ 0x80000000 + names := map[uint32]string{ + 0x01: "LC_SEGMENT", + 0x02: "LC_SYMTAB", + 0x03: "LC_SYMSEG", + 0x04: "LC_THREAD", + 0x05: "LC_UNIXTHREAD", + 0x06: "LC_LOADFVMLIB", + 0x07: "LC_IDFVMLIB", + 0x08: "LC_IDENT", + 0x09: "LC_FVMFILE", + 0x0a: "LC_PREPAGE", + 0x0b: "LC_DYSYMTAB", + 0x0c: "LC_LOAD_DYLIB", + 0x0d: "LC_ID_DYLIB", + 0x0e: "LC_LOAD_DYLINKER", + 0x0f: "LC_ID_DYLINKER", + 0x10: "LC_PREBOUND_DYLIB", + 0x11: "LC_ROUTINES", + 0x12: "LC_SUB_FRAMEWORK", + 0x13: "LC_SUB_UMBRELLA", + 0x14: "LC_SUB_CLIENT", + 0x15: "LC_SUB_LIBRARY", + 0x16: "LC_TWOLEVEL_HINTS", + 0x17: "LC_PREBIND_CKSUM", + 0x18: "LC_LOAD_WEAK_DYLIB", + 0x19: "LC_SEGMENT_64", + 0x1a: "LC_ROUTINES_64", + 0x1b: "LC_UUID", + 0x1c: "LC_RPATH", + 0x1d: "LC_CODE_SIGNATURE", + 0x1e: "LC_SEGMENT_SPLIT_INFO", + 0x1f: "LC_REEXPORT_DYLIB", + 0x20: "LC_LAZY_LOAD_DYLIB", + 0x21: "LC_ENCRYPTION_INFO", + 0x22: "LC_DYLD_INFO", + 0x23: "LC_LOAD_UPWARD_DYLIB", + 0x24: "LC_VERSION_MIN_MACOSX", + 0x25: "LC_VERSION_MIN_IPHONEOS", + 0x26: "LC_FUNCTION_STARTS", + 0x27: "LC_DYLD_ENVIRONMENT", + 0x28: "LC_MAIN", + 0x29: "LC_DATA_IN_CODE", + 0x2a: "LC_SOURCE_VERSION", + 0x2b: "LC_DYLIB_CODE_SIGN_DRS", + 0x2c: "LC_ENCRYPTION_INFO_64", + 0x2d: "LC_LINKER_OPTION", + 0x2e: "LC_LINKER_OPTIMIZATION_HINT", + 0x2f: "LC_VERSION_MIN_TVOS", + 0x30: "LC_VERSION_MIN_WATCHOS", + 0x31: "LC_NOTE", + 0x32: "LC_BUILD_VERSION", + 0x33: "LC_DYLD_EXPORTS_TRIE", + 0x34: "LC_DYLD_CHAINED_FIXUPS", + 0x35: "LC_FILESET_ENTRY", + } + if n, ok := names[v]; ok { + return n, req + } + return fmt.Sprintf("LC_0x%x", v), req +} diff --git a/internal/leviathan/parsers/peimports.go b/internal/leviathan/parsers/peimports.go new file mode 100644 index 0000000..af79b9b --- /dev/null +++ b/internal/leviathan/parsers/peimports.go @@ -0,0 +1,150 @@ +package parsers + +import ( + "encoding/binary" + "errors" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// ImportSym is one resolved PE import (a function name, or an ordinal). +type ImportSym struct { + DLL string `json:"dll"` + Func string `json:"func"` + Ordinal int `json:"ordinal"` + ByOrdinal bool `json:"byOrdinal"` +} + +// PEImports walks the import directory of a PE file and returns the ordered +// list of imported symbols. It reuses the same RVA mapping and header parsing +// as ParsePE but yields a clean typed result (for imphash, capability +// analysis, etc.) rather than a display tree. +func PEImports(d *document.Document) ([]ImportSym, error) { + sections, dataDirs, is64, err := peLayout(d) + if err != nil { + return nil, err + } + if len(dataDirs) < 2 || dataDirs[1].RVA == 0 { + return nil, errors.New("no import directory") + } + off, ok := peRVAToOffset(sections, dataDirs[1].RVA) + if !ok { + return nil, errors.New("import RVA unmapped") + } + var out []ImportSym + for i := 0; i < 1024; i++ { + at := int64(off) + int64(i*20) + buf, err := d.Range(at, 20) + if err != nil || len(buf) < 20 { + break + } + oft := binary.LittleEndian.Uint32(buf[0:4]) + nameRVA := binary.LittleEndian.Uint32(buf[12:16]) + firstThunk := binary.LittleEndian.Uint32(buf[16:20]) + if oft == 0 && nameRVA == 0 && firstThunk == 0 { + break + } + dll := "" + if nameOff, ok := peRVAToOffset(sections, nameRVA); ok { + dll = readCString(d, int64(nameOff), 256) + } + if dll == "" { + continue + } + out = append(out, peThunkSyms(d, sections, dll, oft, firstThunk, is64)...) + if len(out) > 100000 { + break + } + } + return out, nil +} + +func peThunkSyms(d *document.Document, sections []peSectionInfo, dll string, oft, firstThunk uint32, is64 bool) []ImportSym { + thunkRVA := oft + if thunkRVA == 0 { + thunkRVA = firstThunk + } + thunkOff, ok := peRVAToOffset(sections, thunkRVA) + if !ok { + return nil + } + width := int64(4) + ordinalMask := uint64(0x80000000) + if is64 { + width = 8 + ordinalMask = 0x8000000000000000 + } + var out []ImportSym + for i := 0; i < 8192; i++ { + at := int64(thunkOff) + int64(i)*width + buf, err := d.Range(at, width) + if err != nil || int64(len(buf)) < width { + break + } + var thunk uint64 + if is64 { + thunk = binary.LittleEndian.Uint64(buf) + } else { + thunk = uint64(binary.LittleEndian.Uint32(buf)) + } + if thunk == 0 { + break + } + if thunk&ordinalMask != 0 { + out = append(out, ImportSym{DLL: dll, Ordinal: int(thunk & 0xffff), ByOrdinal: true}) + } else if nameOff, ok := peRVAToOffset(sections, uint32(thunk)); ok { + name := readImportByName(d, int64(nameOff)) + if name != "" { + out = append(out, ImportSym{DLL: dll, Func: name}) + } + } + } + return out +} + +// peLayout extracts the section table, data directories and 32/64-bitness from +// a PE without building the display field tree. +func peLayout(d *document.Document) ([]peSectionInfo, []peDataDirectory, bool, error) { + hdr, err := d.Range(0, 64) + if err != nil || len(hdr) < 64 { + return nil, nil, false, errors.New("too short") + } + lfanew := int64(binary.LittleEndian.Uint32(hdr[0x3c:0x40])) + peHdr, err := d.Range(lfanew, 24) + if err != nil || len(peHdr) < 24 || string(peHdr[:4]) != "PE\x00\x00" { + return nil, nil, false, errors.New("no PE signature") + } + numSections := int(binary.LittleEndian.Uint16(peHdr[6:8])) + sizeOptHdr := int64(binary.LittleEndian.Uint16(peHdr[20:22])) + optOff := lfanew + 24 + + var dataDirs []peDataDirectory + is64 := false + if sizeOptHdr > 0 { + opt, err := d.Range(optOff, sizeOptHdr) + if err == nil && len(opt) >= 2 { + _, dirs := parsePEOptionalHeader(opt, optOff) + dataDirs = dirs + is64 = binary.LittleEndian.Uint16(opt[0:2]) == 0x20b + } + } + + sectOff := lfanew + 24 + sizeOptHdr + sections := make([]peSectionInfo, 0, numSections) + for i := 0; i < numSections; i++ { + sh, err := d.Range(sectOff, 40) + if err != nil || len(sh) < 40 { + break + } + sections = append(sections, peSectionInfo{ + Name: string(trimZero(sh[0:8])), + VirtualSize: binary.LittleEndian.Uint32(sh[8:12]), + VirtualAddress: binary.LittleEndian.Uint32(sh[12:16]), + RawSize: binary.LittleEndian.Uint32(sh[16:20]), + RawOffset: binary.LittleEndian.Uint32(sh[20:24]), + Characteristics: binary.LittleEndian.Uint32(sh[36:40]), + }) + sectOff += 40 + } + return sections, dataDirs, is64, nil +} diff --git a/internal/leviathan/patch/script.go b/internal/leviathan/patch/script.go new file mode 100644 index 0000000..5727448 --- /dev/null +++ b/internal/leviathan/patch/script.go @@ -0,0 +1,232 @@ +// Package patch implements the simple Leviathan patch script language. +// +// set OFFSET HEX... — write hex bytes at OFFSET +// fill START LEN BYTE — fill a range with a single byte +// xor START LEN KEY — XOR a range with an ASCII or hex: key +// copy SRC LEN DST — copy bytes from SRC to DST +// # comments and blank lines are allowed. +package patch + +import ( + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// Step is one parsed instruction. +type Step struct { + Op string + Args []string + Line int + Source string +} + +// Parse parses the script into steps. +func Parse(src string) ([]Step, error) { + out := []Step{} + for i, raw := range strings.Split(src, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "//") { + continue + } + fields := strings.Fields(line) + op := strings.ToLower(fields[0]) + out = append(out, Step{ + Op: op, Args: fields[1:], Line: i + 1, Source: raw, + }) + } + return out, nil +} + +// Plan validates all steps against document size without applying. +// Returns per-step info or an error on validation failure. +func Plan(d *document.Document, steps []Step) ([]map[string]any, error) { + size := d.Size() + info := make([]map[string]any, 0, len(steps)) + for _, s := range steps { + switch s.Op { + case "set": + if len(s.Args) < 2 { + return info, fmt.Errorf("line %d: set needs OFFSET and HEX", s.Line) + } + off, err := parseOffset(s.Args[0]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + b, err := parseHexRest(s.Args[1:]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + if !validRange(size, off, int64(len(b))) { + return info, fmt.Errorf("line %d: out of range", s.Line) + } + info = append(info, map[string]any{ + "op": "set", "line": s.Line, "offset": off, "length": len(b), + }) + case "fill": + if len(s.Args) != 3 { + return info, fmt.Errorf("line %d: fill needs START LEN BYTE", s.Line) + } + off, err := parseOffset(s.Args[0]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + n, err := parseOffset(s.Args[1]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + by, err := parseHexByte(s.Args[2]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + if !validRange(size, off, n) { + return info, fmt.Errorf("line %d: out of range", s.Line) + } + info = append(info, map[string]any{ + "op": "fill", "line": s.Line, "offset": off, "length": n, "byte": by, + }) + case "xor": + if len(s.Args) != 3 { + return info, fmt.Errorf("line %d: xor needs START LEN KEY", s.Line) + } + off, err := parseOffset(s.Args[0]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + n, err := parseOffset(s.Args[1]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + key, err := parseKey(s.Args[2]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + if !validRange(size, off, n) { + return info, fmt.Errorf("line %d: out of range", s.Line) + } + info = append(info, map[string]any{ + "op": "xor", "line": s.Line, "offset": off, "length": n, "keyLen": len(key), + }) + case "copy": + if len(s.Args) != 3 { + return info, fmt.Errorf("line %d: copy needs SRC LEN DST", s.Line) + } + src, err := parseOffset(s.Args[0]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + n, err := parseOffset(s.Args[1]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + dst, err := parseOffset(s.Args[2]) + if err != nil { + return info, fmt.Errorf("line %d: %v", s.Line, err) + } + if !validRange(size, src, n) || !validRange(size, dst, n) { + return info, fmt.Errorf("line %d: out of range", s.Line) + } + info = append(info, map[string]any{ + "op": "copy", "line": s.Line, "src": src, "dst": dst, "length": n, + }) + default: + return info, fmt.Errorf("line %d: unknown op %q", s.Line, s.Op) + } + } + return info, nil +} + +// Apply runs the script against the document. Callers may want to +// first snapshot a checkpoint so apply can be reverted. +func Apply(d *document.Document, steps []Step) error { + if _, err := Plan(d, steps); err != nil { + return err + } + for _, s := range steps { + switch s.Op { + case "set": + off, _ := parseOffset(s.Args[0]) + b, _ := parseHexRest(s.Args[1:]) + for i, bb := range b { + if _, err := d.Set(off+int64(i), bb); err != nil { + return err + } + } + case "fill": + off, _ := parseOffset(s.Args[0]) + n, _ := parseOffset(s.Args[1]) + by, _ := parseHexByte(s.Args[2]) + if err := d.Fill(off, n, by); err != nil { + return err + } + case "xor": + off, _ := parseOffset(s.Args[0]) + n, _ := parseOffset(s.Args[1]) + key, _ := parseKey(s.Args[2]) + if err := d.XOR(off, n, key); err != nil { + return err + } + case "copy": + src, _ := parseOffset(s.Args[0]) + n, _ := parseOffset(s.Args[1]) + dst, _ := parseOffset(s.Args[2]) + if err := d.Copy(src, n, dst); err != nil { + return err + } + } + } + return nil +} + +func validRange(size, start, length int64) bool { + return start >= 0 && start <= size && length >= 0 && length <= size-start +} + +func parseOffset(s string) (int64, error) { + s = strings.TrimSpace(s) + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + v, err := strconv.ParseInt(s[2:], 16, 64) + return v, err + } + return strconv.ParseInt(s, 10, 64) +} + +func parseHexRest(args []string) ([]byte, error) { + joined := strings.Join(args, "") + joined = strings.ReplaceAll(joined, " ", "") + joined = strings.ReplaceAll(joined, ",", "") + return hex.DecodeString(joined) +} + +func parseHexByte(s string) (byte, error) { + if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { + v, err := strconv.ParseUint(s[2:], 16, 8) + return byte(v), err + } + b, err := hex.DecodeString(s) + if err != nil { + v, err := strconv.ParseUint(s, 10, 8) + if err != nil { + return 0, errors.New("not a byte") + } + return byte(v), nil + } + if len(b) != 1 { + return 0, errors.New("expected one byte") + } + return b[0], nil +} + +func parseKey(s string) ([]byte, error) { + if strings.HasPrefix(s, "hex:") { + return hex.DecodeString(s[4:]) + } + if s == "" { + return nil, errors.New("empty key") + } + return []byte(s), nil +} diff --git a/internal/leviathan/pdf/filters.go b/internal/leviathan/pdf/filters.go new file mode 100644 index 0000000..fe707e2 --- /dev/null +++ b/internal/leviathan/pdf/filters.go @@ -0,0 +1,411 @@ +package pdf + +import ( + "bytes" + "errors" + "fmt" + + "github.com/andydixon/nexus/internal/leviathan/transforms" +) + +const MaxDecodedStreamOutput = 64 << 20 + +// ApplyFilter decodes one PDF stream filter. When a predictor is in use +// it must be applied separately after this function returns raw filter +// output; see ApplyPredictor. +func ApplyFilter(name string, in []byte) ([]byte, error) { + switch name { + case "FlateDecode", "Fl": + if out, err := transforms.InflateZlib(in); err == nil { + return out, nil + } + return transforms.InflateRaw(in) + case "ASCIIHexDecode", "AHx": + return decodeASCIIHex(in) + case "ASCII85Decode", "A85": + return decodeASCII85(in) + case "RunLengthDecode", "RL": + return decodeRunLength(in) + case "LZWDecode", "LZW": + return decodeLZW(in) + case "CCITTFaxDecode", "CCF": + return nil, errors.New("CCITTFaxDecode not supported (returns raw bytes)") + case "JBIG2Decode": + return nil, errors.New("JBIG2Decode not supported") + case "DCTDecode", "DCT": + // JPEG payload — return as-is, caller can save as .jpg. + return in, nil + case "JPXDecode": + return in, nil + case "Crypt": + return nil, errors.New("Crypt filter not supported") + } + return nil, fmt.Errorf("unknown filter %q", name) +} + +// ApplyPredictor undoes the PNG-family predictor used with Flate/LZW. If +// predictor <= 1 the data is returned unchanged. +func ApplyPredictor(in []byte, predictor, columns, colors, bits int) ([]byte, error) { + if predictor <= 1 { + return in, nil + } + if len(in) > MaxDecodedStreamOutput { + return nil, fmt.Errorf("predictor input exceeds %d byte limit", MaxDecodedStreamOutput) + } + bytesPerPixel, rowLen, err := predictorGeometry(columns, colors, bits) + if err != nil { + return nil, err + } + + switch { + case predictor == 2: + // TIFF predictor — apply left delta per sample. + out := make([]byte, len(in)) + copy(out, in) + step := bytesPerPixel + for i := step; i < len(out); i++ { + if i%rowLen < step { + continue + } + out[i] = out[i] + out[i-step] + } + return out, nil + case predictor >= 10 && predictor <= 15: + // PNG predictors — each row is prefixed with a filter-type byte. + stride := rowLen + 1 + out := make([]byte, 0, len(in)) + var prev []byte + for i := 0; i < len(in); i += stride { + if i+stride > len(in) { + break + } + filterType := in[i] + row := make([]byte, rowLen) + copy(row, in[i+1:i+stride]) + switch filterType { + case 0: // None + case 1: // Sub + for j := bytesPerPixel; j < rowLen; j++ { + row[j] += row[j-bytesPerPixel] + } + case 2: // Up + if prev != nil { + for j := 0; j < rowLen; j++ { + row[j] += prev[j] + } + } + case 3: // Average + for j := 0; j < rowLen; j++ { + left := byte(0) + if j >= bytesPerPixel { + left = row[j-bytesPerPixel] + } + up := byte(0) + if prev != nil { + up = prev[j] + } + row[j] += byte((int(left) + int(up)) / 2) + } + case 4: // Paeth + for j := 0; j < rowLen; j++ { + a := 0 + if j >= bytesPerPixel { + a = int(row[j-bytesPerPixel]) + } + b := 0 + if prev != nil { + b = int(prev[j]) + } + c := 0 + if prev != nil && j >= bytesPerPixel { + c = int(prev[j-bytesPerPixel]) + } + p := a + b - c + pa := abs(p - a) + pb := abs(p - b) + pc := abs(p - c) + pred := 0 + switch { + case pa <= pb && pa <= pc: + pred = a + case pb <= pc: + pred = b + default: + pred = c + } + row[j] += byte(pred) + } + default: + return nil, fmt.Errorf("PNG predictor: invalid filter type %d", filterType) + } + out = append(out, row...) + prev = row + } + return out, nil + } + return nil, fmt.Errorf("predictor %d not supported", predictor) +} + +func predictorGeometry(columns, colors, bits int) (int, int, error) { + if columns <= 0 || colors <= 0 { + return 0, 0, errors.New("predictor columns and colors must be positive") + } + switch bits { + case 1, 2, 4, 8, 16: + default: + return 0, 0, fmt.Errorf("predictor bits per component %d is invalid", bits) + } + pixelBits := int64(colors) * int64(bits) + rowBits := int64(columns) * pixelBits + if pixelBits <= 0 || rowBits <= 0 || rowBits > int64(MaxDecodedStreamOutput)*8 { + return 0, 0, errors.New("predictor dimensions exceed resource limits") + } + return int((pixelBits + 7) / 8), int((rowBits + 7) / 8), nil +} + +func ensureDecodedSize(current, additional int) error { + if additional < 0 || current > MaxDecodedStreamOutput-additional { + return fmt.Errorf("decoded stream exceeds %d byte limit", MaxDecodedStreamOutput) + } + return nil +} + +func abs(x int) int { + if x < 0 { + return -x + } + return x +} + +// --------------------------------------------------------------------------- +// ASCIIHexDecode +// --------------------------------------------------------------------------- + +func decodeASCIIHex(in []byte) ([]byte, error) { + out := make([]byte, 0, len(in)/2) + hi := -1 + for _, b := range in { + switch { + case b >= '0' && b <= '9': + b = b - '0' + case b >= 'a' && b <= 'f': + b = b - 'a' + 10 + case b >= 'A' && b <= 'F': + b = b - 'A' + 10 + case b == '>': + if hi >= 0 { + out = append(out, byte(hi)<<4) + } + return out, nil + case isPDFWhitespace(b): + continue + default: + continue + } + if hi < 0 { + hi = int(b) + } else { + out = append(out, byte(hi)<<4|b) + hi = -1 + } + } + if hi >= 0 { + out = append(out, byte(hi)<<4) + } + return out, nil +} + +func isPDFWhitespace(b byte) bool { + return b == 0 || b == '\t' || b == '\n' || b == '\f' || b == '\r' || b == ' ' +} + +// --------------------------------------------------------------------------- +// ASCII85Decode +// --------------------------------------------------------------------------- + +func decodeASCII85(in []byte) ([]byte, error) { + // Strip optional <~ ... ~> framing and whitespace. + start := 0 + if bytes.HasPrefix(in, []byte("<~")) { + start = 2 + } + out := make([]byte, 0, len(in)) + var group [5]byte + n := 0 + for i := start; i < len(in); i++ { + c := in[i] + if isPDFWhitespace(c) { + continue + } + if c == '~' { + break + } + if c == 'z' && n == 0 { + if err := ensureDecodedSize(len(out), 4); err != nil { + return nil, err + } + out = append(out, 0, 0, 0, 0) + continue + } + if c < '!' || c > 'u' { + return nil, fmt.Errorf("ASCII85: invalid byte 0x%02x", c) + } + group[n] = c - '!' + n++ + if n == 5 { + v := uint32(group[0])*85*85*85*85 + + uint32(group[1])*85*85*85 + + uint32(group[2])*85*85 + + uint32(group[3])*85 + + uint32(group[4]) + if err := ensureDecodedSize(len(out), 4); err != nil { + return nil, err + } + out = append(out, byte(v>>24), byte(v>>16), byte(v>>8), byte(v)) + n = 0 + } + } + if n > 0 { + for i := n; i < 5; i++ { + group[i] = 84 + } + v := uint32(group[0])*85*85*85*85 + + uint32(group[1])*85*85*85 + + uint32(group[2])*85*85 + + uint32(group[3])*85 + + uint32(group[4]) + full := []byte{byte(v >> 24), byte(v >> 16), byte(v >> 8), byte(v)} + if err := ensureDecodedSize(len(out), n-1); err != nil { + return nil, err + } + out = append(out, full[:n-1]...) + } + return out, nil +} + +// --------------------------------------------------------------------------- +// RunLengthDecode +// --------------------------------------------------------------------------- + +func decodeRunLength(in []byte) ([]byte, error) { + out := make([]byte, 0, len(in)) + i := 0 + for i < len(in) { + b := in[i] + i++ + switch { + case b == 128: + return out, nil + case b < 128: + n := int(b) + 1 + if i+n > len(in) { + return nil, errors.New("RunLength: truncated literal") + } + if err := ensureDecodedSize(len(out), n); err != nil { + return nil, err + } + out = append(out, in[i:i+n]...) + i += n + default: + n := 257 - int(b) + if i >= len(in) { + return nil, errors.New("RunLength: truncated run") + } + if err := ensureDecodedSize(len(out), n); err != nil { + return nil, err + } + for k := 0; k < n; k++ { + out = append(out, in[i]) + } + i++ + } + } + return out, nil +} + +// --------------------------------------------------------------------------- +// LZWDecode (variable-width, early-change = 1) +// --------------------------------------------------------------------------- + +func decodeLZW(in []byte) ([]byte, error) { + const ( + clearCode = 256 + eodCode = 257 + ) + reader := &bitReader{data: in} + table := make([][]byte, 258) + for i := 0; i < 256; i++ { + table[i] = []byte{byte(i)} + } + codeWidth := 9 + out := []byte{} + prev := []byte(nil) + for { + c, err := reader.read(codeWidth) + if err != nil { + return out, nil + } + if c == eodCode { + return out, nil + } + if c == clearCode { + table = table[:258] + codeWidth = 9 + prev = nil + continue + } + var entry []byte + switch { + case c < len(table): + entry = table[c] + case c == len(table) && prev != nil: + entry = append([]byte{}, prev...) + entry = append(entry, prev[0]) + default: + return out, errors.New("LZW: invalid code") + } + if err := ensureDecodedSize(len(out), len(entry)); err != nil { + return nil, err + } + out = append(out, entry...) + if prev != nil { + ne := append([]byte{}, prev...) + ne = append(ne, entry[0]) + table = append(table, ne) + // PDF LZW uses "early change": widen one code before full. + switch len(table) { + case 511, 1023, 2047: + codeWidth++ + case 4096: + // Implicit CLEAR expected; some encoders omit it. + table = table[:258] + codeWidth = 9 + prev = nil + continue + } + } + prev = entry + } +} + +type bitReader struct { + data []byte + bi int // bit offset into data +} + +func (r *bitReader) read(n int) (int, error) { + if n <= 0 || n > 32 { + return 0, errors.New("bitReader: bad width") + } + v := 0 + for i := 0; i < n; i++ { + byteIdx := r.bi / 8 + if byteIdx >= len(r.data) { + return 0, errors.New("bitReader: EOF") + } + bit := (r.data[byteIdx] >> uint(7-(r.bi%8))) & 1 + v = (v << 1) | int(bit) + r.bi++ + } + return v, nil +} diff --git a/internal/leviathan/pdf/pdf.go b/internal/leviathan/pdf/pdf.go new file mode 100644 index 0000000..ce18ef1 --- /dev/null +++ b/internal/leviathan/pdf/pdf.go @@ -0,0 +1,352 @@ +// Package pdf provides PDF surgery: object enumeration, full stream +// filter chain, cross-reference parsing (classical tables and xref +// streams), compressed object-stream decoding, and surfacing of hostile +// or otherwise-interesting constructs (JavaScript, embedded files, URI +// and launch actions, XFA, OpenAction triggers). +// +// It is not a rendering engine — it will not produce pixels. It is a +// structural tool for humans and detection pipelines. +package pdf + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +const maxPDFParseBytes = 128 << 20 + +// Object is one PDF indirect object. +type Object struct { + Num int64 + Gen int64 + Offset int64 + Dict string + HasStream bool + StreamOff int64 + StreamLen int64 + Filters []string + Type string + Subtype string + Source string // "direct" or "objstm" + Container int64 // when Source == "objstm", the ObjStm number +} + +// Parse returns a summary of the PDF structure, including cross- +// reference entries, an object list (direct + recovered from ObjStms), +// surfaced suspicious constructs, and top-level flags. +func Parse(d *document.Document) (map[string]any, error) { + size := d.Size() + head, _ := d.Range(0, 16) + if len(head) < 5 || string(head[:5]) != "%PDF-" { + return nil, errors.New("not a PDF header") + } + if size > maxPDFParseBytes { + return nil, fmt.Errorf("pdf parse skipped: file > %d MiB", maxPDFParseBytes>>20) + } + version := string(head[5:8]) + + whole, err := d.Range(0, size) + if err != nil { + return nil, err + } + + objs := scanIndirectObjects(whole) + + // Try xref first; fall back gracefully if it isn't parseable. + xref, xrefMode, xrefErr := ParseXRef(d, whole) + xrefNote := "" + if xrefErr != nil { + xrefNote = xrefErr.Error() + } + + // Decode any ObjStm streams we find and append recovered children. + recovered := recoverObjectStreams(whole, d, objs) + objs = append(objs, recovered...) + + // Surface flags. + surfaced := Surface(objs) + encrypted := bytes.Contains(whole, []byte("/Encrypt")) + linearised := HasLinearisation(whole) + + // Locate trailer text (for quick eyeball). + trailerText := "" + if tIdx := bytes.LastIndex(whole, []byte("trailer")); tIdx >= 0 { + end := bytes.Index(whole[tIdx:], []byte("startxref")) + if end >= 0 { + trailerText = string(whole[tIdx : tIdx+end]) + } + } + + return map[string]any{ + "version": version, + "size": size, + "encrypted": encrypted, + "linearised": linearised, + "trailer": trailerText, + "objects": objectsToMaps(objs), + "xref": xrefEntriesToMaps(xref), + "xrefMode": xrefMode, + "xrefNote": xrefNote, + "suspicious": surfaceToMaps(surfaced), + }, nil +} + +func scanIndirectObjects(whole []byte) []Object { + objRE := regexp.MustCompile(`(?m)(\d+)\s+(\d+)\s+obj\b`) + locs := objRE.FindAllSubmatchIndex(whole, -1) + objs := make([]Object, 0, len(locs)) + for _, loc := range locs { + start := int64(loc[0]) + num, _ := strconv.ParseInt(string(whole[loc[2]:loc[3]]), 10, 64) + gen, _ := strconv.ParseInt(string(whole[loc[4]:loc[5]]), 10, 64) + o := Object{Num: num, Gen: gen, Offset: start, Source: "direct"} + endIdx := bytes.Index(whole[loc[1]:], []byte("endobj")) + if endIdx < 0 { + continue + } + body := whole[loc[1] : loc[1]+endIdx] + streamIdx := bytes.Index(body, []byte("stream")) + var dict []byte + if streamIdx >= 0 { + dict = body[:streamIdx] + s := streamIdx + len("stream") + if s < len(body) && body[s] == '\r' { + s++ + } + if s < len(body) && body[s] == '\n' { + s++ + } + o.HasStream = true + o.StreamOff = int64(loc[1]) + int64(s) + endStreamIdx := bytes.LastIndex(body, []byte("endstream")) + if endStreamIdx < 0 { + continue + } + endStreamAbs := int64(loc[1]) + int64(endStreamIdx) + // trim trailing newline/cr + for k := 0; k < 2; k++ { + if endStreamAbs > 0 && endStreamAbs-1 < int64(len(whole)) { + if whole[endStreamAbs-1] == '\n' || whole[endStreamAbs-1] == '\r' { + endStreamAbs-- + } + } + } + o.StreamLen = endStreamAbs - o.StreamOff + } else { + dict = body + } + o.Dict = strings.TrimSpace(string(dict)) + o.Filters = detectFilters(o.Dict) + o.Type = detectType(o.Dict) + o.Subtype = detectSubtype(o.Dict) + objs = append(objs, o) + } + return objs +} + +func recoverObjectStreams(whole []byte, d *document.Document, objs []Object) []Object { + out := []Object{} + for _, o := range objs { + if !o.HasStream { + continue + } + if !strings.Contains(o.Dict, "/Type /ObjStm") && !strings.Contains(o.Dict, "/Type/ObjStm") { + continue + } + raw, err := d.Range(o.StreamOff, o.StreamLen) + if err != nil { + continue + } + // Apply filters in order before handing to ObjStm decoder. + children, err := DecodeObjStm(raw, o.Dict) + if err != nil { + continue + } + for num, body := range children { + out = append(out, Object{ + Num: num, + Gen: 0, + Offset: o.StreamOff, + Dict: body, + HasStream: false, + Type: detectType(body), + Subtype: detectSubtype(body), + Source: "objstm", + Container: o.Num, + }) + } + } + return out +} + +func objectsToMaps(objs []Object) []map[string]any { + out := make([]map[string]any, len(objs)) + for i, o := range objs { + filters := make([]any, len(o.Filters)) + for j, f := range o.Filters { + filters[j] = f + } + out[i] = map[string]any{ + "num": o.Num, + "gen": o.Gen, + "offset": o.Offset, + "dict": o.Dict, + "hasStream": o.HasStream, + "streamOff": o.StreamOff, + "streamLen": o.StreamLen, + "filters": filters, + "type": o.Type, + "subtype": o.Subtype, + "source": o.Source, + "container": o.Container, + } + } + return out +} + +func xrefEntriesToMaps(entries []XRefEntry) []map[string]any { + out := make([]map[string]any, len(entries)) + for i, e := range entries { + out[i] = map[string]any{ + "type": e.Type, + "num": e.ObjNum, + "gen": e.Gen, + "offset": e.Offset, + "objStm": e.ObjStmNum, + "index": e.Index, + } + } + return out +} + +func surfaceToMaps(hits []SuspiciousHit) []map[string]any { + out := make([]map[string]any, len(hits)) + for i, h := range hits { + out[i] = map[string]any{ + "kind": h.Kind, + "num": h.ObjNum, + "gen": h.Gen, + "offset": h.Offset, + "detail": h.Detail, + } + } + return out +} + +var filtersRE = regexp.MustCompile(`/Filter\s*(?:\[([^\]]*)\]|/([A-Za-z0-9]+))`) + +func detectFilters(dict string) []string { + m := filtersRE.FindStringSubmatch(dict) + if m == nil { + return nil + } + if m[2] != "" { + return []string{m[2]} + } + parts := strings.Fields(m[1]) + out := make([]string, 0, len(parts)) + for _, p := range parts { + p = strings.TrimPrefix(p, "/") + if p != "" { + out = append(out, p) + } + } + return out +} + +var ( + typeRE = regexp.MustCompile(`/Type\s*/([A-Za-z0-9]+)`) + subtypeRE = regexp.MustCompile(`/Subtype\s*/([A-Za-z0-9]+)`) +) + +func detectType(dict string) string { + m := typeRE.FindStringSubmatch(dict) + if len(m) > 1 { + return m[1] + } + return "" +} + +func detectSubtype(dict string) string { + m := subtypeRE.FindStringSubmatch(dict) + if len(m) > 1 { + return m[1] + } + return "" +} + +// DecodeStream reads the stream for object objNum and runs it through +// the declared filter chain (with predictor). Works for both direct +// objects and those recovered from ObjStm containers. +func DecodeStream(d *document.Document, objNum int64) (map[string]any, error) { + summary, err := Parse(d) + if err != nil { + return nil, err + } + for _, o := range summary["objects"].([]map[string]any) { + n, _ := o["num"].(int64) + if n != objNum { + continue + } + hasStream, _ := o["hasStream"].(bool) + if !hasStream { + return nil, fmt.Errorf("object %d has no stream", objNum) + } + off, _ := o["streamOff"].(int64) + ln, _ := o["streamLen"].(int64) + raw, err := d.Range(off, ln) + if err != nil { + return nil, err + } + dict, _ := o["dict"].(string) + filters, _ := o["filters"].([]any) + predictor, columns, colors, bits := parseDecodeParms(dict) + + decoded := append([]byte(nil), raw...) + applied := []string{} + var runErr error + for _, f := range filters { + name, _ := f.(string) + decoded, runErr = ApplyFilter(name, decoded) + if runErr != nil { + return map[string]any{ + "objNum": objNum, + "raw": raw, + "decoded": nil, + "error": runErr.Error(), + "applied": applied, + "filters": filters, + }, nil + } + applied = append(applied, name) + } + if predictor > 1 { + decoded, runErr = ApplyPredictor(decoded, predictor, columns, colors, bits) + if runErr != nil { + return map[string]any{ + "objNum": objNum, + "raw": raw, + "decoded": nil, + "error": runErr.Error(), + "applied": applied, + "filters": filters, + }, nil + } + applied = append(applied, fmt.Sprintf("predictor=%d", predictor)) + } + return map[string]any{ + "objNum": objNum, + "raw": raw, + "decoded": decoded, + "applied": applied, + "filters": filters, + }, nil + } + return nil, fmt.Errorf("object %d not found", objNum) +} diff --git a/internal/leviathan/pdf/pdf_test.go b/internal/leviathan/pdf/pdf_test.go new file mode 100644 index 0000000..d10bb05 --- /dev/null +++ b/internal/leviathan/pdf/pdf_test.go @@ -0,0 +1,37 @@ +package pdf + +import ( + "os" + "strings" + "testing" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +func TestParseRejectsOversizedPDFBeforeWholeRange(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "huge-*.pdf") + if err != nil { + t.Fatalf("CreateTemp() failed: %v", err) + } + if _, err := file.WriteString("%PDF-1.7\n"); err != nil { + t.Fatalf("WriteString() failed: %v", err) + } + if err := file.Truncate(maxPDFParseBytes + 1); err != nil { + t.Fatalf("Truncate() failed: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("Close() failed: %v", err) + } + + doc, err := document.Open(file.Name()) + if err != nil { + t.Fatalf("document.Open() failed: %v", err) + } + _, err = Parse(doc) + if err == nil { + t.Fatal("expected oversized PDF to be rejected") + } + if !strings.Contains(err.Error(), "pdf parse skipped") { + t.Fatalf("unexpected error: %v", err) + } +} diff --git a/internal/leviathan/pdf/surface.go b/internal/leviathan/pdf/surface.go new file mode 100644 index 0000000..5dc0395 --- /dev/null +++ b/internal/leviathan/pdf/surface.go @@ -0,0 +1,128 @@ +package pdf + +import ( + "bytes" + "regexp" + "strings" +) + +// SuspiciousHit marks a dictionary fragment worth a closer look. It is +// surfaced to the UI with its containing object number so a user can +// jump straight to it. +type SuspiciousHit struct { + Kind string `json:"kind"` + ObjNum int64 `json:"objNum"` + Gen int64 `json:"gen"` + Offset int64 `json:"offset"` + Detail string `json:"detail,omitempty"` +} + +// Surface walks the enumerated object list and flags anything routinely +// abused by hostile PDFs: embedded files, JavaScript, launch actions, +// URI actions, form-submission targets, OpenAction / AA triggers. +func Surface(objects []Object) []SuspiciousHit { + hits := []SuspiciousHit{} + for _, o := range objects { + dict := o.Dict + push := func(kind, detail string) { + hits = append(hits, SuspiciousHit{ + Kind: kind, ObjNum: o.Num, Gen: o.Gen, + Offset: o.Offset, Detail: detail, + }) + } + // Key markers. Order matters only for readability of the UI. + if containsName(dict, "JavaScript") || containsName(dict, "JS") { + push("javascript", firstLine(dict, 240)) + } + if containsName(dict, "EmbeddedFile") || containsName(dict, "Filespec") { + push("embedded-file", extractName(dict, "/F")) + } + if containsName(dict, "Launch") { + push("launch-action", extractLaunchTarget(dict)) + } + if containsName(dict, "URI") { + if uri := extractURI(dict); uri != "" { + push("uri-action", uri) + } + } + if containsName(dict, "SubmitForm") { + push("submit-form", "") + } + if containsName(dict, "OpenAction") { + push("open-action", "") + } + if strings.Contains(dict, "/AA") { + push("additional-action", "") + } + if strings.Contains(dict, "/Type /XFA") || strings.Contains(dict, "/XFA ") { + push("xfa-form", "") + } + if strings.Contains(dict, "/RichMedia") { + push("rich-media", "") + } + if strings.Contains(dict, "/Type /Metadata") && strings.Contains(dict, "/Subtype /XML") { + push("xmp-metadata", "") + } + } + return hits +} + +func containsName(dict, name string) bool { + // Match `/Name` as a whole token — not inside /Names/ or /FooJavaScript. + pat := regexp.MustCompile(`/` + regexp.QuoteMeta(name) + `\b`) + return pat.MatchString(dict) +} + +// Go's regexp caps bounded repetitions at 1000 — do not raise this. +var uriRE = regexp.MustCompile(`/URI\s*\(([^)]{1,1000})\)`) + +func extractURI(dict string) string { + m := uriRE.FindStringSubmatch(dict) + if m == nil { + return "" + } + return m[1] +} + +var fnameRE = regexp.MustCompile(`/F\s*\(([^)]{1,512})\)`) + +func extractName(dict, key string) string { + _ = key + m := fnameRE.FindStringSubmatch(dict) + if m == nil { + return "" + } + return m[1] +} + +var launchRE = regexp.MustCompile(`/F\s*\(([^)]{1,512})\)`) + +func extractLaunchTarget(dict string) string { + m := launchRE.FindStringSubmatch(dict) + if m == nil { + return "" + } + return m[1] +} + +func firstLine(s string, max int) string { + nl := strings.IndexByte(s, '\n') + if nl > 0 { + s = s[:nl] + } + s = strings.TrimSpace(s) + if len(s) > max { + s = s[:max] + "…" + } + return s +} + +// HasLinearisation reports whether a PDF is linearised (Fast Web View). +// A linearised PDF has an obj dict near the start containing /Linearized. +func HasLinearisation(whole []byte) bool { + n := 4096 + if n > len(whole) { + n = len(whole) + } + return bytes.Contains(whole[:n], []byte("/Linearized")) +} diff --git a/internal/leviathan/pdf/xref.go b/internal/leviathan/pdf/xref.go new file mode 100644 index 0000000..55dad00 --- /dev/null +++ b/internal/leviathan/pdf/xref.go @@ -0,0 +1,422 @@ +package pdf + +import ( + "bytes" + "errors" + "fmt" + "regexp" + "strconv" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// XRefEntry describes one cross-reference entry after parsing either a +// classical xref table or a modern xref stream. +// +// Type 0 — free object +// Type 1 — in-use object; Offset is the byte offset in the file +// Type 2 — object inside an object stream; ObjStmNum identifies the +// container ObjStm and Index is the 0-based position within it +type XRefEntry struct { + Type int `json:"type"` + ObjNum int64 `json:"objNum"` + Gen int64 `json:"gen"` + Offset int64 `json:"offset"` + ObjStmNum int64 `json:"objStmNum,omitempty"` + Index int64 `json:"index,omitempty"` +} + +const maxObjectStreamEntries = 1 << 20 + +// ParseXRef attempts to reconstruct the cross-reference map. It handles +// both classical "xref\n0 N\n..." tables and xref streams (/Type /XRef). +// Returns a flat slice of entries covering every object it could resolve. +func ParseXRef(d *document.Document, whole []byte) ([]XRefEntry, string, error) { + mode := "" + entries := map[int64]XRefEntry{} + + // Find "startxref" near the end. + idx := bytes.LastIndex(whole, []byte("startxref")) + if idx < 0 { + return nil, "", errors.New("no startxref") + } + after := whole[idx+len("startxref"):] + after = bytes.TrimLeft(after, " \r\n\t") + var xrefPos int64 + fmt.Sscanf(string(after), "%d", &xrefPos) + if xrefPos <= 0 || xrefPos >= int64(len(whole)) { + return nil, "", fmt.Errorf("bad startxref offset %d", xrefPos) + } + + // Walk the xref chain via /Prev pointers. + visited := map[int64]bool{} + pos := xrefPos + for pos > 0 && !visited[pos] { + visited[pos] = true + // Classical table if the first non-whitespace token is "xref". + tail := whole[pos:] + trimmed := bytes.TrimLeft(tail, " \r\n\t") + if bytes.HasPrefix(trimmed, []byte("xref")) { + if mode == "" { + mode = "table" + } else if mode != "table" { + mode = "hybrid" + } + nextPrev, err := parseClassicalXRef(trimmed, entries) + if err != nil { + return mapToSlice(entries), mode, err + } + pos = nextPrev + continue + } + // Otherwise an xref stream: an indirect object at `pos`. + if mode == "" { + mode = "stream" + } else if mode != "stream" { + mode = "hybrid" + } + nextPrev, err := parseXRefStream(d, whole, pos, entries) + if err != nil { + return mapToSlice(entries), mode, err + } + pos = nextPrev + } + return mapToSlice(entries), mode, nil +} + +func mapToSlice(m map[int64]XRefEntry) []XRefEntry { + out := make([]XRefEntry, 0, len(m)) + for _, e := range m { + out = append(out, e) + } + return out +} + +var xrefRangeRE = regexp.MustCompile(`(?m)^\s*(\d+)\s+(\d+)\s*$`) +var xrefRowRE = regexp.MustCompile(`^(\d{10})\s+(\d{5})\s+([nf])`) +var prevRE = regexp.MustCompile(`/Prev\s+(\d+)`) + +func parseClassicalXRef(block []byte, entries map[int64]XRefEntry) (int64, error) { + // Skip "xref" line. + nl := bytes.IndexByte(block, '\n') + if nl < 0 { + return 0, errors.New("xref: no newline after keyword") + } + cursor := block[nl+1:] + for { + m := xrefRangeRE.FindSubmatchIndex(cursor) + if m == nil { + break + } + if m[0] > 0 { + // Subsection must begin at the start of the cursor. + break + } + start, _ := strconv.ParseInt(string(cursor[m[2]:m[3]]), 10, 64) + count, _ := strconv.ParseInt(string(cursor[m[4]:m[5]]), 10, 64) + // Advance past the subsection line. + nl2 := bytes.IndexByte(cursor, '\n') + if nl2 < 0 { + break + } + cursor = cursor[nl2+1:] + for i := int64(0); i < count; i++ { + if len(cursor) < 20 { + return 0, errors.New("xref: truncated subsection") + } + row := cursor[:20] + rm := xrefRowRE.FindSubmatch(row) + if rm != nil { + off, _ := strconv.ParseInt(string(rm[1]), 10, 64) + gen, _ := strconv.ParseInt(string(rm[2]), 10, 64) + typ := 1 + if rm[3][0] == 'f' { + typ = 0 + } + num := start + i + if _, seen := entries[num]; !seen { + entries[num] = XRefEntry{ + Type: typ, ObjNum: num, Gen: gen, Offset: off, + } + } + } + cursor = cursor[20:] + } + } + // Locate trailer and /Prev. + tIdx := bytes.Index(block, []byte("trailer")) + if tIdx < 0 { + return 0, nil + } + trailer := block[tIdx:] + if e := bytes.Index(trailer, []byte("startxref")); e > 0 { + trailer = trailer[:e] + } + if pm := prevRE.FindSubmatch(trailer); pm != nil { + prev, _ := strconv.ParseInt(string(pm[1]), 10, 64) + return prev, nil + } + return 0, nil +} + +func parseXRefStream(d *document.Document, whole []byte, pos int64, entries map[int64]XRefEntry) (int64, error) { + // Locate "obj" after pos, then "stream"/"endstream". + if pos < 0 || int(pos) >= len(whole) { + return 0, errors.New("xref stream: bad offset") + } + body := whole[pos:] + objIdx := bytes.Index(body, []byte("obj")) + if objIdx < 0 { + return 0, errors.New("xref stream: no obj") + } + dictEnd := bytes.Index(body[objIdx:], []byte("stream")) + if dictEnd < 0 { + return 0, errors.New("xref stream: no stream") + } + dict := string(body[objIdx : objIdx+dictEnd]) + streamStart := objIdx + dictEnd + len("stream") + if streamStart < len(body) && body[streamStart] == '\r' { + streamStart++ + } + if streamStart < len(body) && body[streamStart] == '\n' { + streamStart++ + } + endIdx := bytes.Index(body[streamStart:], []byte("endstream")) + if endIdx < 0 { + return 0, errors.New("xref stream: no endstream") + } + raw := body[streamStart : streamStart+endIdx] + // trim trailing newline. + if len(raw) > 0 && raw[len(raw)-1] == '\n' { + raw = raw[:len(raw)-1] + } + if len(raw) > 0 && raw[len(raw)-1] == '\r' { + raw = raw[:len(raw)-1] + } + + // Decode the xref-stream dictionary fields. + filters := parseDictNameList(dict, "/Filter") + w := parseDictIntArray(dict, "/W") + index := parseDictIntArray(dict, "/Index") + sizeN := parseDictInt(dict, "/Size") + predictor, columns, colors, bits := parseDecodeParms(dict) + + // Apply filters. + data := append([]byte(nil), raw...) + var err error + for _, f := range filters { + data, err = ApplyFilter(f, data) + if err != nil { + return 0, fmt.Errorf("xref stream: filter %s: %w", f, err) + } + } + if predictor > 1 { + data, err = ApplyPredictor(data, predictor, columns, colors, bits) + if err != nil { + return 0, err + } + } + if len(w) != 3 { + return 0, fmt.Errorf("xref stream: bad /W array %v", w) + } + entryLen := 0 + for _, width := range w { + if width < 0 || width > 8 { + return 0, fmt.Errorf("xref stream: invalid field width %d", width) + } + entryLen += width + } + if entryLen <= 0 { + return 0, errors.New("xref stream: zero-length entries") + } + if len(index) == 0 { + if sizeN < 0 { + return 0, errors.New("xref stream: negative /Size") + } + index = []int{0, sizeN} + } + if len(index)%2 != 0 { + return 0, errors.New("xref stream: /Index must contain start/count pairs") + } + // Walk subsections. + pi := 0 + for s := 0; s+1 < len(index); s += 2 { + first := int64(index[s]) + count := int64(index[s+1]) + if first < 0 || count < 0 { + return 0, errors.New("xref stream: negative /Index value") + } + for i := int64(0); i < count; i++ { + if pi+entryLen > len(data) { + break + } + row := data[pi : pi+entryLen] + pi += entryLen + t := readBE(row[:w[0]]) + f2 := readBE(row[w[0] : w[0]+w[1]]) + f3 := readBE(row[w[0]+w[1] : w[0]+w[1]+w[2]]) + if w[0] == 0 { + t = 1 // default Type 1 per spec + } + num := first + i + if _, seen := entries[num]; seen { + continue + } + switch t { + case 0: + entries[num] = XRefEntry{Type: 0, ObjNum: num, Gen: f3} + case 1: + entries[num] = XRefEntry{Type: 1, ObjNum: num, Gen: f3, Offset: f2} + case 2: + entries[num] = XRefEntry{ + Type: 2, ObjNum: num, ObjStmNum: f2, Index: f3, + } + } + } + } + if prev := parseDictInt(dict, "/Prev"); prev > 0 { + return int64(prev), nil + } + return 0, nil +} + +func readBE(b []byte) int64 { + v := int64(0) + for _, c := range b { + v = (v << 8) | int64(c) + } + return v +} + +// --------------------------------------------------------------------------- +// ObjStm — compressed object streams +// --------------------------------------------------------------------------- + +// DecodeObjStm decodes an /ObjStm stream and returns a map of +// object-number → raw object body (text, never with "stream"). +func DecodeObjStm(raw []byte, dict string) (map[int64]string, error) { + filters := parseDictNameList(dict, "/Filter") + predictor, columns, colors, bits := parseDecodeParms(dict) + data := append([]byte(nil), raw...) + var err error + for _, f := range filters { + data, err = ApplyFilter(f, data) + if err != nil { + return nil, err + } + } + if predictor > 1 { + data, err = ApplyPredictor(data, predictor, columns, colors, bits) + if err != nil { + return nil, err + } + } + n := parseDictInt(dict, "/N") + first := parseDictInt(dict, "/First") + if n <= 0 || first <= 0 || first >= len(data) { + return nil, fmt.Errorf("ObjStm: bad /N=%d /First=%d", n, first) + } + header := strings.Fields(string(data[:first])) + if n > maxObjectStreamEntries || n > len(header)/2 { + return nil, fmt.Errorf("ObjStm: header has %d tokens, need %d", len(header), n*2) + } + out := map[int64]string{} + offsets := make([]int, n) + nums := make([]int64, n) + for i := 0; i < n; i++ { + num, _ := strconv.ParseInt(header[i*2], 10, 64) + off, _ := strconv.Atoi(header[i*2+1]) + nums[i] = num + offsets[i] = off + } + for i := 0; i < n; i++ { + start := first + offsets[i] + end := len(data) + if i+1 < n { + end = first + offsets[i+1] + } + if start < 0 || end > len(data) || start > end { + continue + } + out[nums[i]] = strings.TrimSpace(string(data[start:end])) + } + return out, nil +} + +// --------------------------------------------------------------------------- +// Small dictionary helpers +// --------------------------------------------------------------------------- + +var decodeParmRE = regexp.MustCompile(`/DecodeParms\s*<<([^>]*)>>`) + +func parseDictInt(dict, key string) int { + pat := regexp.MustCompile(regexp.QuoteMeta(key) + `\s+(-?\d+)\b`) + m := pat.FindStringSubmatch(dict) + if m == nil { + return 0 + } + n, _ := strconv.Atoi(m[1]) + return n +} + +func parseDictIntArray(dict, key string) []int { + pat := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*\[([^\]]*)\]`) + m := pat.FindStringSubmatch(dict) + if m == nil { + return nil + } + parts := strings.Fields(m[1]) + out := make([]int, 0, len(parts)) + for _, p := range parts { + n, err := strconv.Atoi(p) + if err != nil { + continue + } + out = append(out, n) + } + return out +} + +func parseDictNameList(dict, key string) []string { + // Handles both `/Filter /FlateDecode` and `/Filter [/A85 /Flate]`. + pat1 := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*/([A-Za-z0-9]+)`) + if m := pat1.FindStringSubmatch(dict); m != nil { + return []string{m[1]} + } + pat2 := regexp.MustCompile(regexp.QuoteMeta(key) + `\s*\[([^\]]*)\]`) + if m := pat2.FindStringSubmatch(dict); m != nil { + parts := strings.Fields(m[1]) + out := make([]string, 0, len(parts)) + for _, p := range parts { + out = append(out, strings.TrimPrefix(p, "/")) + } + return out + } + return nil +} + +func parseDecodeParms(dict string) (predictor, columns, colors, bits int) { + m := decodeParmRE.FindStringSubmatch(dict) + if m == nil { + return 1, 1, 1, 8 + } + inner := m[1] + predictor = parseDictInt(inner, "/Predictor") + if predictor == 0 { + predictor = 1 + } + columns = parseDictInt(inner, "/Columns") + if columns == 0 { + columns = 1 + } + colors = parseDictInt(inner, "/Colors") + if colors == 0 { + colors = 1 + } + bits = parseDictInt(inner, "/BitsPerComponent") + if bits == 0 { + bits = 8 + } + return +} diff --git a/internal/leviathan/spectrogram/spectrogram.go b/internal/leviathan/spectrogram/spectrogram.go new file mode 100644 index 0000000..de37675 --- /dev/null +++ b/internal/leviathan/spectrogram/spectrogram.go @@ -0,0 +1,798 @@ +package spectrogram + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "image" + "image/color" + "image/png" + "io" + "math" + "path/filepath" + "strings" + + _ "image/gif" + _ "image/jpeg" + _ "image/png" + + "github.com/hajimehoshi/go-mp3" +) + +const ( + defaultSampleRate = 44100 + defaultDuration = 5.0 + // MaxImageBytes caps compressed image inputs before decode. Pixel count is + // checked separately because small compressed images can still expand huge. + MaxImageBytes = 64 << 20 + // MaxAudioBytes caps encoded WAV/MP3 inputs before full in-memory decode. + MaxAudioBytes = 128 << 20 + maxImagePixels = 24_000_000 + maxDecodedBytes = 64 << 20 + twoPi = 2 * math.Pi +) + +// EncodeOptions controls image-to-audio spectrogram synthesis. +type EncodeOptions struct { + DurationSec float64 + SampleRate int + MinHz float64 + MaxHz float64 + Bins int + Columns int + Invert bool + PreserveAR bool + Gain float64 +} + +// InsertOptions controls embedding synthesized spectrogram audio into a carrier. +type InsertOptions struct { + EncodeOptions + StartSec float64 + MixLevel float64 +} + +// ViewOptions controls spectrogram image rendering from audio. +type ViewOptions struct { + FFTSize int + Height int + MaxWidth int + Aspect float64 + MinHz float64 + MaxHz float64 + Brightness float64 + Contrast float64 +} + +type audioBuffer struct { + SampleRate int + Channels int + Samples []float64 +} + +// ImageToWAV converts an image into a WAV whose spectrogram reveals that image. +func ImageToWAV(imageData []byte, opts EncodeOptions) ([]byte, map[string]any, error) { + grid, meta, opts, err := imageGrid(imageData, opts) + if err != nil { + return nil, nil, err + } + mono := synthesize(grid, opts) + audio := audioBuffer{SampleRate: opts.SampleRate, Channels: 1, Samples: mono} + wav, err := encodeWAV(audio) + if err != nil { + return nil, nil, err + } + summary := map[string]any{ + "mode": "image-to-wav", + "imageFormat": meta["format"], + "imageWidth": meta["width"], + "imageHeight": meta["height"], + "imageAspect": meta["aspect"], + "durationSec": opts.DurationSec, + "sampleRate": opts.SampleRate, + "minHz": opts.MinHz, + "maxHz": opts.MaxHz, + "bins": opts.Bins, + "columns": opts.Columns, + "preserveAR": opts.PreserveAR, + "wavBytes": len(wav), + } + return wav, summary, nil +} + +// InsertImageIntoAudio mixes a synthesized image spectrogram into WAV or MP3 carrier audio. +func InsertImageIntoAudio(imageData, audioData []byte, audioName string, opts InsertOptions) ([]byte, map[string]any, error) { + carrier, err := DecodeAudio(audioData, audioName) + if err != nil { + return nil, nil, err + } + opts.EncodeOptions.SampleRate = carrier.SampleRate + grid, meta, encodeOpts, err := imageGrid(imageData, opts.EncodeOptions) + if err != nil { + return nil, nil, err + } + tone := synthesize(grid, encodeOpts) + if opts.MixLevel <= 0 { + opts.MixLevel = 0.35 + } + if opts.MixLevel > 1 { + opts.MixLevel = 1 + } + if opts.StartSec < 0 { + opts.StartSec = 0 + } + startFrame := int(math.Round(opts.StartSec * float64(carrier.SampleRate))) + requiredFrames := startFrame + len(tone) + currentFrames := len(carrier.Samples) / carrier.Channels + if requiredFrames > currentFrames { + extended := make([]float64, requiredFrames*carrier.Channels) + copy(extended, carrier.Samples) + carrier.Samples = extended + } + for i, sample := range tone { + frame := startFrame + i + for ch := 0; ch < carrier.Channels; ch++ { + idx := frame*carrier.Channels + ch + carrier.Samples[idx] += sample * opts.MixLevel + } + } + limitPeak(carrier.Samples, 0.98) + wav, err := encodeWAV(carrier) + if err != nil { + return nil, nil, err + } + summary := map[string]any{ + "mode": "insert-into-audio", + "imageFormat": meta["format"], + "imageWidth": meta["width"], + "imageHeight": meta["height"], + "imageAspect": meta["aspect"], + "carrierSampleRate": carrier.SampleRate, + "carrierChannels": carrier.Channels, + "startSec": opts.StartSec, + "mixLevel": opts.MixLevel, + "durationSec": encodeOpts.DurationSec, + "minHz": encodeOpts.MinHz, + "maxHz": encodeOpts.MaxHz, + "bins": encodeOpts.Bins, + "columns": encodeOpts.Columns, + "preserveAR": encodeOpts.PreserveAR, + "wavBytes": len(wav), + } + return wav, summary, nil +} + +// AudioSpectrogramPNG renders WAV or MP3 audio as a PNG spectrogram. +func AudioSpectrogramPNG(audioData []byte, audioName string, opts ViewOptions) ([]byte, map[string]any, error) { + audio, err := DecodeAudio(audioData, audioName) + if err != nil { + return nil, nil, err + } + mono := toMono(audio) + if len(mono) == 0 { + return nil, nil, errors.New("audio contains no samples") + } + opts = normalizeViewOptions(opts, audio.SampleRate) + img := renderSpectrogram(mono, audio.SampleRate, opts) + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + return nil, nil, err + } + duration := float64(len(mono)) / float64(audio.SampleRate) + summary := map[string]any{ + "mode": "audio-to-spectrogram", + "sampleRate": audio.SampleRate, + "channels": audio.Channels, + "durationSec": duration, + "fftSize": opts.FFTSize, + "width": img.Bounds().Dx(), + "height": img.Bounds().Dy(), + "aspectRatio": opts.Aspect, + "dynamicRange": 80, + "minHz": opts.MinHz, + "maxHz": opts.MaxHz, + "pngBytes": buf.Len(), + } + return buf.Bytes(), summary, nil +} + +// DecodeAudio decodes WAV directly and MP3 through go-mp3. +func DecodeAudio(data []byte, name string) (audioBuffer, error) { + if len(data) == 0 { + return audioBuffer{}, errors.New("audio file is empty") + } + if len(data) > MaxAudioBytes { + return audioBuffer{}, fmt.Errorf("audio input is too large: %d bytes exceeds %d bytes", len(data), MaxAudioBytes) + } + if isWAV(data) { + return decodeWAV(data) + } + ext := strings.ToLower(filepath.Ext(name)) + if ext == ".mp3" || isMP3(data) { + return decodeMP3(data) + } + return audioBuffer{}, errors.New("unsupported audio format: load a WAV or MP3 file") +} + +func imageGrid(data []byte, opts EncodeOptions) ([][]float64, map[string]any, EncodeOptions, error) { + if len(data) == 0 { + return nil, nil, opts, errors.New("image file is empty") + } + if len(data) > MaxImageBytes { + return nil, nil, opts, fmt.Errorf("image input is too large: %d bytes exceeds %d bytes", len(data), MaxImageBytes) + } + cfg, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, nil, opts, fmt.Errorf("decode image config: %w", err) + } + if cfg.Width <= 0 || cfg.Height <= 0 { + return nil, nil, opts, errors.New("image has no pixels") + } + if int64(cfg.Width)*int64(cfg.Height) > maxImagePixels { + return nil, nil, opts, fmt.Errorf("image dimensions are too large: %dx%d exceeds %d pixels", cfg.Width, cfg.Height, maxImagePixels) + } + img, format, err := image.Decode(bytes.NewReader(data)) + if err != nil { + return nil, nil, opts, fmt.Errorf("decode image: %w", err) + } + bounds := img.Bounds() + if bounds.Dx() <= 0 || bounds.Dy() <= 0 { + return nil, nil, opts, errors.New("image has no pixels") + } + opts = normalizeEncodeOptions(opts, bounds.Dx(), bounds.Dy()) + grid := make([][]float64, opts.Bins) + for bin := 0; bin < opts.Bins; bin++ { + grid[bin] = make([]float64, opts.Columns) + sourceY := bounds.Min.Y + int((float64(opts.Bins-1-bin)+0.5)*float64(bounds.Dy())/float64(opts.Bins)) + if sourceY >= bounds.Max.Y { + sourceY = bounds.Max.Y - 1 + } + for col := 0; col < opts.Columns; col++ { + sourceX := bounds.Min.X + int((float64(col)+0.5)*float64(bounds.Dx())/float64(opts.Columns)) + if sourceX >= bounds.Max.X { + sourceX = bounds.Max.X - 1 + } + grid[bin][col] = pixelAmplitude(img.At(sourceX, sourceY), opts.Invert) + } + } + meta := map[string]any{"format": format, "width": bounds.Dx(), "height": bounds.Dy(), "aspect": float64(bounds.Dy()) / float64(bounds.Dx())} + return grid, meta, opts, nil +} + +func normalizeEncodeOptions(opts EncodeOptions, imageWidth, imageHeight int) EncodeOptions { + if opts.DurationSec <= 0 { + opts.DurationSec = defaultDuration + } + opts.DurationSec = clampFloat(opts.DurationSec, 0.25, 60) + if opts.SampleRate <= 0 { + opts.SampleRate = defaultSampleRate + } + opts.SampleRate = clampInt(opts.SampleRate, 8000, 96000) + if opts.MinHz <= 0 { + opts.MinHz = 200 + } + if opts.MaxHz <= opts.MinHz { + opts.MaxHz = math.Min(float64(opts.SampleRate)/2-200, 20000) + } + opts.MinHz = clampFloat(opts.MinHz, 20, float64(opts.SampleRate)/2-100) + opts.MaxHz = clampFloat(opts.MaxHz, opts.MinHz+100, float64(opts.SampleRate)/2-50) + if opts.PreserveAR && imageWidth > 0 && imageHeight > 0 { + aspect := float64(imageHeight) / float64(imageWidth) + baseRange := opts.MaxHz - opts.MinHz + desiredRange := baseRange * aspect + opts.MaxHz = clampFloat(opts.MinHz+desiredRange, opts.MinHz+100, float64(opts.SampleRate)/2-50) + } + if opts.Bins <= 0 { + opts.Bins = clampInt(imageHeight, 48, 192) + } + opts.Bins = clampInt(opts.Bins, 16, 384) + if opts.Columns <= 0 { + opts.Columns = clampInt(imageWidth, 64, 1024) + } + opts.Columns = clampInt(opts.Columns, 16, 2048) + if opts.Gain <= 0 { + opts.Gain = 0.85 + } + opts.Gain = clampFloat(opts.Gain, 0.05, 1) + return opts +} + +func normalizeViewOptions(opts ViewOptions, sampleRate int) ViewOptions { + if opts.FFTSize <= 0 { + opts.FFTSize = 2048 + } + opts.FFTSize = nearestPowerOfTwo(clampInt(opts.FFTSize, 256, 8192)) + if opts.Height <= 0 { + opts.Height = 512 + } + opts.Height = clampInt(opts.Height, 64, 1024) + if opts.MaxWidth <= 0 { + opts.MaxWidth = 1400 + } + opts.MaxWidth = clampInt(opts.MaxWidth, 64, 2400) + if opts.Aspect <= 0 { + opts.Aspect = 1 + } + opts.Aspect = clampFloat(opts.Aspect, 0.25, 4) + if opts.MinHz < 0 { + opts.MinHz = 0 + } + if opts.MaxHz <= opts.MinHz { + opts.MaxHz = float64(sampleRate) / 2 + } + opts.MinHz = clampFloat(opts.MinHz, 0, float64(sampleRate)/2-10) + opts.MaxHz = clampFloat(opts.MaxHz, opts.MinHz+10, float64(sampleRate)/2) + if opts.Brightness <= 0 { + opts.Brightness = 1 + } + opts.Brightness = clampFloat(opts.Brightness, 0.1, 4) + if opts.Contrast <= 0 { + opts.Contrast = 1 + } + opts.Contrast = clampFloat(opts.Contrast, 0.1, 4) + return opts +} + +func pixelAmplitude(c color.Color, invert bool) float64 { + r, g, b, a := c.RGBA() + alpha := float64(a) / 65535 + if alpha <= 0.001 { + return 0 + } + luma := (0.2126*float64(r) + 0.7152*float64(g) + 0.0722*float64(b)) / 65535 + amp := luma + if invert { + amp = 1 - luma + } + amp *= alpha + return math.Pow(clampFloat(amp, 0, 1), 1.15) +} + +func synthesize(grid [][]float64, opts EncodeOptions) []float64 { + bins := len(grid) + cols := len(grid[0]) + frames := int(math.Round(opts.DurationSec * float64(opts.SampleRate))) + if frames < 1 { + frames = 1 + } + out := make([]float64, frames) + phase := make([]float64, bins) + inc := make([]float64, bins) + for i := range inc { + var pos float64 + if bins > 1 { + pos = float64(i) / float64(bins-1) + } + freq := opts.MinHz + (opts.MaxHz-opts.MinHz)*pos + inc[i] = twoPi * freq / float64(opts.SampleRate) + } + for i := 0; i < frames; i++ { + var x float64 + if frames > 1 { + x = float64(i) * float64(cols-1) / float64(frames-1) + } + c0 := int(math.Floor(x)) + c1 := c0 + 1 + if c1 >= cols { + c1 = cols - 1 + } + t := x - float64(c0) + var sample, weight float64 + for bin := 0; bin < bins; bin++ { + amp := grid[bin][c0]*(1-t) + grid[bin][c1]*t + if amp < 0.002 { + continue + } + phase[bin] += inc[bin] + if phase[bin] > twoPi { + phase[bin] -= twoPi + } + sample += math.Sin(phase[bin]) * amp + weight += amp + } + if weight > 1 { + sample /= math.Sqrt(weight) + } + out[i] = sample + } + normalizePeak(out, opts.Gain) + fade(out, opts.SampleRate/100) + return out +} + +func renderSpectrogram(samples []float64, sampleRate int, opts ViewOptions) image.Image { + fftSize := opts.FFTSize + hop := fftSize / 4 + if hop < 1 { + hop = 1 + } + frameCount := 1 + if len(samples) > fftSize { + frameCount = 1 + (len(samples)-fftSize)/hop + } + width := frameCount + if width > opts.MaxWidth { + width = opts.MaxWidth + } + if width < 1 { + width = 1 + } + height := clampInt(int(math.Round(float64(opts.Height)*opts.Aspect)), 32, 4096) + img := image.NewRGBA(image.Rect(0, 0, width, height)) + values := make([]float32, width*height) + maxDB := math.Inf(-1) + window := hamming(fftSize) + for x := 0; x < width; x++ { + frameIndex := 0 + if width > 1 && frameCount > 1 { + frameIndex = int(math.Round(float64(x) * float64(frameCount-1) / float64(width-1))) + } + start := frameIndex * hop + buf := make([]complex128, fftSize) + for i := 0; i < fftSize; i++ { + if start+i < len(samples) { + buf[i] = complex(samples[start+i]*window[i], 0) + } + } + fft(buf) + for y := 0; y < height; y++ { + pos := 0.0 + if height > 1 { + pos = float64(height-1-y) / float64(height-1) + } + freq := opts.MinHz + (opts.MaxHz-opts.MinHz)*pos + bin := int(math.Round(freq * float64(fftSize) / float64(sampleRate))) + if bin < 0 { + bin = 0 + } + if bin >= fftSize/2 { + bin = fftSize/2 - 1 + } + mag := cmplxAbs(buf[bin]) / float64(fftSize) + db := 20 * math.Log10(mag+1e-10) + values[y*width+x] = float32(db) + if db > maxDB { + maxDB = db + } + } + } + if math.IsInf(maxDB, -1) { + maxDB = -80 + } + minDB := maxDB - 80 + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + db := float64(values[y*width+x]) + norm := clampFloat((db-minDB)/(maxDB-minDB), 0, 1) + norm = clampFloat((norm-0.5)*opts.Contrast+0.5, 0, 1) + norm = clampFloat(norm*opts.Brightness, 0, 1) + img.Set(x, y, palette(norm)) + } + } + return img +} + +func decodeWAV(data []byte) (audioBuffer, error) { + if !isWAV(data) { + return audioBuffer{}, errors.New("not a WAV file") + } + var fmtChunk, dataChunk []byte + pos := 12 + for pos+8 <= len(data) { + id := string(data[pos : pos+4]) + size := int(binary.LittleEndian.Uint32(data[pos+4 : pos+8])) + pos += 8 + if size < 0 || pos+size > len(data) { + return audioBuffer{}, errors.New("invalid WAV chunk size") + } + chunk := data[pos : pos+size] + switch id { + case "fmt ": + fmtChunk = chunk + case "data": + dataChunk = chunk + } + pos += size + if size%2 == 1 { + pos++ + } + } + if len(fmtChunk) < 16 || len(dataChunk) == 0 { + return audioBuffer{}, errors.New("WAV is missing fmt or data chunk") + } + audioFormat := binary.LittleEndian.Uint16(fmtChunk[0:2]) + channels := int(binary.LittleEndian.Uint16(fmtChunk[2:4])) + sampleRate := int(binary.LittleEndian.Uint32(fmtChunk[4:8])) + blockAlign := int(binary.LittleEndian.Uint16(fmtChunk[12:14])) + bits := int(binary.LittleEndian.Uint16(fmtChunk[14:16])) + if channels < 1 || channels > 8 || sampleRate < 1 || blockAlign < channels { + return audioBuffer{}, errors.New("unsupported WAV stream parameters") + } + if len(dataChunk) > maxDecodedBytes { + return audioBuffer{}, errors.New("audio data is too large to decode safely") + } + bytesPerSample := bits / 8 + if bytesPerSample < 1 || blockAlign < channels*bytesPerSample { + return audioBuffer{}, errors.New("unsupported WAV bit depth") + } + frames := len(dataChunk) / blockAlign + samples := make([]float64, frames*channels) + for frame := 0; frame < frames; frame++ { + base := frame * blockAlign + for ch := 0; ch < channels; ch++ { + off := base + ch*bytesPerSample + samples[frame*channels+ch] = decodeWAVSample(dataChunk[off:], audioFormat, bits) + } + } + return audioBuffer{SampleRate: sampleRate, Channels: channels, Samples: samples}, nil +} + +func decodeWAVSample(b []byte, audioFormat uint16, bits int) float64 { + switch audioFormat { + case 1: + switch bits { + case 8: + return (float64(b[0]) - 128) / 128 + case 16: + return float64(int16(binary.LittleEndian.Uint16(b[:2]))) / 32768 + case 24: + v := int32(b[0]) | int32(b[1])<<8 | int32(b[2])<<16 + if v&0x800000 != 0 { + v |= ^0xffffff + } + return float64(v) / 8388608 + case 32: + return float64(int32(binary.LittleEndian.Uint32(b[:4]))) / 2147483648 + } + case 3: + switch bits { + case 32: + return float64(math.Float32frombits(binary.LittleEndian.Uint32(b[:4]))) + case 64: + return math.Float64frombits(binary.LittleEndian.Uint64(b[:8])) + } + } + return 0 +} + +func decodeMP3(data []byte) (audioBuffer, error) { + decoder, err := mp3.NewDecoder(bytes.NewReader(data)) + if err != nil { + return audioBuffer{}, err + } + limited := io.LimitReader(decoder, maxDecodedBytes+1) + pcm, err := io.ReadAll(limited) + if err != nil { + return audioBuffer{}, err + } + if len(pcm) > maxDecodedBytes { + return audioBuffer{}, errors.New("decoded MP3 is too large to process safely") + } + if len(pcm) < 2 { + return audioBuffer{}, errors.New("MP3 decoded no samples") + } + sampleCount := len(pcm) / 2 + channels := 2 + if sampleCount%2 != 0 { + channels = 1 + } + samples := make([]float64, sampleCount) + for i := 0; i < sampleCount; i++ { + samples[i] = float64(int16(binary.LittleEndian.Uint16(pcm[i*2:i*2+2]))) / 32768 + } + return audioBuffer{SampleRate: decoder.SampleRate(), Channels: channels, Samples: samples}, nil +} + +func encodeWAV(audio audioBuffer) ([]byte, error) { + if audio.SampleRate <= 0 || audio.Channels <= 0 { + return nil, errors.New("invalid audio parameters") + } + var buf bytes.Buffer + dataSize := int64(len(audio.Samples)) * 2 + if dataSize > int64(^uint32(0))-44 { + return nil, errors.New("audio is too large for a WAV file") + } + buf.WriteString("RIFF") + _ = binary.Write(&buf, binary.LittleEndian, uint32(36+dataSize)) + buf.WriteString("WAVE") + buf.WriteString("fmt ") + _ = binary.Write(&buf, binary.LittleEndian, uint32(16)) + _ = binary.Write(&buf, binary.LittleEndian, uint16(1)) + _ = binary.Write(&buf, binary.LittleEndian, uint16(audio.Channels)) + _ = binary.Write(&buf, binary.LittleEndian, uint32(audio.SampleRate)) + byteRate := audio.SampleRate * audio.Channels * 2 + _ = binary.Write(&buf, binary.LittleEndian, uint32(byteRate)) + _ = binary.Write(&buf, binary.LittleEndian, uint16(audio.Channels*2)) + _ = binary.Write(&buf, binary.LittleEndian, uint16(16)) + buf.WriteString("data") + _ = binary.Write(&buf, binary.LittleEndian, uint32(dataSize)) + for _, sample := range audio.Samples { + sample = clampFloat(sample, -1, 1) + _ = binary.Write(&buf, binary.LittleEndian, int16(math.Round(sample*32767))) + } + return buf.Bytes(), nil +} + +func toMono(audio audioBuffer) []float64 { + if audio.Channels == 1 { + out := make([]float64, len(audio.Samples)) + copy(out, audio.Samples) + return out + } + frames := len(audio.Samples) / audio.Channels + out := make([]float64, frames) + for frame := 0; frame < frames; frame++ { + var sum float64 + for ch := 0; ch < audio.Channels; ch++ { + sum += audio.Samples[frame*audio.Channels+ch] + } + out[frame] = sum / float64(audio.Channels) + } + return out +} + +func isWAV(data []byte) bool { + return len(data) >= 12 && string(data[:4]) == "RIFF" && string(data[8:12]) == "WAVE" +} + +func isMP3(data []byte) bool { + return len(data) >= 3 && (string(data[:3]) == "ID3" || (data[0] == 0xff && data[1]&0xe0 == 0xe0)) +} + +func normalizePeak(samples []float64, target float64) { + var peak float64 + for _, s := range samples { + if a := math.Abs(s); a > peak { + peak = a + } + } + if peak <= 0 { + return + } + scale := target / peak + for i := range samples { + samples[i] *= scale + } +} + +func limitPeak(samples []float64, target float64) { + var peak float64 + for _, s := range samples { + if a := math.Abs(s); a > peak { + peak = a + } + } + if peak <= target || peak <= 0 { + return + } + scale := target / peak + for i := range samples { + samples[i] *= scale + } +} + +func fade(samples []float64, fadeLen int) { + if fadeLen <= 0 || len(samples) == 0 { + return + } + if fadeLen*2 > len(samples) { + fadeLen = len(samples) / 2 + } + for i := 0; i < fadeLen; i++ { + gain := float64(i) / float64(fadeLen) + samples[i] *= gain + samples[len(samples)-1-i] *= gain + } +} + +func hamming(n int) []float64 { + w := make([]float64, n) + if n <= 1 { + if n == 1 { + w[0] = 1 + } + return w + } + for i := range w { + w[i] = 0.54 - 0.46*math.Cos(twoPi*float64(i)/float64(n-1)) + } + return w +} + +func fft(a []complex128) { + n := len(a) + j := 0 + for i := 1; i < n; i++ { + bit := n >> 1 + for ; j&bit != 0; bit >>= 1 { + j &^= bit + } + j |= bit + if i < j { + a[i], a[j] = a[j], a[i] + } + } + for length := 2; length <= n; length <<= 1 { + angle := -twoPi / float64(length) + wlen := complex(math.Cos(angle), math.Sin(angle)) + for i := 0; i < n; i += length { + w := complex(1, 0) + for k := 0; k < length/2; k++ { + u := a[i+k] + v := a[i+k+length/2] * w + a[i+k] = u + v + a[i+k+length/2] = u - v + w *= wlen + } + } + } +} + +func cmplxAbs(c complex128) float64 { + return math.Hypot(real(c), imag(c)) +} + +func palette(v float64) color.RGBA { + v = clampFloat(v, 0, 1) + colors := []color.RGBA{ + {R: 68, G: 1, B: 84, A: 255}, + {R: 72, G: 40, B: 120, A: 255}, + {R: 62, G: 73, B: 137, A: 255}, + {R: 49, G: 104, B: 142, A: 255}, + {R: 38, G: 130, B: 142, A: 255}, + {R: 31, G: 158, B: 137, A: 255}, + {R: 53, G: 183, B: 121, A: 255}, + {R: 110, G: 206, B: 88, A: 255}, + {R: 181, G: 222, B: 43, A: 255}, + {R: 253, G: 231, B: 37, A: 255}, + } + scaled := v * float64(len(colors)-1) + idx := int(math.Floor(scaled)) + if idx >= len(colors)-1 { + return colors[len(colors)-1] + } + t := scaled - float64(idx) + a, b := colors[idx], colors[idx+1] + return color.RGBA{ + R: uint8(float64(a.R) + (float64(b.R)-float64(a.R))*t), + G: uint8(float64(a.G) + (float64(b.G)-float64(a.G))*t), + B: uint8(float64(a.B) + (float64(b.B)-float64(a.B))*t), + A: 255, + } +} + +func nearestPowerOfTwo(n int) int { + if n <= 1 { + return 1 + } + p := 1 + for p < n { + p <<= 1 + } + prev := p >> 1 + if p-n < n-prev { + return p + } + return prev +} + +func clampInt(v, min, max int) int { + if v < min { + return min + } + if v > max { + return max + } + return v +} + +func clampFloat(v, min, max float64) float64 { + if v < min { + return min + } + if v > max { + return max + } + return v +} diff --git a/internal/leviathan/spectrogram/spectrogram_test.go b/internal/leviathan/spectrogram/spectrogram_test.go new file mode 100644 index 0000000..0507c64 --- /dev/null +++ b/internal/leviathan/spectrogram/spectrogram_test.go @@ -0,0 +1,177 @@ +package spectrogram + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/png" + "testing" +) + +func TestImageToWAVAndRenderSpectrogram(t *testing.T) { + img := testPNG(t) + wav, summary, err := ImageToWAV(img, EncodeOptions{ + DurationSec: 0.25, + SampleRate: 8000, + MinHz: 300, + MaxHz: 3000, + Bins: 32, + Columns: 32, + Invert: true, + }) + if err != nil { + t.Fatalf("ImageToWAV failed: %v", err) + } + if !isWAV(wav) { + t.Fatalf("expected WAV header") + } + if summary["wavBytes"].(int) != len(wav) { + t.Fatalf("summary wavBytes mismatch") + } + + rendered, viewSummary, err := AudioSpectrogramPNG(wav, "test.wav", ViewOptions{ + FFTSize: 512, + Height: 96, + Aspect: 2, + MaxWidth: 128, + }) + if err != nil { + t.Fatalf("AudioSpectrogramPNG failed: %v", err) + } + if _, err := png.Decode(bytes.NewReader(rendered)); err != nil { + t.Fatalf("rendered spectrogram is not a PNG: %v", err) + } + if viewSummary["sampleRate"].(int) != 8000 { + t.Fatalf("unexpected sample rate: %v", viewSummary["sampleRate"]) + } + if viewSummary["height"].(int) != 192 { + t.Fatalf("expected aspect-adjusted height 192, got %v", viewSummary["height"]) + } +} + +func TestPreserveAspectRatioAdjustsFrequencyRange(t *testing.T) { + img := testPNGSize(t, 32, 16) + _, summary, err := ImageToWAV(img, EncodeOptions{ + DurationSec: 0.1, + SampleRate: 8000, + MinHz: 200, + MaxHz: 3000, + Bins: 16, + Columns: 16, + PreserveAR: true, + Invert: true, + }) + if err != nil { + t.Fatalf("ImageToWAV failed: %v", err) + } + if summary["preserveAR"] != true { + t.Fatalf("expected preserveAR summary") + } + if got := summary["maxHz"].(float64); got != 1600 { + t.Fatalf("expected landscape image to halve frequency range to 1600Hz, got %v", got) + } +} + +func TestImageToWAVRejectsOversizedImageDimensionsBeforeDecode(t *testing.T) { + _, _, err := ImageToWAV(hugePNGHeader(t, 100000, 100000), EncodeOptions{}) + if err == nil { + t.Fatal("expected oversized image dimensions to be rejected") + } + if !bytes.Contains([]byte(err.Error()), []byte("image dimensions are too large")) { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestInsertImageIntoAudioExtendsCarrier(t *testing.T) { + img := testPNG(t) + carrier, _, err := ImageToWAV(img, EncodeOptions{ + DurationSec: 0.1, + SampleRate: 8000, + MinHz: 300, + MaxHz: 2500, + Bins: 16, + Columns: 16, + Invert: true, + }) + if err != nil { + t.Fatalf("carrier generation failed: %v", err) + } + out, summary, err := InsertImageIntoAudio(img, carrier, "carrier.wav", InsertOptions{ + EncodeOptions: EncodeOptions{ + DurationSec: 0.2, + MinHz: 300, + MaxHz: 2500, + Bins: 16, + Columns: 16, + Invert: true, + }, + StartSec: 0.05, + MixLevel: 0.25, + }) + if err != nil { + t.Fatalf("InsertImageIntoAudio failed: %v", err) + } + if !isWAV(out) { + t.Fatalf("expected WAV output") + } + if summary["mode"] != "insert-into-audio" { + t.Fatalf("unexpected mode: %v", summary["mode"]) + } + if len(out) <= len(carrier) { + t.Fatalf("expected carrier to be extended") + } +} + +func testPNG(t *testing.T) []byte { + t.Helper() + return testPNGSize(t, 32, 32) +} + +func testPNGSize(t *testing.T, width, height int) []byte { + t.Helper() + img := image.NewRGBA(image.Rect(0, 0, width, height)) + for y := 0; y < height; y++ { + for x := 0; x < width; x++ { + img.Set(x, y, color.RGBA{R: 255, G: 255, B: 255, A: 255}) + } + } + limit := width + if height < limit { + limit = height + } + for i := 0; i < limit; i++ { + img.Set(i, i, color.RGBA{A: 255}) + img.Set(width-1-i, i, color.RGBA{A: 255}) + } + var buf bytes.Buffer + if err := png.Encode(&buf, img); err != nil { + t.Fatalf("encode png: %v", err) + } + return buf.Bytes() +} + +func hugePNGHeader(t *testing.T, width, height uint32) []byte { + t.Helper() + var buf bytes.Buffer + buf.Write([]byte{0x89, 'P', 'N', 'G', '\r', '\n', 0x1a, '\n'}) + ihdr := make([]byte, 13) + binary.BigEndian.PutUint32(ihdr[0:4], width) + binary.BigEndian.PutUint32(ihdr[4:8], height) + ihdr[8] = 8 + ihdr[9] = 2 + writePNGChunk(&buf, "IHDR", ihdr) + writePNGChunk(&buf, "IEND", nil) + return buf.Bytes() +} + +func writePNGChunk(buf *bytes.Buffer, kind string, data []byte) { + _ = binary.Write(buf, binary.BigEndian, uint32(len(data))) + buf.WriteString(kind) + buf.Write(data) + crc := crc32.NewIEEE() + crc.Write([]byte(kind)) + crc.Write(data) + _ = binary.Write(buf, binary.BigEndian, crc.Sum32()) +} diff --git a/internal/leviathan/squashfs/squashfs.go b/internal/leviathan/squashfs/squashfs.go new file mode 100644 index 0000000..1b3af6b --- /dev/null +++ b/internal/leviathan/squashfs/squashfs.go @@ -0,0 +1,782 @@ +// Package squashfs implements a read-only SquashFS 4.0 extractor. +// +// Scope is deliberately pragmatic: gzip, xz and zstd-compressed images, +// basic and extended file and directory inodes, symlink inodes. Other +// compressors (lzma/lzo/lz4) return a structured error so the UI can show +// what's missing. +// +// The module never trusts its input: every table offset and length is +// bounded against the image size and a hard cap on uncompressed metadata +// block size. A malformed or truncated image returns an error rather +// than causing a wild read. +package squashfs + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "errors" + "fmt" + "io" + "path" + "sort" + + "github.com/andydixon/nexus/internal/leviathan/document" + "github.com/klauspost/compress/zstd" + "github.com/ulikunitz/xz" +) + +const ( + magicLE uint32 = 0x73717368 // "hsqs" + metadataBlockSize = 8192 + dataBlockMax = 1 << 20 // 1 MiB guard for a single data block + maxEntries = 1 << 20 // total files + dirs we're willing to walk +) + +// Compression IDs from the spec. +const ( + compGZIP = 1 + compLZMA = 2 + compLZO = 3 + compXZ = 4 + compLZ4 = 5 + compZSTD = 6 +) + +// Inode type codes. +const ( + itBasicDir = 1 + itBasicFile = 2 + itBasicSymlink = 3 + itBasicBlockDev = 4 + itBasicCharDev = 5 + itBasicFifo = 6 + itBasicSocket = 7 + itExtDir = 8 + itExtFile = 9 + itExtSymlink = 10 + itExtBlockDev = 11 + itExtCharDev = 12 + itExtFifo = 13 + itExtSocket = 14 +) + +// Superblock is the parsed 96-byte header. +type Superblock struct { + Magic uint32 + InodeCount uint32 + ModTime uint32 + BlockSize uint32 + FragCount uint32 + CompressionID uint16 + BlockLog uint16 + Flags uint16 + IDCount uint16 + VersionMajor uint16 + VersionMinor uint16 + RootInodeRef uint64 + BytesUsed uint64 + IDTableStart uint64 + XattrTableStart uint64 + InodeTable uint64 + DirTable uint64 + FragTable uint64 + ExportTable uint64 +} + +// Entry describes a filesystem entry discovered by the walker. +type Entry struct { + Path string + Type string // "file", "dir", "symlink", "special" + Size int64 + Mode uint16 + UID uint16 // id_idx (not resolved) + GID uint16 + Mtime uint32 + InodeNumber uint32 + InodeRef uint64 // 64-bit metadata reference (for targeted extraction) + LinkTarget string // for symlinks + // For file extraction; zero for non-files. + blocksStart uint64 + fileSize int64 + fragIndex uint32 + fragOffset uint32 + blockSizes []uint32 // raw (top bit=uncompressed) +} + +// FS is the opened image handle. +type FS struct { + d *document.Document + sb Superblock +} + +// Open validates the superblock and returns an FS for further queries. +func Open(d *document.Document) (*FS, error) { + head, err := d.Range(0, 96) + if err != nil { + return nil, fmt.Errorf("read superblock: %w", err) + } + sb, err := parseSuperblock(head) + if err != nil { + return nil, err + } + if sb.VersionMajor != 4 { + return nil, fmt.Errorf("unsupported squashfs version %d.%d (only 4.x)", sb.VersionMajor, sb.VersionMinor) + } + if sb.BytesUsed > uint64(d.Size()) { + return nil, fmt.Errorf("superblock bytes_used=%d exceeds document size=%d", sb.BytesUsed, d.Size()) + } + return &FS{d: d, sb: sb}, nil +} + +// Superblock returns the parsed header (copy). +func (f *FS) Superblock() Superblock { return f.sb } + +// CompressionName returns the human name of the image compressor. +func (f *FS) CompressionName() string { + return compressionName(f.sb.CompressionID) +} + +func compressionName(id uint16) string { + switch id { + case compGZIP: + return "gzip" + case compLZMA: + return "lzma" + case compLZO: + return "lzo" + case compXZ: + return "xz" + case compLZ4: + return "lz4" + case compZSTD: + return "zstd" + } + return fmt.Sprintf("unknown(%d)", id) +} + +func parseSuperblock(b []byte) (Superblock, error) { + var s Superblock + if len(b) < 96 { + return s, errors.New("superblock: short read") + } + s.Magic = binary.LittleEndian.Uint32(b[0:4]) + if s.Magic != magicLE { + return s, fmt.Errorf("bad magic 0x%08x (want 0x%08x)", s.Magic, magicLE) + } + s.InodeCount = binary.LittleEndian.Uint32(b[4:8]) + s.ModTime = binary.LittleEndian.Uint32(b[8:12]) + s.BlockSize = binary.LittleEndian.Uint32(b[12:16]) + s.FragCount = binary.LittleEndian.Uint32(b[16:20]) + s.CompressionID = binary.LittleEndian.Uint16(b[20:22]) + s.BlockLog = binary.LittleEndian.Uint16(b[22:24]) + s.Flags = binary.LittleEndian.Uint16(b[24:26]) + s.IDCount = binary.LittleEndian.Uint16(b[26:28]) + s.VersionMajor = binary.LittleEndian.Uint16(b[28:30]) + s.VersionMinor = binary.LittleEndian.Uint16(b[30:32]) + s.RootInodeRef = binary.LittleEndian.Uint64(b[32:40]) + s.BytesUsed = binary.LittleEndian.Uint64(b[40:48]) + s.IDTableStart = binary.LittleEndian.Uint64(b[48:56]) + s.XattrTableStart = binary.LittleEndian.Uint64(b[56:64]) + s.InodeTable = binary.LittleEndian.Uint64(b[64:72]) + s.DirTable = binary.LittleEndian.Uint64(b[72:80]) + s.FragTable = binary.LittleEndian.Uint64(b[80:88]) + s.ExportTable = binary.LittleEndian.Uint64(b[88:96]) + return s, nil +} + +// ---------- metadata block reader ---------- + +func (f *FS) decompress(buf []byte) ([]byte, error) { + return f.decompressWithLimit(buf, metadataBlockSize+16) +} + +func (f *FS) decompressWithLimit(buf []byte, limit int64) ([]byte, error) { + var r io.ReadCloser + switch f.sb.CompressionID { + case compGZIP: + zr, err := zlib.NewReader(bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("zlib init: %w", err) + } + r = zr + case compXZ: + xr, err := xz.NewReader(bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("xz init: %w", err) + } + r = io.NopCloser(xr) + case compZSTD: + zr, err := zstd.NewReader(bytes.NewReader(buf)) + if err != nil { + return nil, fmt.Errorf("zstd init: %w", err) + } + r = zr.IOReadCloser() + default: + return nil, fmt.Errorf("compressor %q not implemented in this build", compressionName(f.sb.CompressionID)) + } + defer r.Close() + return readLimited(r, limit) +} + +// readMetadataBlock reads one metadata block starting at absolute offset +// `off` and returns (data, bytesConsumed, error). bytesConsumed includes +// the 2-byte header. +func (f *FS) readMetadataBlock(off int64) ([]byte, int64, error) { + if off < 0 || off+2 > f.d.Size() { + return nil, 0, fmt.Errorf("metadata block offset %d out of range", off) + } + hdrBuf, err := f.d.Range(off, 2) + if err != nil { + return nil, 0, err + } + hdr := binary.LittleEndian.Uint16(hdrBuf) + compressed := hdr&0x8000 == 0 + size := int64(hdr & 0x7fff) + if size == 0 || size > metadataBlockSize { + return nil, 0, fmt.Errorf("metadata block size %d invalid", size) + } + body, err := f.d.Range(off+2, size) + if err != nil { + return nil, 0, err + } + if !compressed { + return append([]byte(nil), body...), size + 2, nil + } + out, err := f.decompress(body) + if err != nil { + return nil, 0, err + } + return out, size + 2, nil +} + +// metaCursor streams uncompressed data out of a chain of metadata blocks. +// table[i] is the absolute file offset of the i-th metadata block, plus +// a running byte cursor within the uncompressed stream. +type metaCursor struct { + f *FS + base int64 // absolute file offset of first metadata block + blkAbs int64 // absolute file offset of *current* metadata block + inBlk int // index within current uncompressed block + buffer []byte // current decompressed block +} + +func (f *FS) metaCursorAt(base int64, blockOffset uint32, inBlk uint16) (*metaCursor, error) { + mc := &metaCursor{f: f, base: base, blkAbs: base + int64(blockOffset)} + if err := mc.load(); err != nil { + return nil, err + } + if int(inBlk) > len(mc.buffer) { + return nil, fmt.Errorf("inode intra-block offset %d > block length %d", inBlk, len(mc.buffer)) + } + mc.inBlk = int(inBlk) + return mc, nil +} + +func (mc *metaCursor) load() error { + data, _, err := mc.f.readMetadataBlock(mc.blkAbs) + if err != nil { + return err + } + mc.buffer = data + mc.inBlk = 0 + return nil +} + +// advance one metadata block. The on-disk size is the consumed length of +// the current compressed block, which we must read from the header again. +func (mc *metaCursor) nextBlock() error { + // Re-read just the 2-byte header of current block to find its length. + hdrBuf, err := mc.f.d.Range(mc.blkAbs, 2) + if err != nil { + return err + } + hdr := binary.LittleEndian.Uint16(hdrBuf) + size := int64(hdr & 0x7fff) + mc.blkAbs += 2 + size + return mc.load() +} + +// readN returns n uncompressed bytes, crossing metadata-block boundaries +// as required. +func (mc *metaCursor) readN(n int) ([]byte, error) { + if n <= 0 { + return nil, nil + } + out := make([]byte, 0, n) + for n > 0 { + if mc.inBlk >= len(mc.buffer) { + if err := mc.nextBlock(); err != nil { + return nil, err + } + } + take := len(mc.buffer) - mc.inBlk + if take > n { + take = n + } + out = append(out, mc.buffer[mc.inBlk:mc.inBlk+take]...) + mc.inBlk += take + n -= take + } + return out, nil +} + +// ---------- inode ---------- + +type inodeHeader struct { + Type uint16 + Mode uint16 + UID uint16 + GID uint16 + Mtime uint32 + Num uint32 +} + +func readInodeHeader(b []byte) (inodeHeader, error) { + var h inodeHeader + if len(b) < 16 { + return h, errors.New("inode header short") + } + h.Type = binary.LittleEndian.Uint16(b[0:2]) + h.Mode = binary.LittleEndian.Uint16(b[2:4]) + h.UID = binary.LittleEndian.Uint16(b[4:6]) + h.GID = binary.LittleEndian.Uint16(b[6:8]) + h.Mtime = binary.LittleEndian.Uint32(b[8:12]) + h.Num = binary.LittleEndian.Uint32(b[12:16]) + return h, nil +} + +// readInodeAt reads the inode described by a 64-bit ref and returns the +// filled Entry plus the metadata location of the directory contents (if +// it's a directory). +type inodeContent struct { + entry Entry + dirStart uint32 // offset into DirTable (bytes from sb.DirTable) + dirBlockOff uint16 // intra-metadata-block offset for first dir header + dirSize uint32 // directory listing size, real value (file_size - 3) + parentInode uint32 +} + +func (f *FS) readInode(ref uint64) (*inodeContent, error) { + blkOff := uint32(ref >> 16) + inBlk := uint16(ref & 0xffff) + mc, err := f.metaCursorAt(int64(f.sb.InodeTable), blkOff, inBlk) + if err != nil { + return nil, err + } + hdrBytes, err := mc.readN(16) + if err != nil { + return nil, err + } + h, err := readInodeHeader(hdrBytes) + if err != nil { + return nil, err + } + ic := &inodeContent{ + entry: Entry{ + Mode: h.Mode, + UID: h.UID, + GID: h.GID, + Mtime: h.Mtime, + InodeNumber: h.Num, + InodeRef: ref, + }, + } + switch h.Type { + case itBasicDir: + body, err := mc.readN(16) + if err != nil { + return nil, err + } + ic.entry.Type = "dir" + ic.dirStart = binary.LittleEndian.Uint32(body[0:4]) + // nlink := binary.LittleEndian.Uint32(body[4:8]) + fileSize := binary.LittleEndian.Uint16(body[8:10]) + ic.dirBlockOff = binary.LittleEndian.Uint16(body[10:12]) + ic.parentInode = binary.LittleEndian.Uint32(body[12:16]) + // file_size in dir inodes is real dir bytes + 3. Zero-size (3) → empty dir. + if fileSize >= 3 { + ic.dirSize = uint32(fileSize) - 3 + } + case itExtDir: + body, err := mc.readN(24) + if err != nil { + return nil, err + } + ic.entry.Type = "dir" + // nlink := binary.LittleEndian.Uint32(body[0:4]) + fileSize := binary.LittleEndian.Uint32(body[4:8]) + ic.dirStart = binary.LittleEndian.Uint32(body[8:12]) + ic.parentInode = binary.LittleEndian.Uint32(body[12:16]) + indexCount := binary.LittleEndian.Uint16(body[16:18]) + ic.dirBlockOff = binary.LittleEndian.Uint16(body[18:20]) + // xattr := binary.LittleEndian.Uint32(body[20:24]) + if fileSize >= 3 { + ic.dirSize = fileSize - 3 + } + // Skip index entries: indexCount × (4 + 4 + 4 + name[...variable]). + for i := uint16(0); i < indexCount; i++ { + hdr, err := mc.readN(12) + if err != nil { + return nil, err + } + nameSize := binary.LittleEndian.Uint32(hdr[8:12]) + if nameSize > 256 { + return nil, fmt.Errorf("dir index name_size too large: %d", nameSize) + } + if _, err := mc.readN(int(nameSize) + 1); err != nil { + return nil, err + } + } + case itBasicFile: + body, err := mc.readN(16) + if err != nil { + return nil, err + } + ic.entry.Type = "file" + ic.entry.blocksStart = uint64(binary.LittleEndian.Uint32(body[0:4])) + ic.entry.fragIndex = binary.LittleEndian.Uint32(body[4:8]) + ic.entry.fragOffset = binary.LittleEndian.Uint32(body[8:12]) + fileSize := binary.LittleEndian.Uint32(body[12:16]) + ic.entry.fileSize = int64(fileSize) + ic.entry.Size = int64(fileSize) + blocks, err := readBlockSizes(mc, fileSize, ic.entry.fragIndex, f.sb.BlockSize) + if err != nil { + return nil, err + } + ic.entry.blockSizes = blocks + case itExtFile: + body, err := mc.readN(40) + if err != nil { + return nil, err + } + ic.entry.Type = "file" + ic.entry.blocksStart = binary.LittleEndian.Uint64(body[0:8]) + fileSize := binary.LittleEndian.Uint64(body[8:16]) + // sparse := binary.LittleEndian.Uint64(body[16:24]) + // nlink := binary.LittleEndian.Uint32(body[24:28]) + ic.entry.fragIndex = binary.LittleEndian.Uint32(body[28:32]) + ic.entry.fragOffset = binary.LittleEndian.Uint32(body[32:36]) + // xattr := binary.LittleEndian.Uint32(body[36:40]) + ic.entry.fileSize = int64(fileSize) + ic.entry.Size = int64(fileSize) + if fileSize > uint64(1)<<48 { + return nil, fmt.Errorf("unreasonable file size %d", fileSize) + } + blocks, err := readBlockSizes(mc, uint32(fileSize), ic.entry.fragIndex, f.sb.BlockSize) + if err != nil { + return nil, err + } + ic.entry.blockSizes = blocks + case itBasicSymlink, itExtSymlink: + body, err := mc.readN(8) + if err != nil { + return nil, err + } + // nlink := binary.LittleEndian.Uint32(body[0:4]) + target := binary.LittleEndian.Uint32(body[4:8]) + if target > 4096 { + return nil, fmt.Errorf("symlink target length %d too large", target) + } + link, err := mc.readN(int(target)) + if err != nil { + return nil, err + } + ic.entry.Type = "symlink" + ic.entry.LinkTarget = string(link) + ic.entry.Size = int64(target) + if h.Type == itExtSymlink { + // trailing xattr u32 + if _, err := mc.readN(4); err != nil { + return nil, err + } + } + default: + ic.entry.Type = "special" + } + return ic, nil +} + +func readBlockSizes(mc *metaCursor, fileSize uint32, fragIdx uint32, blockSize uint32) ([]uint32, error) { + // If there is a fragment (fragIdx != 0xFFFFFFFF), the tail of the file + // (< blockSize) is stored there — do not count it as a full block. + full := fileSize / blockSize + if fragIdx == 0xFFFFFFFF && fileSize%blockSize != 0 { + full++ // final partial block stored on disk as a block + } + if full > (1 << 20) { + return nil, fmt.Errorf("unreasonable block count %d", full) + } + out := make([]uint32, full) + for i := range out { + b, err := mc.readN(4) + if err != nil { + return nil, err + } + out[i] = binary.LittleEndian.Uint32(b) + } + return out, nil +} + +// ---------- directory walking ---------- + +// readDirEntries returns all (name, inode-ref) tuples in a directory. +func (f *FS) readDirEntries(parent *inodeContent) ([]dirEntry, error) { + if parent.dirSize == 0 { + return nil, nil + } + // dirStart is an offset into the dir table (relative to sb.DirTable). + // dirBlockOff is the intra-block start. + mc, err := f.metaCursorAt(int64(f.sb.DirTable), parent.dirStart, parent.dirBlockOff) + if err != nil { + return nil, err + } + remaining := int(parent.dirSize) + var out []dirEntry + for remaining > 0 { + hdr, err := mc.readN(12) + if err != nil { + return nil, err + } + remaining -= 12 + count := binary.LittleEndian.Uint32(hdr[0:4]) + 1 // stored as count-1 + startBlock := binary.LittleEndian.Uint32(hdr[4:8]) + baseInode := binary.LittleEndian.Uint32(hdr[8:12]) + for i := uint32(0); i < count; i++ { + entHdr, err := mc.readN(8) + if err != nil { + return nil, err + } + remaining -= 8 + entOff := binary.LittleEndian.Uint16(entHdr[0:2]) + iDelta := int16(binary.LittleEndian.Uint16(entHdr[2:4])) + entType := binary.LittleEndian.Uint16(entHdr[4:6]) + nameSize := binary.LittleEndian.Uint16(entHdr[6:8]) + if nameSize+1 > 256 { + return nil, fmt.Errorf("directory entry name_size %d too large", nameSize) + } + name, err := mc.readN(int(nameSize) + 1) // stored as name_size-1 + if err != nil { + return nil, err + } + remaining -= int(nameSize) + 1 + ref := (uint64(startBlock) << 16) | uint64(entOff) + out = append(out, dirEntry{ + name: string(name), + ref: ref, + inodeNum: uint32(int64(baseInode) + int64(iDelta)), + entType: entType, + }) + if len(out) > maxEntries { + return nil, fmt.Errorf("directory entry cap %d exceeded", maxEntries) + } + } + } + return out, nil +} + +type dirEntry struct { + name string + ref uint64 + inodeNum uint32 + entType uint16 +} + +// Walk walks the entire filesystem from the root, emitting every Entry. +// The returned list is sorted by path. Walk is bounded by maxEntries +// and by recursion depth. +func (f *FS) Walk() ([]Entry, error) { + root, err := f.readInode(f.sb.RootInodeRef) + if err != nil { + return nil, fmt.Errorf("root inode: %w", err) + } + if root.entry.Type != "dir" { + return nil, fmt.Errorf("root is not a directory (type %q)", root.entry.Type) + } + out := []Entry{} + root.entry.Path = "/" + out = append(out, root.entry) + if err := f.walk(root, "/", &out, 0); err != nil { + return nil, err + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out, nil +} + +func (f *FS) walk(node *inodeContent, prefix string, out *[]Entry, depth int) error { + if depth > 64 { + return errors.New("directory depth exceeds 64 — possible cycle") + } + entries, err := f.readDirEntries(node) + if err != nil { + return err + } + for _, de := range entries { + child, err := f.readInode(de.ref) + if err != nil { + return fmt.Errorf("%s/%s: %w", prefix, de.name, err) + } + full := path.Join(prefix, de.name) + child.entry.Path = full + *out = append(*out, child.entry) + if len(*out) > maxEntries { + return fmt.Errorf("entry cap %d exceeded", maxEntries) + } + if child.entry.Type == "dir" { + if err := f.walk(child, full, out, depth+1); err != nil { + return err + } + } + } + return nil +} + +// ---------- file extraction ---------- + +// fragmentEntry describes one entry in the fragment table. +type fragmentEntry struct { + Start uint64 + Size uint32 // top bit = uncompressed + Compressed bool +} + +// fragmentTable reads the entire fragment lookup table and returns the +// flat list of entries by index. Result is cached on first call. +func (f *FS) fragmentTable() ([]fragmentEntry, error) { + if f.sb.FragCount == 0 { + return nil, nil + } + // The fragment_table_start points at a u64[] of metadata-block offsets, + // one per 512 fragments (ceil). + indirCount := (int(f.sb.FragCount) + 511) / 512 + indirBytes, err := f.d.Range(int64(f.sb.FragTable), int64(indirCount*8)) + if err != nil { + return nil, fmt.Errorf("frag indirection: %w", err) + } + out := make([]fragmentEntry, 0, f.sb.FragCount) + remaining := int(f.sb.FragCount) + for i := 0; i < indirCount && remaining > 0; i++ { + absOff := int64(binary.LittleEndian.Uint64(indirBytes[i*8 : i*8+8])) + block, _, err := f.readMetadataBlock(absOff) + if err != nil { + return nil, err + } + // Each fragment entry is 16 bytes: u64 start, u32 size, u32 unused. + take := remaining + if take > 512 { + take = 512 + } + if len(block) < take*16 { + return nil, fmt.Errorf("fragment block short: have %d need %d", len(block), take*16) + } + for j := 0; j < take; j++ { + off := j * 16 + sz := binary.LittleEndian.Uint32(block[off+8 : off+12]) + out = append(out, fragmentEntry{ + Start: binary.LittleEndian.Uint64(block[off : off+8]), + Size: sz & 0x00ffffff, // documented size field occupies low 24 bits + Compressed: sz&0x01000000 == 0, + }) + } + remaining -= take + } + return out, nil +} + +// ExtractFile returns the full decompressed contents of the file at the +// given InodeRef. Returns an error if the inode is not a file. +func (f *FS) ExtractFile(ref uint64) ([]byte, error) { + ic, err := f.readInode(ref) + if err != nil { + return nil, err + } + if ic.entry.Type != "file" { + return nil, fmt.Errorf("inode %d is not a file (%s)", ic.entry.InodeNumber, ic.entry.Type) + } + out := make([]byte, 0, ic.entry.fileSize) + cursor := int64(ic.entry.blocksStart) + for i, bs := range ic.entry.blockSizes { + size := int64(bs & 0x00ffffff) + compressed := bs&0x01000000 == 0 + if size == 0 { + // sparse hole: emit blockSize zeroes (or trailing partial) + remaining := ic.entry.fileSize - int64(len(out)) + n := int64(f.sb.BlockSize) + if remaining < n { + n = remaining + } + out = append(out, make([]byte, n)...) + continue + } + if size > dataBlockMax { + return nil, fmt.Errorf("data block %d size %d exceeds cap", i, size) + } + body, err := f.d.Range(cursor, size) + if err != nil { + return nil, fmt.Errorf("data block %d: %w", i, err) + } + cursor += size + if !compressed { + out = append(out, body...) + continue + } + dec, err := f.decompressBlock(body) + if err != nil { + return nil, fmt.Errorf("data block %d: %w", i, err) + } + out = append(out, dec...) + } + if ic.entry.fragIndex != 0xFFFFFFFF { + frags, err := f.fragmentTable() + if err != nil { + return nil, err + } + if int(ic.entry.fragIndex) >= len(frags) { + return nil, fmt.Errorf("fragment index %d out of range (%d entries)", ic.entry.fragIndex, len(frags)) + } + fe := frags[ic.entry.fragIndex] + fragBody, err := f.d.Range(int64(fe.Start), int64(fe.Size)) + if err != nil { + return nil, fmt.Errorf("fragment data: %w", err) + } + var payload []byte + if fe.Compressed { + payload, err = f.decompressBlock(fragBody) + if err != nil { + return nil, err + } + } else { + payload = fragBody + } + tailLen := int(ic.entry.fileSize) - len(out) + if int(ic.entry.fragOffset)+tailLen > len(payload) { + return nil, fmt.Errorf("fragment slice out of range: off=%d len=%d payload=%d", + ic.entry.fragOffset, tailLen, len(payload)) + } + out = append(out, payload[ic.entry.fragOffset:int(ic.entry.fragOffset)+tailLen]...) + } + if int64(len(out)) != ic.entry.fileSize { + return nil, fmt.Errorf("extraction length mismatch: got %d, want %d", len(out), ic.entry.fileSize) + } + return out, nil +} + +// decompressBlock is like decompress but bounded by dataBlockMax; used for fragment payloads. +func (f *FS) decompressBlock(buf []byte) ([]byte, error) { + return f.decompressWithLimit(buf, dataBlockMax) +} + +func readLimited(r io.Reader, limit int64) ([]byte, error) { + if limit <= 0 { + return nil, errors.New("invalid decompression limit") + } + lr := &io.LimitedReader{R: r, N: limit + 1} + out, err := io.ReadAll(lr) + if err != nil { + return nil, err + } + if int64(len(out)) > limit { + return nil, fmt.Errorf("decompressed block exceeds cap %d", limit) + } + return out, nil +} diff --git a/internal/leviathan/stego/metadata.go b/internal/leviathan/stego/metadata.go new file mode 100644 index 0000000..b8c89a0 --- /dev/null +++ b/internal/leviathan/stego/metadata.go @@ -0,0 +1,694 @@ +package stego + +import ( + "bytes" + "compress/zlib" + "encoding/binary" + "errors" + "fmt" + "io" + "strings" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// MetaEntry is one piece of textual/EXIF metadata recovered from an image. +type MetaEntry struct { + Offset int64 `json:"offset"` + Length int64 `json:"length"` + Container string `json:"container"` + Kind string `json:"kind"` + Keyword string `json:"keyword"` + Value string `json:"value"` + Note string `json:"note,omitempty"` +} + +// ScanMetadata walks PNG chunks or JPEG segments (APPn / COM) and decodes +// human-readable metadata. Output is best-effort — malformed containers +// return whatever has been parsed up to the failure point. +func ScanMetadata(d *document.Document) ([]MetaEntry, error) { + size := d.Size() + if size < 8 { + return nil, errors.New("file too small") + } + head, err := d.Range(0, minInt64(16, size)) + if err != nil { + return nil, err + } + switch { + case len(head) >= 8 && string(head[:8]) == "\x89PNG\r\n\x1a\n": + return scanPNGMeta(d) + case len(head) >= 3 && head[0] == 0xff && head[1] == 0xd8 && head[2] == 0xff: + return scanJPEGMeta(d) + case len(head) >= 6 && (string(head[:6]) == "GIF87a" || string(head[:6]) == "GIF89a"): + return scanGIFMeta(d) + } + return nil, errors.New("unsupported container (PNG, JPEG and GIF handled)") +} + +func scanPNGMeta(d *document.Document) ([]MetaEntry, error) { + size := d.Size() + out := []MetaEntry{} + off := int64(8) + for off+12 <= size { + hdr, err := d.Range(off, 8) + if err != nil { + break + } + ln := int64(binary.BigEndian.Uint32(hdr[0:4])) + typ := string(hdr[4:8]) + if ln < 0 || off+12+ln > size { + break + } + if typ == "tEXt" || typ == "zTXt" || typ == "iTXt" { + data, err := d.Range(off+8, ln) + if err == nil { + entry := decodePNGTextChunk(typ, data) + entry.Offset = off + entry.Length = 12 + ln + entry.Container = "png" + out = append(out, entry) + } + } + off += 12 + ln + if typ == "IEND" { + break + } + } + return out, nil +} + +func decodePNGTextChunk(typ string, data []byte) MetaEntry { + e := MetaEntry{Kind: typ} + i := bytes.IndexByte(data, 0) + if i < 0 { + e.Note = "no keyword separator" + e.Value = safeText(data) + return e + } + e.Keyword = string(data[:i]) + rest := data[i+1:] + switch typ { + case "tEXt": + e.Value = safeText(rest) + case "zTXt": + if len(rest) < 1 { + e.Note = "truncated" + return e + } + // rest[0] = compression method (0 = zlib) + method := rest[0] + raw, err := zlibDecompress(rest[1:]) + if err != nil { + e.Note = fmt.Sprintf("zlib fail (method=%d): %v", method, err) + return e + } + e.Value = safeText(raw) + case "iTXt": + // compression flag, compression method, language tag\0, translated keyword\0, text + if len(rest) < 2 { + e.Note = "truncated" + return e + } + compressed := rest[0] == 1 + rest = rest[2:] + j := bytes.IndexByte(rest, 0) + if j < 0 { + e.Value = safeText(rest) + return e + } + lang := string(rest[:j]) + rest = rest[j+1:] + k := bytes.IndexByte(rest, 0) + if k < 0 { + e.Value = safeText(rest) + return e + } + trk := string(rest[:k]) + rest = rest[k+1:] + if compressed { + if raw, err := zlibDecompress(rest); err == nil { + rest = raw + } else { + e.Note = "zlib fail: " + err.Error() + } + } + v := safeText(rest) + if lang != "" || trk != "" { + v = fmt.Sprintf("[%s/%s] %s", lang, trk, v) + } + e.Value = v + } + return e +} + +func scanJPEGMeta(d *document.Document) ([]MetaEntry, error) { + size := d.Size() + out := []MetaEntry{} + off := int64(2) // skip SOI + for off+4 <= size { + hdr, err := d.Range(off, 2) + if err != nil || hdr[0] != 0xff { + break + } + marker := hdr[1] + // Standalone markers (no payload): SOI/EOI/RSTn + if marker == 0xd9 || marker == 0xd8 || (marker >= 0xd0 && marker <= 0xd7) { + off += 2 + continue + } + // Read segment length (big-endian, includes the length bytes themselves) + lnBytes, err := d.Range(off+2, 2) + if err != nil { + break + } + segLen := int64(binary.BigEndian.Uint16(lnBytes)) + if segLen < 2 || off+2+segLen > size { + break + } + payload, err := d.Range(off+4, segLen-2) + if err != nil { + break + } + switch { + case marker == 0xfe: // COM + out = append(out, MetaEntry{ + Offset: off, Length: 2 + segLen, + Container: "jpeg", Kind: "COM", + Value: safeText(payload), + }) + case marker == 0xe1: // APP1 — EXIF / XMP + entries := decodeJPEGApp1(payload) + for _, e := range entries { + e.Offset = off + e.Length = 2 + segLen + e.Container = "jpeg" + out = append(out, e) + } + case marker == 0xe0: // APP0 — JFIF + out = append(out, MetaEntry{ + Offset: off, Length: 2 + segLen, + Container: "jpeg", Kind: "APP0", + Keyword: sniffMarker(payload), + Value: fmt.Sprintf("%d bytes", len(payload)), + }) + case marker >= 0xe2 && marker <= 0xef: // APP2..APP15 + out = append(out, MetaEntry{ + Offset: off, Length: 2 + segLen, + Container: "jpeg", Kind: fmt.Sprintf("APP%d", marker-0xe0), + Keyword: sniffMarker(payload), + Value: fmt.Sprintf("%d bytes", len(payload)), + }) + } + off += 2 + segLen + // SOS (Start of Scan) — entropy-coded data follows, stop walking. + if marker == 0xda { + break + } + } + return out, nil +} + +func sniffMarker(p []byte) string { + i := bytes.IndexByte(p, 0) + if i < 0 || i > 16 { + if len(p) > 16 { + return string(p[:16]) + } + return "" + } + return string(p[:i]) +} + +func decodeJPEGApp1(p []byte) []MetaEntry { + out := []MetaEntry{} + if len(p) >= 6 && string(p[:6]) == "Exif\x00\x00" { + tiff := p[6:] + entries, err := decodeEXIF(tiff) + if err == nil { + out = append(out, entries...) + } else { + out = append(out, MetaEntry{Kind: "APP1/EXIF", Note: err.Error()}) + } + return out + } + if len(p) >= 28 && string(p[:28]) == "http://ns.adobe.com/xap/1.0/" { + xmp := p[29:] + out = append(out, MetaEntry{ + Kind: "APP1/XMP", Keyword: "xmp", Value: safeText(xmp), + }) + return out + } + out = append(out, MetaEntry{ + Kind: "APP1", Keyword: sniffMarker(p), Value: fmt.Sprintf("%d bytes", len(p)), + }) + return out +} + +// EXIF reader: TIFF header + IFD0 + IFD1 + Exif SubIFD + GPS SubIFD + +// Interoperability IFD. Handles ASCII/BYTE/SHORT/LONG/RATIONAL/SRATIONAL +// values, plus UNDEFINED as hex preview. Detects MakerNote presence. +func decodeEXIF(tiff []byte) ([]MetaEntry, error) { + if len(tiff) < 8 { + return nil, errors.New("short TIFF header") + } + var bo binary.ByteOrder + switch string(tiff[:2]) { + case "II": + bo = binary.LittleEndian + case "MM": + bo = binary.BigEndian + default: + return nil, errors.New("bad TIFF byte-order mark") + } + if bo.Uint16(tiff[2:4]) != 0x002a { + return nil, errors.New("bad TIFF magic") + } + ifd0 := int(bo.Uint32(tiff[4:8])) + out := []MetaEntry{} + ptrs := ifdPointers{} + nextIFD := 0 + out = append(out, readIFD(tiff, ifd0, bo, "IFD0", &ptrs, &nextIFD)...) + // IFD1 (thumbnail) if present. + if nextIFD > 0 { + out = append(out, readIFD(tiff, nextIFD, bo, "IFD1", nil, nil)...) + } + if ptrs.exif > 0 { + out = append(out, readIFD(tiff, ptrs.exif, bo, "ExifIFD", &ptrs, nil)...) + } + if ptrs.gps > 0 { + out = append(out, readIFD(tiff, ptrs.gps, bo, "GPSIFD", nil, nil)...) + } + if ptrs.interop > 0 { + out = append(out, readIFD(tiff, ptrs.interop, bo, "InteropIFD", nil, nil)...) + } + return out, nil +} + +type ifdPointers struct { + exif int + gps int + interop int +} + +func readIFD(tiff []byte, off int, bo binary.ByteOrder, name string, ptrs *ifdPointers, nextPtr *int) []MetaEntry { + out := []MetaEntry{} + if off < 0 || off+2 > len(tiff) { + return out + } + n := int(bo.Uint16(tiff[off : off+2])) + base := off + 2 + // Cap directory count to head off hostile input. + if n > 512 { + n = 512 + } + for i := 0; i < n; i++ { + p := base + i*12 + if p+12 > len(tiff) { + break + } + tag := bo.Uint16(tiff[p : p+2]) + typ := bo.Uint16(tiff[p+2 : p+4]) + count := int(bo.Uint32(tiff[p+4 : p+8])) + valRaw := tiff[p+8 : p+12] + // Sub-IFD pointers never appear as regular entries in the output. + if ptrs != nil { + switch tag { + case 0x8769: // Exif SubIFD pointer + ptrs.exif = int(bo.Uint32(valRaw)) + continue + case 0x8825: // GPS IFD pointer + ptrs.gps = int(bo.Uint32(valRaw)) + continue + case 0xa005: // Interoperability IFD pointer + ptrs.interop = int(bo.Uint32(valRaw)) + continue + } + } + e := MetaEntry{Kind: name, Keyword: exifTagName(tag, name)} + if tag == 0x927c { // MakerNote + e.Value = fmt.Sprintf("%d bytes (vendor-specific, not decoded)", count) + out = append(out, e) + continue + } + e.Value = readEXIFValue(tiff, bo, typ, count, valRaw) + out = append(out, e) + } + // "Next IFD" pointer follows the directory entries (4 bytes). + if nextPtr != nil { + tail := base + n*12 + if tail+4 <= len(tiff) { + *nextPtr = int(bo.Uint32(tiff[tail : tail+4])) + } + } + return out +} + +func readEXIFValue(tiff []byte, bo binary.ByteOrder, typ uint16, count int, valRaw []byte) string { + typeSize := map[uint16]int{1: 1, 2: 1, 3: 2, 4: 4, 5: 8, 7: 1, 9: 4, 10: 8} + sz, ok := typeSize[typ] + if !ok { + return fmt.Sprintf("type=%d count=%d", typ, count) + } + total := sz * count + var data []byte + if total <= 4 { + data = valRaw[:total] + } else { + off := int(bo.Uint32(valRaw)) + if off < 0 || off+total > len(tiff) { + return fmt.Sprintf("type=%d count=%d (oob)", typ, count) + } + data = tiff[off : off+total] + } + switch typ { + case 2: // ASCII (null-terminated) + return safeText(bytes.TrimRight(data, "\x00")) + case 1, 7: // BYTE, UNDEFINED — show as hex preview + if len(data) > 32 { + return fmt.Sprintf("%x… (%d bytes)", data[:32], len(data)) + } + return fmt.Sprintf("%x", data) + case 3: // SHORT + parts := []string{} + for i := 0; i < count && i < 8; i++ { + parts = append(parts, fmt.Sprintf("%d", bo.Uint16(data[i*2:i*2+2]))) + } + if count > 8 { + parts = append(parts, "…") + } + return strings.Join(parts, ", ") + case 4: // LONG + parts := []string{} + for i := 0; i < count && i < 6; i++ { + parts = append(parts, fmt.Sprintf("%d", bo.Uint32(data[i*4:i*4+4]))) + } + if count > 6 { + parts = append(parts, "…") + } + return strings.Join(parts, ", ") + case 5, 10: // RATIONAL / SRATIONAL + parts := []string{} + for i := 0; i < count && i < 4; i++ { + n := bo.Uint32(data[i*8 : i*8+4]) + dn := bo.Uint32(data[i*8+4 : i*8+8]) + if dn == 0 { + parts = append(parts, fmt.Sprintf("%d/0", n)) + } else { + parts = append(parts, fmt.Sprintf("%d/%d", n, dn)) + } + } + return strings.Join(parts, ", ") + case 9: // SLONG + parts := []string{} + for i := 0; i < count && i < 6; i++ { + parts = append(parts, fmt.Sprintf("%d", int32(bo.Uint32(data[i*4:i*4+4])))) + } + return strings.Join(parts, ", ") + } + return fmt.Sprintf("type=%d count=%d", typ, count) +} + +func exifTagName(tag uint16, ifd string) string { + if ifd == "GPSIFD" { + if n, ok := gpsTags[tag]; ok { + return n + } + return fmt.Sprintf("GPS tag 0x%04x", tag) + } + if n, ok := exifTags[tag]; ok { + return n + } + return fmt.Sprintf("tag 0x%04x", tag) +} + +// exifTags covers the standard TIFF/Exif tags routinely seen in +// JPEG/TIFF images produced by consumer cameras and phones. +var exifTags = map[uint16]string{ + // TIFF / IFD0 + 0x0100: "ImageWidth", 0x0101: "ImageLength", + 0x0102: "BitsPerSample", 0x0103: "Compression", + 0x0106: "PhotometricInterpretation", 0x010a: "FillOrder", + 0x010d: "DocumentName", 0x010e: "ImageDescription", + 0x010f: "Make", 0x0110: "Model", 0x0111: "StripOffsets", + 0x0112: "Orientation", 0x0115: "SamplesPerPixel", + 0x0116: "RowsPerStrip", 0x0117: "StripByteCounts", + 0x011a: "XResolution", 0x011b: "YResolution", + 0x011c: "PlanarConfiguration", 0x0128: "ResolutionUnit", + 0x012d: "TransferFunction", 0x0131: "Software", 0x0132: "DateTime", + 0x013b: "Artist", 0x013c: "HostComputer", + 0x013e: "WhitePoint", 0x013f: "PrimaryChromaticities", + 0x0211: "YCbCrCoefficients", 0x0212: "YCbCrSubSampling", + 0x0213: "YCbCrPositioning", 0x0214: "ReferenceBlackWhite", + 0x02bc: "XMP", + 0x8298: "Copyright", 0x83bb: "IPTCNAA", + 0x8769: "ExifIFDPointer", 0x8773: "InterColorProfile", + 0x8825: "GPSIFDPointer", + // Exif SubIFD + 0x829a: "ExposureTime", 0x829d: "FNumber", + 0x8822: "ExposureProgram", 0x8824: "SpectralSensitivity", + 0x8827: "ISOSpeedRatings", 0x8828: "OECF", + 0x9000: "ExifVersion", + 0x9003: "DateTimeOriginal", 0x9004: "DateTimeDigitized", + 0x9010: "OffsetTime", 0x9011: "OffsetTimeOriginal", 0x9012: "OffsetTimeDigitized", + 0x9101: "ComponentsConfiguration", 0x9102: "CompressedBitsPerPixel", + 0x9201: "ShutterSpeedValue", 0x9202: "ApertureValue", + 0x9203: "BrightnessValue", 0x9204: "ExposureBiasValue", + 0x9205: "MaxApertureValue", 0x9206: "SubjectDistance", + 0x9207: "MeteringMode", 0x9208: "LightSource", 0x9209: "Flash", + 0x920a: "FocalLength", + 0x9214: "SubjectArea", 0x927c: "MakerNote", 0x9286: "UserComment", + 0x9290: "SubSecTime", 0x9291: "SubSecTimeOriginal", 0x9292: "SubSecTimeDigitized", + 0xa000: "FlashpixVersion", 0xa001: "ColorSpace", + 0xa002: "PixelXDimension", 0xa003: "PixelYDimension", + 0xa004: "RelatedSoundFile", 0xa005: "InteropIFDPointer", + 0xa20b: "FlashEnergy", 0xa20c: "SpatialFrequencyResponse", + 0xa20e: "FocalPlaneXResolution", 0xa20f: "FocalPlaneYResolution", + 0xa210: "FocalPlaneResolutionUnit", + 0xa214: "SubjectLocation", 0xa215: "ExposureIndex", + 0xa217: "SensingMethod", 0xa300: "FileSource", 0xa301: "SceneType", + 0xa302: "CFAPattern", 0xa401: "CustomRendered", + 0xa402: "ExposureMode", 0xa403: "WhiteBalance", + 0xa404: "DigitalZoomRatio", 0xa405: "FocalLengthIn35mmFilm", + 0xa406: "SceneCaptureType", 0xa407: "GainControl", + 0xa408: "Contrast", 0xa409: "Saturation", 0xa40a: "Sharpness", + 0xa40b: "DeviceSettingDescription", 0xa40c: "SubjectDistanceRange", + 0xa420: "ImageUniqueID", 0xa430: "OwnerName", 0xa431: "SerialNumber", + 0xa432: "LensSpecification", 0xa433: "LensMake", 0xa434: "LensModel", + 0xa435: "LensSerialNumber", + // Thumbnail IFD1 re-uses IFD0 tags; additional ones: + 0x0201: "JPEGInterchangeFormat", 0x0202: "JPEGInterchangeFormatLength", +} + +var gpsTags = map[uint16]string{ + 0x0000: "GPSVersionID", 0x0001: "GPSLatitudeRef", 0x0002: "GPSLatitude", + 0x0003: "GPSLongitudeRef", 0x0004: "GPSLongitude", + 0x0005: "GPSAltitudeRef", 0x0006: "GPSAltitude", + 0x0007: "GPSTimeStamp", 0x0008: "GPSSatellites", 0x0009: "GPSStatus", + 0x000a: "GPSMeasureMode", 0x000b: "GPSDOP", 0x000c: "GPSSpeedRef", + 0x000d: "GPSSpeed", 0x000e: "GPSTrackRef", 0x000f: "GPSTrack", + 0x0010: "GPSImgDirectionRef", 0x0011: "GPSImgDirection", + 0x0012: "GPSMapDatum", 0x0013: "GPSDestLatitudeRef", 0x0014: "GPSDestLatitude", + 0x0015: "GPSDestLongitudeRef", 0x0016: "GPSDestLongitude", + 0x0017: "GPSDestBearingRef", 0x0018: "GPSDestBearing", + 0x0019: "GPSDestDistanceRef", 0x001a: "GPSDestDistance", + 0x001b: "GPSProcessingMethod", 0x001c: "GPSAreaInformation", + 0x001d: "GPSDateStamp", 0x001e: "GPSDifferential", 0x001f: "GPSHPositioningError", +} + +func zlibDecompress(b []byte) ([]byte, error) { + r, err := zlib.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + defer r.Close() + // 2 MiB cap — textual metadata should never be near this. + lim := &io.LimitedReader{R: r, N: 2 * 1024 * 1024} + return io.ReadAll(lim) +} + +func safeText(b []byte) string { + // Preserve text but trim extreme size and strip control chars other than tab/newline. + if len(b) > 4096 { + b = b[:4096] + } + sb := strings.Builder{} + sb.Grow(len(b)) + for _, c := range b { + if c == '\t' || c == '\n' || c == '\r' || (c >= 0x20 && c <= 0x7e) { + sb.WriteByte(c) + continue + } + if c >= 0x80 { + // Pass through; browser will render as UTF-8 if valid. + sb.WriteByte(c) + continue + } + // Drop other controls. + } + return sb.String() +} + +func minInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +// scanGIFMeta walks a GIF file and surfaces every extension block that +// can carry human-readable data: Comment (0x21 0xFE), Plain Text +// (0x21 0x01), and Application (0x21 0xFF — XMP, Netscape loop, etc.). +func scanGIFMeta(d *document.Document) ([]MetaEntry, error) { + size := d.Size() + out := []MetaEntry{} + // Skip header (6) + logical screen descriptor (7 fixed) and the + // optional global colour table. + hdr, err := d.Range(0, 13) + if err != nil { + return nil, err + } + off := int64(13) + packed := hdr[10] + if packed&0x80 != 0 { + gctSize := int64(3) * (int64(1) << uint((packed&0x07)+1)) + off += gctSize + } + for off < size { + b, err := d.Range(off, 1) + if err != nil { + break + } + switch b[0] { + case 0x3b: // trailer + return out, nil + case 0x2c: // Image Descriptor: skip descriptor + optional LCT + LZW data + next, ok := skipGIFImage(d, off) + if !ok { + return out, nil + } + off = next + case 0x21: // Extension Introducer + if off+2 > size { + return out, nil + } + lbl, _ := d.Range(off+1, 1) + start := off + payload, next, ok := readGIFSubBlocks(d, off+2) + if !ok { + return out, nil + } + switch lbl[0] { + case 0xfe: + out = append(out, MetaEntry{ + Offset: start, Length: next - start, + Container: "gif", Kind: "Comment", + Value: safeText(payload), + }) + case 0x01: + // Plain Text: first 12 bytes are text-grid parameters. + text := payload + if len(text) > 12 { + text = text[12:] + } else { + text = nil + } + out = append(out, MetaEntry{ + Offset: start, Length: next - start, + Container: "gif", Kind: "PlainText", + Value: safeText(text), + }) + case 0xff: + // Application Extension: first sub-block is an 11-byte + // identifier like "NETSCAPE2.0" or "XMP DataXMP". + if len(payload) >= 11 { + appID := string(bytes.TrimRight(payload[:11], "\x00 ")) + rest := payload[11:] + out = append(out, MetaEntry{ + Offset: start, Length: next - start, + Container: "gif", Kind: "Application", + Keyword: appID, + Value: safeText(rest), + }) + } else { + out = append(out, MetaEntry{ + Offset: start, Length: next - start, + Container: "gif", Kind: "Application", + Value: safeText(payload), + }) + } + default: + // Graphic control (0xf9) and others — skip. + } + off = next + default: + // Unknown byte; bail. + return out, nil + } + } + return out, nil +} + +// readGIFSubBlocks reads a chain of GIF sub-blocks: each starts with a +// length byte (0 terminates). Returns (concatenated payload, offset of +// next byte after terminator, ok). +func readGIFSubBlocks(d *document.Document, off int64) ([]byte, int64, bool) { + size := d.Size() + out := []byte{} + for off < size { + ln, err := d.Range(off, 1) + if err != nil { + return nil, off, false + } + n := int64(ln[0]) + off++ + if n == 0 { + return out, off, true + } + if off+n > size { + return nil, off, false + } + block, err := d.Range(off, n) + if err != nil { + return nil, off, false + } + out = append(out, block...) + off += n + // Hard cap to prevent OOM on malformed input. + if len(out) > 16*1024*1024 { + return out, off, true + } + } + return nil, off, false +} + +// skipGIFImage walks an Image Descriptor block starting at off (the 0x2c +// byte), skipping the descriptor, any local colour table, the LZW +// minimum-code-size byte and the image data sub-blocks. Returns the +// offset of the next block, or (0, false) on truncation. +func skipGIFImage(d *document.Document, off int64) (int64, bool) { + size := d.Size() + if off+10 > size { + return 0, false + } + desc, err := d.Range(off, 10) + if err != nil { + return 0, false + } + packed := desc[9] + off += 10 + if packed&0x80 != 0 { + lctSize := int64(3) * (int64(1) << uint((packed&0x07)+1)) + off += lctSize + } + // LZW minimum code size byte, then sub-blocks. + if off+1 > size { + return 0, false + } + off++ + _, next, ok := readGIFSubBlocks(d, off) + if !ok { + return 0, false + } + return next, true +} diff --git a/internal/leviathan/stego/stego.go b/internal/leviathan/stego/stego.go new file mode 100644 index 0000000..021c184 --- /dev/null +++ b/internal/leviathan/stego/stego.go @@ -0,0 +1,501 @@ +// Package stego provides practical steganography helpers: +// +// - trailer detection (bytes after a declared end) for PNG, JPEG, +// GIF, BMP, WAV and RIFF/WebP containers +// - multi-trailer surfacing: identifies any embedded container +// (PNG/JPEG/ZIP/GIF/PDF/RAR/7z) inside the trailer region +// - LSB extract/inject over an arbitrary byte buffer +// - PNG IDAT, BMP pixel-array and WAV sample-data resolvers so +// callers can point LSB at the correct carrier region +// - textual-metadata extraction for PNG, JPEG, GIF (see metadata.go) +package stego + +import ( + "encoding/binary" + "errors" + + "github.com/andydixon/nexus/internal/leviathan/document" +) + +// --------------------------------------------------------------------------- +// Trailer detection +// --------------------------------------------------------------------------- + +// EmbeddedHit describes a recognised container signature found inside the +// trailer region (the bytes after a file's declared end). +type EmbeddedHit struct { + Offset int64 `json:"offset"` // absolute offset within the outer file + Kind string `json:"kind"` + Note string `json:"note,omitempty"` +} + +// AnalyseAppended inspects the end of the file for plausible trailers: data +// after the declared end of a known container. The result also lists any +// nested container magic bytes discovered inside the trailer region. +func AnalyseAppended(d *document.Document) (map[string]any, error) { + size := d.Size() + head, _ := d.Range(0, 16) + result := map[string]any{ + "fileSize": size, + "declaredEnd": int64(-1), + "trailerSize": int64(0), + "container": "", + "note": "no recognised container", + "embedded": []EmbeddedHit{}, + } + + end := int64(-1) + container := "" + + switch { + case len(head) >= 8 && string(head[:8]) == "\x89PNG\r\n\x1a\n": + container = "png" + end = findPNGEnd(d) + case len(head) >= 3 && head[0] == 0xff && head[1] == 0xd8 && head[2] == 0xff: + container = "jpeg" + end = findJPEGEnd(d) + case len(head) >= 6 && (string(head[:6]) == "GIF87a" || string(head[:6]) == "GIF89a"): + container = "gif" + end = findGIFEnd(d) + case len(head) >= 2 && head[0] == 'B' && head[1] == 'M': + container = "bmp" + end = findBMPEnd(d) + case len(head) >= 12 && string(head[:4]) == "RIFF": + sub := string(head[8:12]) + if sub == "WAVE" { + container = "wav" + } else if sub == "WEBP" { + container = "webp" + } else { + container = "riff" + } + end = findRIFFEnd(d) + } + + if container != "" { + result["container"] = container + } + + if end > 0 && end <= size { + result["declaredEnd"] = end + if end < size { + trailer := size - end + result["trailerSize"] = trailer + result["note"] = "bytes after " + container + " declared end" + if trailer > 0 { + // Cap scanning the trailer at 16 MiB — otherwise we might + // pull a 2-GiB tail into memory on hostile inputs. + scanLen := trailer + if scanLen > 16*1024*1024 { + scanLen = 16 * 1024 * 1024 + } + tail, _ := d.Range(end, scanLen) + result["embedded"] = scanEmbedded(tail, end) + } + } else { + result["note"] = "no trailing bytes" + } + } + + return result, nil +} + +// ExtractTrailer returns the bytes after a container's declared end. +func ExtractTrailer(d *document.Document) ([]byte, error) { + a, err := AnalyseAppended(d) + if err != nil { + return nil, err + } + end, _ := a["declaredEnd"].(int64) + ts, _ := a["trailerSize"].(int64) + if end < 0 || ts <= 0 { + return nil, errors.New("no trailer found") + } + return d.Range(end, ts) +} + +func findPNGEnd(d *document.Document) int64 { + size := d.Size() + off := int64(8) + for off+12 <= size { + hdr, err := d.Range(off, 8) + if err != nil { + break + } + ln := int64(binary.BigEndian.Uint32(hdr[0:4])) + if ln < 0 || off+12+ln > size { + break + } + typ := string(hdr[4:8]) + off += 12 + ln + if typ == "IEND" { + return off + } + } + return -1 +} + +func findJPEGEnd(d *document.Document) int64 { + size := d.Size() + tailLen := int64(128 * 1024) + if tailLen > size { + tailLen = size + } + tail, _ := d.Range(size-tailLen, tailLen) + for i := len(tail) - 2; i >= 0; i-- { + if tail[i] == 0xff && tail[i+1] == 0xd9 { + return size - tailLen + int64(i+2) + } + } + return -1 +} + +func findGIFEnd(d *document.Document) int64 { + size := d.Size() + // The GIF terminator 0x3B is a single byte that follows the final + // data block of the final image. Walking the container correctly is + // overkill; scan from the tail and accept the last 0x3B that is + // preceded by a block terminator (0x00) or a plausible frame end. + tailLen := int64(32 * 1024) + if tailLen > size { + tailLen = size + } + tail, _ := d.Range(size-tailLen, tailLen) + for i := len(tail) - 1; i >= 0; i-- { + if tail[i] == 0x3b { + return size - tailLen + int64(i+1) + } + } + return -1 +} + +func findBMPEnd(d *document.Document) int64 { + size := d.Size() + hdr, err := d.Range(0, 14) + if err != nil { + return -1 + } + bfSize := int64(binary.LittleEndian.Uint32(hdr[2:6])) + if bfSize > 0 && bfSize <= size { + return bfSize + } + return -1 +} + +func findRIFFEnd(d *document.Document) int64 { + size := d.Size() + hdr, err := d.Range(0, 8) + if err != nil { + return -1 + } + // RIFF chunk size (little-endian) is the size of the rest of the + // file *after* the first 8 bytes, so declared end = 8 + riffSize. + riffSize := int64(binary.LittleEndian.Uint32(hdr[4:8])) + end := 8 + riffSize + if end > 0 && end <= size { + // RIFF pads to even alignment. + if end%2 == 1 && end+1 <= size { + end++ + } + return end + } + return -1 +} + +// scanEmbedded walks a trailer buffer for known container magic bytes. +func scanEmbedded(buf []byte, base int64) []EmbeddedHit { + hits := []EmbeddedHit{} + type sig struct { + bytes []byte + kind string + } + sigs := []sig{ + {[]byte("\x89PNG\r\n\x1a\n"), "png"}, + {[]byte{0xff, 0xd8, 0xff}, "jpeg"}, + {[]byte("GIF87a"), "gif"}, + {[]byte("GIF89a"), "gif"}, + {[]byte("PK\x03\x04"), "zip"}, + {[]byte("PK\x05\x06"), "zip-eocd"}, + {[]byte("Rar!\x1a\x07\x00"), "rar"}, + {[]byte("Rar!\x1a\x07\x01\x00"), "rar5"}, + {[]byte("7z\xbc\xaf\x27\x1c"), "7z"}, + {[]byte("%PDF-"), "pdf"}, + {[]byte("\x1f\x8b\x08"), "gzip"}, + {[]byte{0xfd, '7', 'z', 'X', 'Z', 0x00}, "xz"}, + {[]byte("BZh"), "bzip2"}, + {[]byte{0x28, 0xb5, 0x2f, 0xfd}, "zstd"}, + {[]byte{0x7f, 'E', 'L', 'F'}, "elf"}, + {[]byte("MZ"), "mz/pe"}, + {[]byte("ustar"), "tar"}, + {[]byte("OggS"), "ogg"}, + {[]byte("ID3"), "id3/mp3"}, + {[]byte("fLaC"), "flac"}, + {[]byte("\x1aEߣ"), "matroska/webm"}, + {[]byte("RIFF"), "riff"}, + } + for _, s := range sigs { + if len(s.bytes) == 0 { + continue + } + start := 0 + for { + i := indexOf(buf[start:], s.bytes) + if i < 0 { + break + } + hits = append(hits, EmbeddedHit{ + Offset: base + int64(start+i), + Kind: s.kind, + }) + // A plain "MZ" or "RIFF" hit without more context is noisy; + // the caller can filter on kind. Keep walking past this hit. + start += i + len(s.bytes) + // Bound total hits per signature to keep UI manageable. + if len(hits) > 256 { + break + } + } + if len(hits) > 256 { + break + } + } + return hits +} + +func indexOf(hay, needle []byte) int { + if len(needle) == 0 || len(hay) < len(needle) { + return -1 + } + first := needle[0] + last := len(hay) - len(needle) + for i := 0; i <= last; i++ { + if hay[i] != first { + continue + } + match := true + for j := 1; j < len(needle); j++ { + if hay[i+j] != needle[j] { + match = false + break + } + } + if match { + return i + } + } + return -1 +} + +// --------------------------------------------------------------------------- +// Carrier resolvers (PNG IDAT, BMP pixels, WAV samples) +// --------------------------------------------------------------------------- + +// PNGImageData concatenates all IDAT chunks' payload bytes. Callers can then +// feed the result into LSBExtract/LSBInject. The returned buffer is the raw +// (still-deflated) IDAT stream — it is the conventional LSB carrier for +// tools like zsteg, not the decoded pixel array. +func PNGImageData(d *document.Document) ([]byte, error) { + size := d.Size() + off := int64(8) + buf := []byte{} + for off+12 <= size { + hdr, err := d.Range(off, 8) + if err != nil { + return nil, err + } + ln := int64(binary.BigEndian.Uint32(hdr[0:4])) + typ := string(hdr[4:8]) + if typ == "IDAT" { + chunk, err := d.Range(off+8, ln) + if err != nil { + return nil, err + } + buf = append(buf, chunk...) + } + off += 12 + ln + if typ == "IEND" { + break + } + } + if len(buf) == 0 { + return nil, errors.New("no IDAT data") + } + return buf, nil +} + +// BMPCarrier describes a resolved BMP pixel array that can be used as an +// LSB carrier. +type BMPCarrier struct { + Data []byte `json:"data"` + Offset int64 `json:"offset"` // offset of pixel array in the file + Width int32 `json:"width"` + Height int32 `json:"height"` + BitsPerPixel uint16 `json:"bitsPerPixel"` + RowStride int `json:"rowStride"` +} + +// BMPPixels locates and returns the BMP pixel array. Supports 8/24/32 bpp +// uncompressed BI_RGB images, which are by far the most common carriers. +func BMPPixels(d *document.Document) (*BMPCarrier, error) { + size := d.Size() + if size < 54 { + return nil, errors.New("not a BMP (too small)") + } + hdr, err := d.Range(0, 54) + if err != nil { + return nil, err + } + if hdr[0] != 'B' || hdr[1] != 'M' { + return nil, errors.New("not a BMP") + } + bfOffBits := int64(binary.LittleEndian.Uint32(hdr[10:14])) + dibSize := binary.LittleEndian.Uint32(hdr[14:18]) + // BITMAPINFOHEADER (40), BITMAPV4HEADER (108), BITMAPV5HEADER (124), + // OS/2 BITMAPCOREHEADER (12). Only handle the modern family. + if dibSize < 40 { + return nil, errors.New("BMP: unsupported DIB header (BITMAPCOREHEADER)") + } + w := int32(binary.LittleEndian.Uint32(hdr[18:22])) + h := int32(binary.LittleEndian.Uint32(hdr[22:26])) + bpp := binary.LittleEndian.Uint16(hdr[28:30]) + compression := binary.LittleEndian.Uint32(hdr[30:34]) + if compression != 0 && compression != 3 { // BI_RGB / BI_BITFIELDS + return nil, errors.New("BMP: compressed images not supported") + } + if bpp != 8 && bpp != 24 && bpp != 32 { + return nil, errors.New("BMP: only 8/24/32 bpp supported") + } + absH := h + if absH < 0 { + absH = -absH + } + rowStride := ((int(w)*int(bpp) + 31) / 32) * 4 + total := int64(rowStride) * int64(absH) + if bfOffBits < 0 || bfOffBits+total > size { + return nil, errors.New("BMP: pixel array out of range") + } + data, err := d.Range(bfOffBits, total) + if err != nil { + return nil, err + } + return &BMPCarrier{ + Data: data, Offset: bfOffBits, + Width: w, Height: h, BitsPerPixel: bpp, RowStride: rowStride, + }, nil +} + +// WAVCarrier describes a resolved WAV "data" chunk that can be used as an +// LSB carrier. +type WAVCarrier struct { + Data []byte `json:"data"` + Offset int64 `json:"offset"` // offset of sample data + SampleRate uint32 `json:"sampleRate"` + NumChannels uint16 `json:"numChannels"` + BitsPerSample uint16 `json:"bitsPerSample"` + AudioFormat uint16 `json:"audioFormat"` +} + +// WAVSamples locates the "data" sub-chunk of a RIFF/WAVE file and returns +// its payload. Only PCM (format 1) is usable as a straightforward LSB +// carrier; other formats still return bytes but are flagged in the caller. +func WAVSamples(d *document.Document) (*WAVCarrier, error) { + size := d.Size() + if size < 44 { + return nil, errors.New("not a WAV (too small)") + } + hdr, err := d.Range(0, 12) + if err != nil { + return nil, err + } + if string(hdr[:4]) != "RIFF" || string(hdr[8:12]) != "WAVE" { + return nil, errors.New("not a RIFF/WAVE file") + } + out := &WAVCarrier{} + off := int64(12) + for off+8 <= size { + ch, err := d.Range(off, 8) + if err != nil { + return nil, err + } + id := string(ch[:4]) + cs := int64(binary.LittleEndian.Uint32(ch[4:8])) + if cs < 0 || off+8+cs > size { + break + } + switch id { + case "fmt ": + body, err := d.Range(off+8, cs) + if err == nil && len(body) >= 16 { + out.AudioFormat = binary.LittleEndian.Uint16(body[0:2]) + out.NumChannels = binary.LittleEndian.Uint16(body[2:4]) + out.SampleRate = binary.LittleEndian.Uint32(body[4:8]) + out.BitsPerSample = binary.LittleEndian.Uint16(body[14:16]) + } + case "data": + body, err := d.Range(off+8, cs) + if err != nil { + return nil, err + } + out.Data = body + out.Offset = off + 8 + return out, nil + } + // Pad to even. + step := 8 + cs + if step%2 == 1 { + step++ + } + off += step + } + return nil, errors.New("WAV: data chunk not found") +} + +// --------------------------------------------------------------------------- +// LSB codec +// --------------------------------------------------------------------------- + +// LSBExtract returns a bitstring-as-bytes built from the low bit of each +// byte in the selected range. Useful for exploring LSB payloads over raw +// buffers or specific regions. +func LSBExtract(buf []byte, bitsPerByte int) []byte { + if bitsPerByte <= 0 || bitsPerByte > 8 { + bitsPerByte = 1 + } + total := len(buf) * bitsPerByte + out := make([]byte, total/8) + bitIdx := 0 + for _, b := range buf { + for i := 0; i < bitsPerByte; i++ { + bit := (b >> uint(i)) & 1 + if bit == 1 { + out[bitIdx/8] |= 1 << uint(7-(bitIdx%8)) + } + bitIdx++ + } + } + return out +} + +// LSBInject replaces low bits in buf with bits from payload. Returns the +// modified buffer. When capacity is exceeded, injection stops. +func LSBInject(buf, payload []byte, bitsPerByte int) []byte { + if bitsPerByte <= 0 || bitsPerByte > 8 { + bitsPerByte = 1 + } + out := append([]byte(nil), buf...) + mask := byte(1<> uint(7-(bitIdx%8))) & 1 + if bit != 0 { + chunk |= 1 << uint(b) + } + bitIdx++ + } + out[i] = (out[i] &^ mask) | (chunk & mask) + } + return out +} diff --git a/internal/leviathan/transforms/cipher.go b/internal/leviathan/transforms/cipher.go new file mode 100644 index 0000000..f5cfca7 --- /dev/null +++ b/internal/leviathan/transforms/cipher.go @@ -0,0 +1,402 @@ +package transforms + +import ( + "math" +) + +// Caesar shifts A-Z/a-z forward by n, leaving other bytes alone. +func Caesar(in []byte, n int) []byte { + n = ((n % 26) + 26) % 26 + out := make([]byte, len(in)) + for i, b := range in { + switch { + case b >= 'A' && b <= 'Z': + out[i] = byte('A' + (int(b-'A')+n)%26) + case b >= 'a' && b <= 'z': + out[i] = byte('a' + (int(b-'a')+n)%26) + default: + out[i] = b + } + } + return out +} + +// Atbash maps A..Z -> Z..A (and lowercase equivalently). +func Atbash(in []byte) []byte { + out := make([]byte, len(in)) + for i, b := range in { + switch { + case b >= 'A' && b <= 'Z': + out[i] = 'A' + 'Z' - b + case b >= 'a' && b <= 'z': + out[i] = 'a' + 'z' - b + default: + out[i] = b + } + } + return out +} + +// Vigenere applies the Vigenère cipher with an ASCII key. encrypt=true +// shifts forward, false shifts backward. Non-letters are preserved and +// do not advance the key. +func Vigenere(in, key []byte, encrypt bool) []byte { + if len(key) == 0 { + return append([]byte(nil), in...) + } + normKey := make([]int, 0, len(key)) + for _, k := range key { + switch { + case k >= 'A' && k <= 'Z': + normKey = append(normKey, int(k-'A')) + case k >= 'a' && k <= 'z': + normKey = append(normKey, int(k-'a')) + } + } + if len(normKey) == 0 { + return append([]byte(nil), in...) + } + out := make([]byte, len(in)) + ki := 0 + for i, b := range in { + var base byte + var letter bool + if b >= 'A' && b <= 'Z' { + base = 'A' + letter = true + } else if b >= 'a' && b <= 'z' { + base = 'a' + letter = true + } + if letter { + shift := normKey[ki%len(normKey)] + if !encrypt { + shift = -shift + } + out[i] = byte(int(base) + ((int(b-base)+shift)%26+26)%26) + ki++ + } else { + out[i] = b + } + } + return out +} + +// English letter frequency (A..Z) in percent. +var englishFreq = [26]float64{ + 8.167, 1.492, 2.782, 4.253, 12.702, 2.228, 2.015, 6.094, 6.966, 0.153, + 0.772, 4.025, 2.406, 6.749, 7.507, 1.929, 0.095, 5.987, 6.327, 9.056, + 2.758, 0.978, 2.360, 0.150, 1.974, 0.074, +} + +// ScoreEnglish returns a heuristic fit to English letter frequencies. +// Higher score = closer to English. ASCII-printable bonus added. +func ScoreEnglish(in []byte) float64 { + if len(in) == 0 { + return 0 + } + var counts [26]int + total := 0 + printable := 0 + for _, b := range in { + if b >= 'A' && b <= 'Z' { + counts[b-'A']++ + total++ + } else if b >= 'a' && b <= 'z' { + counts[b-'a']++ + total++ + } + if (b >= 0x20 && b <= 0x7e) || b == '\n' || b == '\r' || b == '\t' { + printable++ + } + } + printRatio := float64(printable) / float64(len(in)) + if total == 0 { + return printRatio * 10 // give credit for printability + } + // Chi-squared inverse-like score + chi := 0.0 + for i := 0; i < 26; i++ { + obs := float64(counts[i]) / float64(total) * 100 + exp := englishFreq[i] + chi += math.Abs(obs - exp) // L1 distance; simpler than chi^2 for small samples + } + // Lower chi -> better. Convert to a bounded score. + score := (200.0 - chi) + printRatio*50 + if score < 0 { + score = 0 + } + return score +} + +// BruteCaesar returns all 26 shifts with scores, best-first. +func BruteCaesar(in []byte) []map[string]any { + out := make([]map[string]any, 26) + for n := 0; n < 26; n++ { + dec := Caesar(in, n) + out[n] = map[string]any{ + "shift": n, + "score": ScoreEnglish(dec), + "sample": previewText(dec, 120), + "text": string(dec), + } + } + // sort by score desc (simple insertion) + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j-1]["score"].(float64) < out[j]["score"].(float64); j-- { + out[j-1], out[j] = out[j], out[j-1] + } + } + return out +} + +// BruteXORByte returns scored 1-byte XOR candidates, best-first. +func BruteXORByte(in []byte) []map[string]any { + out := make([]map[string]any, 256) + for k := 0; k < 256; k++ { + buf := make([]byte, len(in)) + for i, b := range in { + buf[i] = b ^ byte(k) + } + out[k] = map[string]any{ + "key": k, + "keyHex": byteHex(byte(k)), + "score": ScoreEnglish(buf), + "sample": previewText(buf, 120), + "text": string(buf), + } + } + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j-1]["score"].(float64) < out[j]["score"].(float64); j-- { + out[j-1], out[j] = out[j], out[j-1] + } + } + return out +} + +// RepeatingXORGuess attempts to recover short repeating XOR keys using +// normalized Hamming distance for key-size ranking and per-column +// single-byte XOR scoring. +func RepeatingXORGuess(in []byte, minKey, maxKey, top int) []map[string]any { + if minKey <= 0 { + minKey = 2 + } + if maxKey < minKey { + maxKey = minKey + } + if maxKey > 64 { + maxKey = 64 + } + if top <= 0 { + top = 5 + } + type keySizeScore struct { + size int + score float64 + } + sizes := []keySizeScore{} + for keySize := minKey; keySize <= maxKey; keySize++ { + if len(in) < keySize*4 { + continue + } + dist := 0.0 + pairs := 0 + for block := 0; block < 4; block++ { + a := block * keySize + b := a + keySize + if b+keySize > len(in) { + break + } + dist += float64(hamming(in[a:a+keySize], in[b:b+keySize])) / float64(keySize) + pairs++ + } + if pairs > 0 { + sizes = append(sizes, keySizeScore{size: keySize, score: dist / float64(pairs)}) + } + } + for i := 1; i < len(sizes); i++ { + for j := i; j > 0 && sizes[j-1].score > sizes[j].score; j-- { + sizes[j-1], sizes[j] = sizes[j], sizes[j-1] + } + } + limit := top + if limit > len(sizes) { + limit = len(sizes) + } + out := make([]map[string]any, 0, limit) + for _, ks := range sizes[:limit] { + key := recoverRepeatingXORKey(in, ks.size) + dec := XORRepeating(in, key) + out = append(out, map[string]any{ + "keySize": ks.size, + "distance": ks.score, + "keyHex": bytesHex(key), + "keyText": previewText(key, len(key)), + "score": ScoreEnglish(dec), + "sample": previewText(dec, 120), + "text": string(dec), + "normalizedScore": ScoreEnglish(dec) - ks.score, + }) + } + for i := 1; i < len(out); i++ { + for j := i; j > 0 && out[j-1]["normalizedScore"].(float64) < out[j]["normalizedScore"].(float64); j-- { + out[j-1], out[j] = out[j], out[j-1] + } + } + return out +} + +func recoverRepeatingXORKey(in []byte, keySize int) []byte { + key := make([]byte, keySize) + for pos := 0; pos < keySize; pos++ { + col := make([]byte, 0, len(in)/keySize+1) + for i := pos; i < len(in); i += keySize { + col = append(col, in[i]) + } + bestScore := -1.0 + bestKey := byte(0) + for k := 0; k < 256; k++ { + buf := make([]byte, len(col)) + for i, b := range col { + buf[i] = b ^ byte(k) + } + score := ScoreEnglish(buf) + if score > bestScore { + bestScore = score + bestKey = byte(k) + } + } + key[pos] = bestKey + } + return key +} + +func XORRepeating(in, key []byte) []byte { + if len(key) == 0 { + return append([]byte(nil), in...) + } + out := make([]byte, len(in)) + for i, b := range in { + out[i] = b ^ key[i%len(key)] + } + return out +} + +func hamming(a, b []byte) int { + n := len(a) + if len(b) < n { + n = len(b) + } + out := 0 + for i := 0; i < n; i++ { + x := a[i] ^ b[i] + for x != 0 { + out++ + x &= x - 1 + } + } + return out +} + +// TryEverything tries a handful of simple decodes and scores them. +func TryEverything(in []byte) []map[string]any { + candidates := []map[string]any{ + {"name": "input", "text": string(in), "score": ScoreEnglish(in)}, + {"name": "rot13", "text": string(Caesar(in, 13)), "score": ScoreEnglish(Caesar(in, 13))}, + {"name": "atbash", "text": string(Atbash(in)), "score": ScoreEnglish(Atbash(in))}, + } + if b, err := Base64Decode(in); err == nil && len(b) > 0 { + candidates = append(candidates, map[string]any{"name": "base64.decode", "text": string(b), "score": ScoreEnglish(b)}) + } + if b, err := HexDecode(in); err == nil && len(b) > 0 { + candidates = append(candidates, map[string]any{"name": "hex.decode", "text": string(b), "score": ScoreEnglish(b)}) + } + if b, err := URLDecode(in); err == nil && len(b) > 0 { + candidates = append(candidates, map[string]any{"name": "url.decode", "text": string(b), "score": ScoreEnglish(b)}) + } + // Top 3 XOR 1-byte candidates + xr := BruteXORByte(in) + for i := 0; i < 3 && i < len(xr); i++ { + candidates = append(candidates, map[string]any{ + "name": "xor1:" + xr[i]["keyHex"].(string), + "text": xr[i]["text"].(string), + "score": xr[i]["score"].(float64), + }) + } + // Top repeating-key XOR candidates. + rx := RepeatingXORGuess(in, 2, 40, 3) + for i := 0; i < 3 && i < len(rx); i++ { + candidates = append(candidates, map[string]any{ + "name": "xor-repeating:" + rx[i]["keyHex"].(string), + "keyHex": rx[i]["keyHex"].(string), + "keyText": rx[i]["keyText"].(string), + "text": rx[i]["text"].(string), + "score": rx[i]["score"].(float64), + }) + } + // Top 2 caesar + cz := BruteCaesar(in) + for i := 0; i < 2 && i < len(cz); i++ { + candidates = append(candidates, map[string]any{ + "name": "caesar:" + byteDec(byte(cz[i]["shift"].(int))), + "text": cz[i]["text"].(string), + "score": cz[i]["score"].(float64), + }) + } + // Sort + for i := 1; i < len(candidates); i++ { + for j := i; j > 0 && candidates[j-1]["score"].(float64) < candidates[j]["score"].(float64); j-- { + candidates[j-1], candidates[j] = candidates[j], candidates[j-1] + } + } + // Samples + for _, c := range candidates { + t, _ := c["text"].(string) + c["sample"] = previewText([]byte(t), 120) + } + return candidates +} + +func previewText(buf []byte, n int) string { + if n > len(buf) { + n = len(buf) + } + out := make([]byte, n) + for i := 0; i < n; i++ { + if buf[i] >= 0x20 && buf[i] <= 0x7e { + out[i] = buf[i] + } else if buf[i] == '\n' { + out[i] = ' ' + } else { + out[i] = '.' + } + } + return string(out) +} + +func byteHex(b byte) string { + const h = "0123456789abcdef" + return string([]byte{h[b>>4], h[b&0x0f]}) +} + +func bytesHex(buf []byte) string { + out := make([]byte, 0, len(buf)*2) + for _, b := range buf { + out = append(out, byteHex(b)...) + } + return string(out) +} + +func byteDec(b byte) string { + if b == 0 { + return "0" + } + out := [3]byte{} + i := 3 + for b > 0 { + i-- + out[i] = '0' + b%10 + b /= 10 + } + return string(out[i:]) +} diff --git a/internal/leviathan/transforms/transforms.go b/internal/leviathan/transforms/transforms.go new file mode 100644 index 0000000..f9344ab --- /dev/null +++ b/internal/leviathan/transforms/transforms.go @@ -0,0 +1,237 @@ +// Package transforms exposes chainable transform primitives. Each +// function takes an input []byte and options, and returns an output +// or an error. They are pure and do not mutate input. +package transforms + +import ( + "bytes" + "compress/flate" + "compress/gzip" + "compress/zlib" + "encoding/base64" + "encoding/hex" + "errors" + "io" + "net/url" +) + +const MaxInflateOutput = 64 << 20 + +// HexEncode returns ASCII hex of src. +func HexEncode(src []byte) ([]byte, error) { + return []byte(hex.EncodeToString(src)), nil +} + +// HexDecode parses ASCII hex into bytes. Whitespace is ignored. +func HexDecode(src []byte) ([]byte, error) { + stripped := stripHex(src) + out := make([]byte, len(stripped)/2) + _, err := hex.Decode(out, stripped) + return out, err +} + +func stripHex(s []byte) []byte { + out := make([]byte, 0, len(s)) + for _, b := range s { + if b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == ',' || b == ':' { + continue + } + if b == '0' && len(out) == 0 { + continue + } + out = append(out, b) + } + return out +} + +// Base64Encode returns the standard Base64 of src. +func Base64Encode(src []byte) ([]byte, error) { + return []byte(base64.StdEncoding.EncodeToString(src)), nil +} + +// Base64Decode accepts standard or URL-safe Base64, with or without padding. +func Base64Decode(src []byte) ([]byte, error) { + s := string(bytes.TrimSpace(src)) + attempts := []*base64.Encoding{ + base64.StdEncoding, + base64.URLEncoding, + base64.RawStdEncoding, + base64.RawURLEncoding, + } + for _, enc := range attempts { + if b, err := enc.DecodeString(s); err == nil { + return b, nil + } + } + return nil, errors.New("not valid base64") +} + +// URLEncode applies percent-encoding. +func URLEncode(src []byte) ([]byte, error) { + return []byte(url.QueryEscape(string(src))), nil +} + +// URLDecode reverses percent-encoding. +func URLDecode(src []byte) ([]byte, error) { + v, err := url.QueryUnescape(string(src)) + if err != nil { + return nil, err + } + return []byte(v), nil +} + +// XOR returns src^key, cycling key. +func XOR(src, key []byte) ([]byte, error) { + if len(key) == 0 { + return nil, errors.New("empty key") + } + out := make([]byte, len(src)) + for i := range src { + out[i] = src[i] ^ key[i%len(key)] + } + return out, nil +} + +// InflateGzip, InflateZlib, InflateRaw attempt to decompress. +func InflateGzip(src []byte) ([]byte, error) { + r, err := gzip.NewReader(bytes.NewReader(src)) + if err != nil { + return nil, err + } + defer r.Close() + return readLimitedInflate(r) +} + +func InflateZlib(src []byte) ([]byte, error) { + r, err := zlib.NewReader(bytes.NewReader(src)) + if err != nil { + return nil, err + } + defer r.Close() + return readLimitedInflate(r) +} + +func InflateRaw(src []byte) ([]byte, error) { + r := flate.NewReader(bytes.NewReader(src)) + defer r.Close() + return readLimitedInflate(r) +} + +func readLimitedInflate(r io.Reader) ([]byte, error) { + lr := io.LimitReader(r, MaxInflateOutput+1) + out, err := io.ReadAll(lr) + if err != nil { + return nil, err + } + if len(out) > MaxInflateOutput { + return nil, errors.New("decompressed output exceeds 64 MiB limit") + } + return out, nil +} + +// TryInflate attempts gzip, zlib, and raw deflate in order. +func TryInflate(src []byte) (out []byte, kind string, err error) { + if b, e := InflateGzip(src); e == nil { + return b, "gzip", nil + } + if b, e := InflateZlib(src); e == nil { + return b, "zlib", nil + } + if b, e := InflateRaw(src); e == nil { + return b, "raw", nil + } + return nil, "", errors.New("no decompression succeeded") +} + +// Step is one entry in a transform pipeline. +type Step struct { + Op string // hex.encode, hex.decode, base64.encode, base64.decode, + // xor, inflate, url.encode, url.decode, caesar, rot13, atbash, + // vigenere.enc, vigenere.dec + Arg string // ASCII parameter (e.g. key for XOR/Vigenere, shift for Caesar) +} + +// ApplyPipeline runs steps left-to-right. Intermediate results are +// returned for inspection. +func ApplyPipeline(input []byte, steps []Step) ([][]byte, error) { + results := make([][]byte, 0, len(steps)+1) + cur := append([]byte(nil), input...) + results = append(results, cur) + for _, s := range steps { + out, err := Apply(cur, s) + if err != nil { + return results, err + } + cur = out + results = append(results, cur) + } + return results, nil +} + +// Apply one step. +func Apply(in []byte, s Step) ([]byte, error) { + switch s.Op { + case "hex.encode": + return HexEncode(in) + case "hex.decode": + return HexDecode(in) + case "base64.encode": + return Base64Encode(in) + case "base64.decode": + return Base64Decode(in) + case "url.encode": + return URLEncode(in) + case "url.decode": + return URLDecode(in) + case "xor": + key, err := parseKey(s.Arg) + if err != nil { + return nil, err + } + return XOR(in, key) + case "inflate": + switch s.Arg { + case "gzip": + return InflateGzip(in) + case "zlib": + return InflateZlib(in) + case "raw": + return InflateRaw(in) + default: + out, _, err := TryInflate(in) + return out, err + } + case "caesar": + n := 13 + if s.Arg != "" { + var v int + for _, c := range s.Arg { + if c >= '0' && c <= '9' { + v = v*10 + int(c-'0') + } + } + n = v + } + return Caesar(in, n), nil + case "rot13": + return Caesar(in, 13), nil + case "atbash": + return Atbash(in), nil + case "vigenere.enc": + return Vigenere(in, []byte(s.Arg), true), nil + case "vigenere.dec": + return Vigenere(in, []byte(s.Arg), false), nil + } + return nil, errors.New("unknown op: " + s.Op) +} + +// parseKey accepts either literal string (ASCII) or hex-prefixed "hex:..". +func parseKey(arg string) ([]byte, error) { + if len(arg) > 4 && arg[:4] == "hex:" { + return hex.DecodeString(arg[4:]) + } + if arg == "" { + return nil, errors.New("empty key") + } + return []byte(arg), nil +} diff --git a/internal/recording/recorder.go b/internal/recording/recorder.go index f8ecbe5..b54a2cf 100644 --- a/internal/recording/recorder.go +++ b/internal/recording/recorder.go @@ -9,6 +9,7 @@ package recording import ( + "bufio" "bytes" "encoding/json" "fmt" @@ -33,6 +34,9 @@ const ( MaxRecordingBytes = int64(250 * 1024 * 1024) // MaxPlaybackBytes bounds the file size loaded for playback. MaxPlaybackBytes = int64(100 * 1024 * 1024) + // MaxPlaybackLineBytes bounds one asciicast line during streaming playback + // parsing. Terminal events should stay well below this in normal use. + MaxPlaybackLineBytes = 8 * 1024 * 1024 ) // Header is the asciicast v2 file header (first line). @@ -298,26 +302,30 @@ func (s *Store) getWithLimit(id string, maxBytes int64) (*Header, []Event, error if maxBytes > 0 && info.Size() > maxBytes { return nil, nil, fmt.Errorf("recording: file is too large for playback (max %d bytes)", maxBytes) } - data, err := os.ReadFile(path) + file, err := os.Open(path) if err != nil { - return nil, nil, fmt.Errorf("recording: failed to read file: %w", err) - } - - lines := strings.Split(strings.TrimSpace(string(data)), "\n") - if len(lines) < 1 { - return nil, nil, fmt.Errorf("recording: empty file") + return nil, nil, fmt.Errorf("recording: failed to open file: %w", err) } + defer file.Close() // Parse header (first line). + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), MaxPlaybackLineBytes) + if !scanner.Scan() { + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("recording: failed to scan header: %w", err) + } + return nil, nil, fmt.Errorf("recording: empty file") + } var hdr Header - if err := json.Unmarshal([]byte(lines[0]), &hdr); err != nil { + if err := json.Unmarshal([]byte(strings.TrimSpace(scanner.Text())), &hdr); err != nil { return nil, nil, fmt.Errorf("recording: invalid header: %w", err) } // Parse events (remaining lines). - events := make([]Event, 0, len(lines)-1) - for _, line := range lines[1:] { - line = strings.TrimSpace(line) + events := make([]Event, 0) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) if line == "" { continue } @@ -347,6 +355,9 @@ func (s *Store) getWithLimit(id string, maxBytes int64) (*Header, []Event, error Data: evtData, }) } + if err := scanner.Err(); err != nil { + return nil, nil, fmt.Errorf("recording: failed to scan events: %w", err) + } return &hdr, events, nil } diff --git a/internal/secrets/secrets.go b/internal/secrets/secrets.go index c722fb7..73ae149 100644 --- a/internal/secrets/secrets.go +++ b/internal/secrets/secrets.go @@ -28,6 +28,8 @@ const ( argonKeyLen = 32 saltLen = 32 + + maxVaultFileBytes = 10 * 1024 * 1024 ) // SecretStore manages encrypted credential storage. @@ -331,6 +333,16 @@ func (s *SecretStore) save() error { // load reads and unmarshals the vault from disc. func (s *SecretStore) load() (*secretVault, error) { + info, err := os.Stat(s.vaultPath()) + if err != nil { + return nil, fmt.Errorf("failed to stat vault file: %w", err) + } + if info.IsDir() { + return nil, errors.New("vault path is a directory") + } + if info.Size() > maxVaultFileBytes { + return nil, fmt.Errorf("vault file is too large: %d bytes exceeds %d bytes", info.Size(), maxVaultFileBytes) + } data, err := os.ReadFile(s.vaultPath()) if err != nil { return nil, fmt.Errorf("failed to read vault file: %w", err) diff --git a/internal/secrets/secrets_test.go b/internal/secrets/secrets_test.go index 6ce9b96..8ea956f 100644 --- a/internal/secrets/secrets_test.go +++ b/internal/secrets/secrets_test.go @@ -1,7 +1,9 @@ package secrets import ( + "os" "path/filepath" + "strings" "testing" ) @@ -267,6 +269,30 @@ func TestVaultPath(t *testing.T) { } } +func TestSecretStoreLoadRejectsOversizedVault(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, vaultFileName) + file, err := os.Create(path) + if err != nil { + t.Fatalf("Create() failed: %v", err) + } + if err := file.Truncate(maxVaultFileBytes + 1); err != nil { + t.Fatalf("Truncate() failed: %v", err) + } + if err := file.Close(); err != nil { + t.Fatalf("Close() failed: %v", err) + } + + store := NewSecretStore(dir) + _, err = store.load() + if err == nil { + t.Fatal("expected oversized vault to be rejected") + } + if !strings.Contains(err.Error(), "vault file is too large") { + t.Fatalf("unexpected error: %v", err) + } +} + // --------------------------------------------------------------------------- // Redaction helpers // --------------------------------------------------------------------------- diff --git a/internal/sessions/manager.go b/internal/sessions/manager.go index 8ba6a28..cc9f991 100644 --- a/internal/sessions/manager.go +++ b/internal/sessions/manager.go @@ -32,6 +32,8 @@ import ( "github.com/andydixon/nexus/internal/vncclient" ) +const maxVNCTunnelPrivateKeyBytes = 1 << 20 + // Session wraps a protocol client with metadata and state. type Session struct { Info models.SessionInfo @@ -1031,6 +1033,16 @@ func (m *Manager) connectVNC(sess *Session, getSecret func(prompt string) (strin cfg.SSHPassword = password case "publickey": if vncConf.SSHPrivateKey != "" { + info, err := os.Stat(vncConf.SSHPrivateKey) + if err != nil { + return fmt.Errorf("reading SSH key metadata: %w", err) + } + if info.IsDir() { + return fmt.Errorf("SSH key path is a directory") + } + if info.Size() > maxVNCTunnelPrivateKeyBytes { + return fmt.Errorf("SSH key is too large: %d bytes exceeds %d bytes", info.Size(), maxVNCTunnelPrivateKeyBytes) + } keyData, err := os.ReadFile(vncConf.SSHPrivateKey) if err != nil { return fmt.Errorf("reading SSH key: %w", err) diff --git a/internal/sshclient/sshclient.go b/internal/sshclient/sshclient.go index 84d5c2c..aaa721d 100644 --- a/internal/sshclient/sshclient.go +++ b/internal/sshclient/sshclient.go @@ -20,6 +20,8 @@ import ( "golang.org/x/net/proxy" ) +const maxSSHPrivateKeyBytes = 1 << 20 + // Client manages a single SSH session lifecycle. // It handles connection establishment, authentication, PTY allocation, // terminal resize propagation, port forwarding, and clean shutdown. @@ -664,6 +666,16 @@ func jumpHostClientConfig(jump models.JumpHost, auth []ssh.AuthMethod, finalConf // parsePrivateKey loads and parses an SSH private key file. If the key is // encrypted and getPassphrase is provided, it will attempt decryption. func (c *Client) parsePrivateKey(path string, getPassphrase func() (string, error)) (ssh.Signer, error) { + info, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("unable to stat key file: %w", err) + } + if info.IsDir() { + return nil, fmt.Errorf("key path is a directory") + } + if info.Size() > maxSSHPrivateKeyBytes { + return nil, fmt.Errorf("key file is too large: %d bytes exceeds %d bytes", info.Size(), maxSSHPrivateKeyBytes) + } keyData, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("unable to read key file: %w", err) diff --git a/internal/updater/updater.go b/internal/updater/updater.go index 1cc673d..0f9a4a1 100644 --- a/internal/updater/updater.go +++ b/internal/updater/updater.go @@ -25,6 +25,10 @@ const ( // requestTimeout limits the HTTP request duration. requestTimeout = 10 * time.Second + + // maxUpdateResponseBytes bounds release and manifest responses before + // parsing them into memory. + maxUpdateResponseBytes = 1 << 20 ) // Release describes a GitHub release with the fields we care about. @@ -135,8 +139,13 @@ func (c *Checker) Check(ctx context.Context) (*UpdateInfo, error) { return nil, fmt.Errorf("updater: GitHub API returned %d", resp.StatusCode) } + body, err := readLimitedUpdateBody(resp.Body) + if err != nil { + return nil, fmt.Errorf("updater: failed to read response: %w", err) + } + var release Release - if err := json.NewDecoder(resp.Body).Decode(&release); err != nil { + if err := json.Unmarshal(body, &release); err != nil { return nil, fmt.Errorf("updater: failed to decode response: %w", err) } @@ -186,7 +195,7 @@ func (c *URLChecker) Check(ctx context.Context) (*UpdateInfo, error) { return nil, fmt.Errorf("updater: update server returned %d", resp.StatusCode) } - body, err := io.ReadAll(resp.Body) + body, err := readLimitedUpdateBody(resp.Body) if err != nil { return nil, fmt.Errorf("updater: failed to read response: %w", err) } @@ -215,6 +224,17 @@ func normaliseVersion(v string) string { return v } +func readLimitedUpdateBody(r io.Reader) ([]byte, error) { + body, err := io.ReadAll(io.LimitReader(r, maxUpdateResponseBytes+1)) + if err != nil { + return nil, err + } + if len(body) > maxUpdateResponseBytes { + return nil, fmt.Errorf("response exceeds %d bytes", maxUpdateResponseBytes) + } + return body, nil +} + func parseManifest(body []byte, current string) (*UpdateInfo, bool) { var manifest Manifest if err := json.Unmarshal(body, &manifest); err != nil { diff --git a/internal/updater/updater_test.go b/internal/updater/updater_test.go index bf55f7a..f26df66 100644 --- a/internal/updater/updater_test.go +++ b/internal/updater/updater_test.go @@ -37,9 +37,9 @@ func TestIsNewer(t *testing.T) { {"1.0.1", "1.0.0", true}, {"1.0.0", "1.0.0", false}, {"0.9.0", "1.0.0", false}, - {"1.0.0", "dev", false}, // dev builds ignored - {"1.0.0", "", false}, // empty current ignored - {"1.2", "1.1.0", true}, // fewer parts + {"1.0.0", "dev", false}, // dev builds ignored + {"1.0.0", "", false}, // empty current ignored + {"1.2", "1.1.0", true}, // fewer parts {"1.0.0", "1.0.0.1", false}, } for _, tt := range tests { @@ -104,3 +104,20 @@ func TestURLCheckerPlainText(t *testing.T) { t.Fatalf("expected update to be available") } } + +func TestURLCheckerRejectsOversizedManifest(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte(strings.Repeat("x", maxUpdateResponseBytes+1))) + })) + defer server.Close() + + checker := NewURLChecker(server.URL, "1.9.0") + _, err := checker.Check(context.Background()) + if err == nil { + t.Fatal("expected oversized manifest to be rejected") + } + if !strings.Contains(err.Error(), "response exceeds") { + t.Fatalf("unexpected error: %v", err) + } +}