diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3c07e78..7a307f7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -124,6 +124,14 @@ releases the lock between batches). This is the only lock in the data path. 3. Nodes are never deleted; a `NodeID` once observed stays valid. 4. Workers share nothing mutable except the bounded channels and atomic progress counters. +5. No goroutine ever takes `RLock` while already holding it. Go's + `RWMutex` blocks new readers once a writer waits, so a recursive read + lock can deadlock against the scan controller. The TUI takes the lock + once per entry point (`View`, `handleKey`, mouse handlers); helpers they + call assume it is held. Background commands (filter, duplicates, + snapshot save) take their own lock on their own goroutine. +6. After a scan finishes, the UI goroutine may become the writer (move to + trash updates aggregates under `Lock`). Trash is refused while scanning. ## Hard links and sizes @@ -170,15 +178,25 @@ all except for a single `scanDoneMsg`. The UI polls at a fixed rate. `View()` is called by Bubble Tea after *every* `Update`. The model therefore caches the rendered frame and only repaints when something visible changed. +Expensive per-directory work (sorting children by size) is cached per data +version. During a scan the version advances at most every 400 ms, so the +layout does not jitter at frame rate even though labels and counters update +at the 10 Hz refresh. + ## Rendering: the canvas All drawing goes to an in-memory `Canvas` of cells (grapheme, fg, bg, attrs). Views and modals paint into the canvas; the canvas is serialised to a string with minimal SGR changes. Bubble Tea's renderer diffs frames and downsamples true-colour to the detected profile (256 / 16 / none), so the drawing code -uses RGB throughout. Selection never relies on colour alone: the selected -rectangle uses a double-line border, bold reversed title and a status-line -description. +uses RGB throughout. For 256-colour terminals the canvas quantizes colours +itself (redmean distance over the 6×6×6 cube and grey ramp, never the +palette-dependent first 16 entries): the stock conversion maps dark tints to +saturated olive/navy and destroys the shading. `--color` overrides +detection (useful over SSH, where `COLORTERM` is often not forwarded). +`NO_COLOR` selects the `mono` theme, which relies on attributes and glyphs +only. Selection never relies on colour alone: the selected rectangle uses a +double-line border, bold reversed title and a status-line description. ## Treemap algorithm @@ -230,8 +248,9 @@ when content does not fit. ## Selection persistence -Selection is a `NodeID` (plus, for the synthetic LOD group, the group's parent -ID and a group marker). After any relayout the renderer looks the ID up in the +Selection is a `NodeID` (or, for the synthetic LOD group, `-1 - parentID`). +Until the user moves the selection it follows the largest block, which +changes while a scan runs; after that it is sticky. After any relayout the renderer looks the ID up in the new rectangle list. If it is absent (filtered out, collapsed into a group, zoom changed) selection falls back to: the group containing it, then its nearest visible ancestor, then the largest block. diff --git a/CHANGELOG.md b/CHANGELOG.md index 33d0433..5ff893e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,3 +6,13 @@ - Non-interactive reports: `--top`, `--largest-files`, `--json`, `--csv`, `--duplicates`, `--no-ui`. - Versioned, checksummed snapshot format; `--compare`. +- Interactive TUI (Bubble Tea v2): live squarified treemap with nested + previews and level-of-detail grouping, spatial arrow navigation, zoom, + breadcrumbs, responsive layouts, info/help/confirm/save modals, mouse. +- Views: directory list, file types, top lists, scan info & errors, + duplicates, snapshot changes. +- Search/filter language with live treemap filtering. +- Themes (default, dark, high-contrast, mono), `--color` override, + perceptual 256-colour quantizer, NO_COLOR support. +- Move to trash (freedesktop / macOS / Windows), `--read-only`. +- `treegen` synthetic tree generator; benchmarks in PERFORMANCE.md. diff --git a/PERFORMANCE.md b/PERFORMANCE.md new file mode 100644 index 0000000..bb98813 --- /dev/null +++ b/PERFORMANCE.md @@ -0,0 +1,93 @@ +# Performance + +All numbers below were measured, not estimated. Re-run them on your own +hardware before relying on them; filesystem caches dominate scan timings. + +## Test machine (2026-10-02) + +| | | +|---|---| +| CPU | Intel Core i7-7700 @ 3.60 GHz, 4 cores / 8 threads | +| RAM | 32 GiB | +| OS | Linux 7.0, Go 1.27.1 | +| Filesystem | ext4 on Linux software RAID (`/dev/md2`) over two 7200 rpm SATA HDDs | + +## Scanner + +Dataset: `~/go/pkg/mod` — 388,307 files + 65,518 directories (458,510 +inventory nodes), 10.5 GiB allocated. **Warm cache only**: dropping the page +cache needs root, which was not available, so these measure CPU and syscall +cost, not disk seeks. Cold-cache behaviour on HDDs will be dominated by seek +time; expect fewer workers to be optimal there. + +`mapsize --no-ui --workers N ~/go/pkg/mod`, wall time, 3 runs each +(`/usr/bin/time`): + +| workers | run 1 | run 2 | run 3 | entries/s (best) | peak RSS | +|---:|---:|---:|---:|---:|---:| +| 1 | 1.77 s | 1.67 s | 1.65 s | ~278,000 | 117 MB | +| 2 | 1.06 s | 0.99 s | 1.00 s | ~463,000 | 120 MB | +| 4 | 0.59 s | 0.59 s | 0.62 s | ~777,000 | 117 MB | +| 8 (balanced default) | 0.48 s | 0.50 s | 0.51 s | ~955,000 | 121 MB | +| 16 | 0.45 s | 0.44 s | 0.43 s | ~1,066,000 | 121 MB | +| 32 | 0.43 s | 0.44 s | 0.42 s | ~1,092,000 | 122 MB | + +For comparison, `du -s` on the same tree (warm): 1.59 s, 34 MB RSS. + +CPU profile at 8 workers: ~60 % of samples in `statx`/`getdents` syscalls; +path reconstruction for job dispatch ~7 %; aggregation is negligible. + +### Memory + +`go test ./internal/scan -run TestMemoryPerNode` with +`MAPSIZE_MEM_ROOT=~/go/pkg/mod`: 458,510 nodes, 69.1 MiB live heap, +**~158 bytes per node** (node record ~112 bytes plus name storage and +directory side tables). Extrapolated, 10 M entries need ~1.6 GiB. + +## UI + +`go test ./internal/tui -bench Big` on a synthetic in-memory tree of +1,008,201 nodes (200 directories × 40 subdirectories × 100 files, plus one +flat directory with 200,000 files), 200×60 terminal: + +| operation | time per op | +|---|---:| +| cold frame at root (sort children, LOD, layout incl. nested previews, paint) | 33.8 ms | +| resize to a new size (layout + paint, child lists cached) | 2.0 ms | +| arrow-key move + re-render | 1.5 ms | +| zoomed into the 200,000-entry directory, cold | 29.5 ms | +| render of a small tree (2.4 K nodes) at 200×60 | 0.66 ms | +| resize relayout of the small tree | 0.45 ms | + +During a scan the size caches are invalidated at most every 400 ms, so a +cold frame (worst case above ~34 ms) costs under 10 % of one core; other +frames reuse caches. Resize events only record the new size; layout happens +once per rendered frame, so a resize storm cannot queue up work. + +### Geometry + +`go test ./internal/treemap -bench .`: + +| operation | time | +|---|---:| +| squarified layout, 300 items | 89 µs | +| spatial neighbour lookup among 300 blocks | 2.1 µs | + +### Filtering (1,008,201 nodes) + +| query | time | +|---|---:| +| `size > 1MB` | 31 ms | +| `*.dat AND size > 500k` | 119 ms | +| `path contains sub3` (reconstructs every path) | 244 ms | + +Filters run on a background goroutine with cancellation and a 120 ms +keystroke debounce; the UI never waits for them. + +## Known costs / future work + +* Path predicates rebuild each node's path; an incremental evaluation along + the tree would make them ~10× cheaper. +* Node records could shrink (owner/group/mode/link count are only needed for + display and some filters) if memory on 10 M+ entry trees matters. +* Cold-cache HDD benchmarks still need to be run with root access. diff --git a/README.md b/README.md index 0983a0b..bb9d612 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,63 @@ runs over SSH. ## Status -Under active development — see [ROADMAP.md](ROADMAP.md) for what works today. +Milestones 0.1–0.8 are implemented; 0.9 (hardening) is in progress. See +[ROADMAP.md](ROADMAP.md) for detail and [PERFORMANCE.md](PERFORMANCE.md) for +measured numbers. + +## What it does + +* **Live treemap.** A squarified, space-filling treemap fills the terminal + and grows while the scan runs. Directories show nested previews of their + contents; colours are file categories (video, archives, disk images, code, + …). Items too small to draw collapse into a "N smaller items" block until + you zoom in. +* **Spatial navigation.** Arrow keys move to the block that is actually to + the left/right/above/below. Enter inspects, Space zooms in, Backspace zooms + out, Home returns to the root. The selection survives rescans, resizes and + filtering. +* **Resizes properly.** Every resize recomputes the layout for the new + geometry (it never stretches the old one); large terminals get a side + panel, small ones a compact layout, tiny ones a polite message. +* **Search and filter** with a small query language — the treemap shows only + matching data: + `ubuntu` · `*.iso` · `size > 5GB` · `ext IN (iso,qcow2,vmdk)` · + `age > 365d` · `path contains cache AND NOT type = dir` · `owner = andy` +* **Investigation views** (Tab): directory list, file types, top lists + (largest files/dirs, oldest/newest large files, most entries, sparse and + hard-linked files), scan information with every error, duplicates. +* **Honest numbers.** Allocated vs logical size, hard links counted once, + sparse files flagged, permission errors and skipped areas always shown; + incomplete totals are labelled as such. +* **Snapshots**: save a scan, reopen it instantly, compare two — the treemap + colours growth and shrinkage. +* **Safe with hostile filenames**: control characters and escape sequences + in names are displayed escaped and never reach the terminal. + +## Keys + +| Key | Action | +|---|---| +| ↑ ↓ ← → / hjkl | move spatially through the map | +| Enter | item information | +| Space / → | zoom into directory | +| Backspace / ← | zoom out | +| Home | scan root | +| / | search / filter (Esc clears) | +| Tab, Shift+Tab, 1–7 | switch views | +| x, g, e | file types, top lists, scan info & errors | +| D | find duplicates | +| a | toggle allocated / logical size | +| c, o | copy path, reveal in file manager | +| d | move to trash (asks; disabled with `--read-only`) | +| s, r | save snapshot, rescan | +| T | cycle theme | +| ? | help | +| Ctrl+C | cancel scan, then quit | +| q | quit | + +Mouse: click selects, double-click zooms, right-click inspects, wheel zooms +the map or scrolls lists. Everything works without a mouse (`--no-mouse`). ## Install @@ -72,13 +128,30 @@ Optional JSON file at `$XDG_CONFIG_HOME/mapsize/config.json` } ``` +### Colour + +Colour support is auto-detected; `--color truecolor|256|16|none` overrides +it (handy over SSH, where `COLORTERM` is often not forwarded). Themes: +`default`, `dark`, `high-contrast`, `mono` (`--theme`, or `T` to cycle). +`NO_COLOR` is respected. On the Linux console, or with `MAPSIZE_ASCII=1`, +box drawing falls back to ASCII. + ### Read-only mode `--read-only` disables every modifying action and shows `READ ONLY` in the -status bar. It does not make mapsize a forensic acquisition tool: scanning +header. (Saving a snapshot file is still allowed; it writes only where you +tell it to.) It does not make mapsize a forensic acquisition tool: scanning uses ordinary filesystem APIs, and duplicate detection reads file content (which may update access times). +### Deletion + +`d` moves the selected item to the system trash (freedesktop.org trash on +Linux, `~/.Trash` on macOS, the Recycle Bin on Windows) after confirmation, +using native file operations — never a shell. Permanent deletion is not +offered. Trash is unavailable while scanning, for snapshots, and in +read-only mode. + ## Documentation * [ARCHITECTURE.md](ARCHITECTURE.md) — design and invariants diff --git a/ROADMAP.md b/ROADMAP.md index f07fe40..4fa3d32 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -3,45 +3,48 @@ Status markers: `[x]` done, `[~]` partial, `[ ]` not started. ## 0.1 — inventory engine -- [ ] Go project skeleton, Makefile, CI -- [ ] portable filesystem scanner, bounded worker pool, no-deadlock scheduler -- [ ] inventory node model, incremental aggregation -- [ ] cancellation, progress counters, error accounting -- [ ] CLI: `--top`, `--json`, `--csv`, `--no-ui` +- [x] Go project skeleton, Makefile, CI +- [x] portable filesystem scanner, bounded worker pool, no-deadlock scheduler +- [x] inventory node model, incremental aggregation +- [x] cancellation, progress counters, error accounting +- [x] CLI: `--top`, `--json`, `--csv`, `--no-ui` ## 0.2 — TUI shell -- [ ] full-screen Bubble Tea v2 app, canvas renderer, frame cache -- [ ] responsive layout (large / medium / small / too-small) -- [ ] resize handling, status bar, progressive display, directory list +- [x] full-screen Bubble Tea v2 app, canvas renderer, frame cache +- [x] responsive layout (large / medium / small / too-small) +- [x] resize handling, status bar, progressive display, directory list ## 0.3 — treemap -- [ ] squarified layout, edge-snapped cell rectangles -- [ ] renderer with borders, nested preview, adaptive labels -- [ ] level-of-detail grouping, category colouring +- [x] squarified layout, edge-snapped cell rectangles +- [x] renderer with borders, nested preview, adaptive labels +- [x] level-of-detail grouping, category colouring ## 0.4 — interaction -- [ ] spatial arrow navigation, stable NodeID selection -- [ ] info modal (scrollable, resize-aware), zoom, breadcrumbs, mouse +- [x] spatial arrow navigation, stable NodeID selection +- [x] info modal (scrollable, resize-aware), zoom, breadcrumbs, mouse ## 0.5 — investigation -- [ ] search & filter language (AND/OR/NOT, parentheses, units, ages) -- [ ] extension statistics, largest files / dirs, error view +- [x] search & filter language (AND/OR/NOT, parentheses, units, ages) +- [x] extension statistics, largest files / dirs, error view ## 0.6 — snapshots -- [ ] versioned binary snapshot format, save/load, comparison, growth treemap +- [x] versioned binary snapshot format, save/load, comparison, growth treemap ## 0.7 — duplicates -- [ ] staged duplicate detection (size → sampled hash → SHA-256), view +- [x] staged duplicate detection (size → sampled hash → SHA-256), view ## 0.8 — operations -- [ ] read-only mode, JSON/CSV export -- [ ] move-to-trash (freedesktop, macOS ~/.Trash, Windows recycle bin) +- [x] read-only mode, JSON/CSV export +- [x] move-to-trash: freedesktop trash (tested on Linux); macOS ~/.Trash and Windows Recycle Bin implemented but not yet tested on those systems ## 0.9 — hardening (ongoing) -- [ ] profiling & benchmarks (see PERFORMANCE.md) +- [~] profiling & benchmarks (warm-cache only so far) (see PERFORMANCE.md) - [ ] memory reduction (name arena) if profiles justify it - [ ] Windows allocated-size via GetCompressedFileSizeW (opt-in) - [ ] configurable keybindings +- [ ] cold-cache HDD scan benchmarks (need root to drop caches) +- [ ] incremental path-predicate evaluation for filters +- [ ] permanent deletion (deliberately not offered yet; trash only) ## 1.0 - [ ] stable snapshot format v1 guarantee, packaged releases diff --git a/internal/platform/ops_linux_test.go b/internal/platform/ops_linux_test.go new file mode 100644 index 0000000..d86712a --- /dev/null +++ b/internal/platform/ops_linux_test.go @@ -0,0 +1,44 @@ +package platform + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestTrashFreedesktop(t *testing.T) { + data := t.TempDir() + t.Setenv("XDG_DATA_HOME", data) + dir := filepath.Join(data, "work") + os.Mkdir(dir, 0o755) + victim := filepath.Join(dir, "weird name ;rm -rf ~;") + if err := os.WriteFile(victim, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := Trash(victim); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(victim); !os.IsNotExist(err) { + t.Fatal("file still present") + } + trashed := filepath.Join(data, "Trash", "files", "weird name ;rm -rf ~;") + if _, err := os.Lstat(trashed); err != nil { + t.Fatalf("not in trash: %v", err) + } + info, err := os.ReadFile(filepath.Join(data, "Trash", "info", "weird name ;rm -rf ~;.trashinfo")) + if err != nil || !strings.Contains(string(info), "Path="+strings.ReplaceAll(dir, " ", "%20")) { + t.Fatalf("bad trashinfo: %q %v", info, err) + } + // A second item with the same name gets a distinct trash name. + os.WriteFile(victim, []byte("y"), 0o644) + if err := Trash(victim); err != nil { + t.Fatal(err) + } + if _, err := os.Lstat(trashed + ".2"); err != nil { + t.Fatal("collision not handled") + } + if err := Trash("relative/path"); err == nil { + t.Fatal("relative paths must be refused") + } +} diff --git a/internal/scan/mem_test.go b/internal/scan/mem_test.go new file mode 100644 index 0000000..3ecef46 --- /dev/null +++ b/internal/scan/mem_test.go @@ -0,0 +1,35 @@ +package scan + +import ( + "context" + "os" + "runtime" + "testing" + "time" +) + +// TestMemoryPerNode reports heap use per inventory node for a real tree. +// Set MAPSIZE_MEM_ROOT to a directory to run it. +func TestMemoryPerNode(t *testing.T) { + root := os.Getenv("MAPSIZE_MEM_ROOT") + if root == "" { + t.Skip("set MAPSIZE_MEM_ROOT") + } + var before, after runtime.MemStats + runtime.GC() + runtime.ReadMemStats(&before) + start := time.Now() + s, err := Start(context.Background(), root, Options{}) + if err != nil { + t.Fatal(err) + } + <-s.Done() + el := time.Since(start) + runtime.GC() + runtime.ReadMemStats(&after) + n := s.Tree.Len() + heap := after.HeapAlloc - before.HeapAlloc + t.Logf("nodes=%d scan=%s heap=%.1f MiB (%.0f bytes/node)", n, el.Round(time.Millisecond), + float64(heap)/(1<<20), float64(heap)/float64(n)) + runtime.KeepAlive(s) +} diff --git a/internal/tui/bench_test.go b/internal/tui/bench_test.go new file mode 100644 index 0000000..b88c4b2 --- /dev/null +++ b/internal/tui/bench_test.go @@ -0,0 +1,114 @@ +package tui + +import ( + "fmt" + "math/rand" + "testing" + + "github.com/charmbracelet/colorprofile" + + "github.com/andydixon/mapsize/internal/inventory" +) + +// bigTree builds ~1M nodes in memory: 200 top-level dirs with nested +// subdirectories, plus one flat directory with 200k files. +func bigTree() *inventory.Tree { + r := rand.New(rand.NewSource(1)) + t := inventory.New("/big", inventory.KindDir) + file := func(p inventory.NodeID, i int) { + id := t.Add(p, fmt.Sprintf("f%d.dat", i), inventory.KindFile) + n := t.Node(id) + n.Size = int64(r.ExpFloat64() * 1e6) + n.Alloc = n.Size + n.Cat = inventory.Category(r.Intn(int(inventory.NumCategories))) + } + flat := t.Add(0, "flat", inventory.KindDir) + for i := range 200_000 { + file(flat, i) + } + for d := range 200 { + top := t.Add(0, fmt.Sprintf("dir%d", d), inventory.KindDir) + for s := range 40 { + sub := t.Add(top, fmt.Sprintf("sub%d", s), inventory.KindDir) + for f := range 100 { + file(sub, f) + } + } + } + t.Recompute() + return t +} + +var bigTreeCache *inventory.Tree + +func benchModel(b *testing.B) *Model { + if bigTreeCache == nil { + bigTreeCache = bigTree() + } + m := newModel(Options{}, nil, bigTreeCache, func() {}) + m.profile = colorprofile.TrueColor + send(m, size(200, 60)) + return m +} + +// Cold frame: sorting children, LOD selection, layout and paint. +func BenchmarkBigColdFrame(b *testing.B) { + m := benchModel(b) + b.ReportMetric(float64(m.tree.Len()), "nodes") + for b.Loop() { + m.invalidate() + m.View() + } +} + +// Resize: child lists cached; layout and paint rerun for the new size. +func BenchmarkBigResize(b *testing.B) { + m := benchModel(b) + m.View() + i := 0 + for b.Loop() { + send(m, size(150+i%100, 40+i%30)) + m.View() + i++ + } +} + +// Zoomed into a 200k-entry directory (cold child sort each time). +func BenchmarkBigFlatDirCold(b *testing.B) { + m := benchModel(b) + m.tree.Children(0, func(id inventory.NodeID, n *inventory.Node) { + if n.Name == "flat" { + m.zoom = id + } + }) + for b.Loop() { + m.invalidate() + m.View() + } +} + +// Arrow-key navigation including the re-render it triggers. +func BenchmarkBigNavigate(b *testing.B) { + m := benchModel(b) + m.View() + keys := []string{"right", "down", "left", "up"} + i := 0 + for b.Loop() { + send(m, key(keys[i%4])) + m.View() + i++ + } +} + +func BenchmarkBigFilter(b *testing.B) { + m := benchModel(b) + for _, q := range []string{"size > 1MB", "*.dat AND size > 500k", "path contains sub3"} { + b.Run(q, func(b *testing.B) { + m.search.ed.set(q) + m.queryChanged() + for b.Loop() { + m.applyFilter()() + } + }) + } +}