diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1 @@ +/target diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..4150d0d --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,589 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "mapsize" +version = "1.2.0" +dependencies = [ + "crossbeam-channel", + "crossterm", + "flate2", + "libc", + "regex", + "serde", + "serde_json", + "sha2", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..044ac2c --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "mapsize" +version = "1.2.0" +edition = "2021" +license = "GPL-3.0-or-later" +description = "Interactive terminal disk usage analyser (Rust port)" + +[dependencies] +libc = "0.2" +regex = "1" +crossbeam-channel = "0.5" +crossterm = "0.28" +flate2 = "1" +sha2 = "0.10" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +unicode-segmentation = "1" +unicode-width = "0.2" + +[profile.release] +lto = true +codegen-units = 1 +strip = true diff --git a/rust/README.md b/rust/README.md new file mode 100644 index 0000000..bb0b780 --- /dev/null +++ b/rust/README.md @@ -0,0 +1,95 @@ +# mapsize — Rust port + +A port of the Go implementation (repo root) to Rust, with the same CLI, TUI, +keys, themes and snapshot format. `.msz` snapshots are byte-compatible in +both directions. The code is Linux/Unix only; the Windows code paths were +not ported. + +``` +cd rust +cargo build --release # target/release/mapsize +cargo test --release # 69 tests (ported from the Go suites, fuzz tests excepted) +``` + +Module layout mirrors `internal/`: `inventory`, `scan`, `platform`, `treemap`, +`filter`, `snapshot` (+ `compare`), `duplicate`, `export`, `config`, +`textutil`, `tui`, `app`. The tree is shared as `Arc>` with the +same single-writer invariants as ARCHITECTURE.md; `cancel::Cancel` replaces +`context.Context`; crossterm replaces Bubble Tea as the event loop (the +canvas renderer is ported as-is, plus line-diffed frame output). + +## Known differences from Go + +* `--log`/`--log-level` are accepted but there is no debug log yet. +* A scanner thread panic restores the terminal and aborts the process instead + of returning an "internal error" message. +* The colour profile is detected from `NO_COLOR`/`TERM`/`COLORTERM` only (no + terminal query); `--color` still overrides. +* JSON export does not HTML-escape `<>&`; `started` is written in UTC. +* The filter lexer no longer hangs on queries containing bytes Go treats as + white space but never skips (e.g. `voilà`, whose UTF-8 contains 0xA0). +* Huge sizes saturate rather than wrap in filters and snapshot comparison. + +## Benchmarks + +Machine: Intel Core i7-7700 (4C/8T), 32 GiB, ext4 on md RAID over 7200 rpm +HDDs. Go 1.27.1, Rust 1.91.1 (`lto`, `codegen-units = 1`). The host was +shared and heavily loaded during the runs (load average 30–50 on 8 threads), +so absolute times are inflated 2–4× compared with PERFORMANCE.md. To keep the +comparison fair, Go and Rust runs were interleaved and the tables report +best-of (micro) or median and best (end-to-end). Re-run on an idle machine +before quoting absolute numbers: + +``` +go test -c -o /tmp/tui.test ./internal/tui && go test -c -o /tmp/treemap.test ./internal/treemap +(cd rust && cargo build --release --example bench) +rust/bench/micro.py /tmp/tui.test /tmp/treemap.test rust/target/release/examples/bench 5 +rust/bench/compare.py bin/mapsize rust/target/release/mapsize ~/go/pkg/mod 7 +``` + +### In-process (Go `go test -bench` suite vs `examples/bench.rs`) + +The tree is 1,008,201 synthetic nodes, rendered at 200×60. + +| benchmark | Go | Rust | speed-up | +|---|---:|---:|---:| +| treemap/Squarify(300) | 299.3 µs | 22.2 µs | 13.48× | +| treemap/Neighbour(300) | 5.0 µs | 2.8 µs | 1.76× | +| tui/BigColdFrame | 86.88 ms | 29.32 ms | 2.96× | +| tui/BigResize | 7.15 ms | 4.76 ms | 1.50× | +| tui/BigFlatDirCold | 86.33 ms | 22.90 ms | 3.77× | +| tui/BigNavigate | 4.85 ms | 4.07 ms | 1.19× | +| tui/BigFilter/size > 1MB | 78.62 ms | 59.51 ms | 1.32× | +| tui/BigFilter/*.dat AND size > 500k | 302.03 ms | 154.90 ms | 1.95× | +| tui/BigFilter/path contains sub3 | 337.79 ms | 136.85 ms | 2.47× | + +### End to end (`~/go/pkg/mod`: 388k files + 65k dirs, warm cache) + +| scenario | Go median | Rust median | Go best | Rust best | Go RSS | Rust RSS | speed-up | +|---|---:|---:|---:|---:|---:|---:|---:| +| scan, 1 workers | 4.68 s | 5.51 s | 4.34 s | 4.86 s | 114 MB | 75 MB | 0.85× | +| scan, 8 workers | 2.03 s | 2.03 s | 1.61 s | 1.42 s | 113 MB | 78 MB | 1.00× | +| scan, 32 workers | 3.09 s | 2.59 s | 2.85 s | 2.07 s | 116 MB | 89 MB | 1.19× | +| scan + save snapshot | 4.69 s | 3.60 s | 4.02 s | 3.29 s | 117 MB | 80 MB | 1.30× | +| load own snapshot | 1.20 s | 0.71 s | 0.71 s | 0.48 s | 85 MB | 78 MB | 1.69× | +| load Go snapshot | 1.03 s | 1.14 s | 0.84 s | 0.48 s | 85 MB | 78 MB | 0.90× | +| scan + JSON export | 6.10 s | 4.74 s | 5.44 s | 4.16 s | 152 MB | 79 MB | 1.29× | +| scan + top 50 largest files | 4.49 s | 4.31 s | 3.62 s | 3.65 s | 117 MB | 79 MB | 1.04× | + +Reading the numbers: + +* **Scanning** is syscall-bound (`statx`/`getdents` were about 60% of Go's + profile), so wall time is roughly at parity. Rust edges ahead at high + worker counts. The single-worker case is about 15% slower, but that run was + the noisiest of the set. +* **Memory**: peak RSS is 30–48% lower (78 MB vs 113 MB while scanning + 458k nodes). The JSON export streams without Go's per-node allocations + (79 MB vs 152 MB). +* **Snapshots**: loading is about 1.7× faster on the median. The two binaries + load each other's files. +* **UI**: cold frames are 3–3.8× faster, and squarified layout is about 13× + faster. Navigation and resize gain less (1.2–1.5×) because they mostly + reuse cached layout. +* **Filters**: 1.3–2.5× faster. This required caching lower-cased directory + paths in a `Vec` with reused buffers, and a `*.ext` fast path in the + `filepath.Match` port. diff --git a/rust/bench/compare.py b/rust/bench/compare.py new file mode 100755 index 0000000..967d749 --- /dev/null +++ b/rust/bench/compare.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Times the Go and Rust mapsize binaries on the same command lines. + + rust/bench/compare.py GO_BIN RUST_BIN PATH [RUNS] + +Runs are interleaved (go, rust, go, rust, …) after one untimed warm-up each, +so both see the same page-cache state. Reports median and best wall time and +median peak RSS per scenario as a Markdown table. Linux only (/usr/bin/time). +""" +import os, statistics, subprocess, sys, tempfile + +go, rust, path = sys.argv[1:4] +runs = int(sys.argv[4]) if len(sys.argv) > 4 else 7 +tmp = tempfile.mkdtemp(prefix="mapsize-bench-") +snap = {"go": os.path.join(tmp, "go.msz"), "rust": os.path.join(tmp, "rust.msz")} +bins = {"go": go, "rust": rust} + +def timed(argv): + r = subprocess.run(["/usr/bin/time", "-f", "%e %M"] + argv, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True) + if r.returncode != 0: + sys.exit(f"failed: {' '.join(argv)}\n{r.stderr}") + secs, kb = r.stderr.strip().splitlines()[-1].split() + return float(secs), int(kb) / 1024 + +scenarios = [(f"scan, {w} workers", lambda b, w=w: [bins[b], "--no-ui", "--workers", str(w), path]) for w in (1, 8, 32)] +scenarios += [ + ("scan + save snapshot", lambda b: [bins[b], "--no-ui", "--save", snap[b], path]), + ("load own snapshot", lambda b: [bins[b], "--no-ui", snap[b]]), + ("load Go snapshot", lambda b: [bins[b], "--no-ui", snap["go"]]), + ("scan + JSON export", lambda b: [bins[b], "--json", path]), + ("scan + top 50 largest files", lambda b: [bins[b], "--largest-files", "50", path]), +] + +print(f"Path: `{path}`, {runs} interleaved runs per binary, warm cache\n") +print("| scenario | Go median | Rust median | Go best | Rust best | Go RSS | Rust RSS | speed-up |") +print("|---|---:|---:|---:|---:|---:|---:|---:|") +for name, mk in scenarios: + argv = {k: mk(k) for k in bins} + res = {k: [] for k in bins} + for k in bins: + timed(argv[k]) # warm-up (also creates the snapshot files) + for _ in range(runs): + for k in bins: + res[k].append(timed(argv[k])) + med = {k: statistics.median(t for t, _ in v) for k, v in res.items()} + best = {k: min(t for t, _ in v) for k, v in res.items()} + rss = {k: statistics.median(m for _, m in v) for k, v in res.items()} + print(f"| {name} | {med['go']:.2f} s | {med['rust']:.2f} s | {best['go']:.2f} s | {best['rust']:.2f} s " + f"| {rss['go']:.0f} MB | {rss['rust']:.0f} MB | {med['go'] / med['rust']:.2f}× |", flush=True) diff --git a/rust/bench/micro.py b/rust/bench/micro.py new file mode 100755 index 0000000..27f4749 --- /dev/null +++ b/rust/bench/micro.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Runs the Go `go test -bench` suite and the Rust bench example +interleaved for several rounds and prints the best time per op of each. + + rust/bench/micro.py GO_TUI_TEST GO_TREEMAP_TEST RUST_BENCH [ROUNDS] + +Build the inputs with `go test -c ./internal/tui`, `go test -c +./internal/treemap` and `cargo build --release --example bench`. The Go tui +test binary must run from internal/tui, so run this from the repo root. +""" +import re, subprocess, sys + +go_tui, go_tm, rust = sys.argv[1:4] +rounds = int(sys.argv[4]) if len(sys.argv) > 4 else 5 +names = { # rust name -> go name + "treemap/Squarify(300)": "Squarify", "treemap/Neighbour(300)": "Neighbour", + "tui/BigColdFrame": "BigColdFrame", "tui/BigResize": "BigResize", + "tui/BigFlatDirCold": "BigFlatDirCold", "tui/BigNavigate": "BigNavigate", + "tui/BigFilter/size > 1MB": "BigFilter/size_>_1MB", + "tui/BigFilter/*.dat AND size > 500k": "BigFilter/*.dat_AND_size_>_500k", + "tui/BigFilter/path contains sub3": "BigFilter/path_contains_sub3", +} +unit = {"ns": 1e-9, "µs": 1e-6, "ms": 1e-3, "s": 1} +go_best, rust_best = {}, {} + +def keep(d, k, v): + d[k] = min(d.get(k, v), v) + +for _ in range(rounds): + for exe, cwd in ((go_tui, "internal/tui"), (go_tm, "internal/treemap")): + out = subprocess.run([exe, "-test.run=NONE", "-test.bench=.", "-test.benchtime=1s"], cwd=cwd, capture_output=True, text=True).stdout + for m in re.finditer(r"^Benchmark(\S+?)-\d+\s+\d+\s+([\d.]+) ns/op", out, re.M): + keep(go_best, m[1], float(m[2]) * 1e-9) + out = subprocess.run([rust], capture_output=True, text=True).stdout + for m in re.finditer(r"^(\S.*?)\s+\d+ ops\s+([\d.]+)(ns|µs|ms|s)/op", out, re.M): + keep(rust_best, m[1].strip(), float(m[2]) * unit[m[3]]) + +def fmt(s): + return f"{s * 1e6:.1f} µs" if s < 1e-3 else f"{s * 1e3:.2f} ms" + +print(f"Best of {rounds} interleaved rounds\n") +print("| benchmark | Go | Rust | speed-up |\n|---|---:|---:|---:|") +for r, g in names.items(): + if r in rust_best and g in go_best: + print(f"| {r} | {fmt(go_best[g])} | {fmt(rust_best[r])} | {go_best[g] / rust_best[r]:.2f}× |") diff --git a/rust/examples/bench.rs b/rust/examples/bench.rs new file mode 100644 index 0000000..4153a75 --- /dev/null +++ b/rust/examples/bench.rs @@ -0,0 +1,58 @@ +//! In-process benchmarks mirroring the Go `go test -bench` suite +//! (internal/tui/bench_test.go, internal/treemap/treemap_test.go): +//! +//! cargo run --release --example bench +//! +//! Each case runs for ~2 s after a warm-up and reports the mean time per op. + +use mapsize::treemap::{self, Direction, Item, Rect}; +use mapsize::tui::bench::{big_tree, BenchModel}; +use std::hint::black_box; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +fn bench(name: &str, mut f: impl FnMut(u64) -> R) { + black_box(f(0)); + let budget = Duration::from_secs(2); + let start = Instant::now(); + let mut n = 0u64; + while start.elapsed() < budget { + black_box(f(n)); + n += 1; + } + let per = start.elapsed() / n as u32; + println!("{name:<40} {n:>8} ops {per:>12.2?}/op"); +} + +fn main() { + let items: Vec = (0..300).map(|i| Item { id: i, size: (1 + i * i) as f64 }).collect(); + let bounds = Rect { x: 0, y: 0, w: 200, h: 60 }; + bench("treemap/Squarify(300)", |_| { + black_box(treemap::squarify(black_box(&items), bounds)); + }); + let blocks = treemap::squarify(&items, bounds); + bench("treemap/Neighbour(300)", |_| { + black_box(treemap::neighbour(black_box(&blocks), 150, Direction::Right)); + }); + + let t0 = Instant::now(); + let tree = Arc::new(RwLock::new(big_tree())); + println!("big tree: {} nodes built in {:.2?}", tree.read().unwrap().len(), t0.elapsed()); + + let mut m = BenchModel::new(tree.clone(), 200, 60); + bench("tui/BigColdFrame", |_| m.cold_frame()); + let mut m = BenchModel::new(tree.clone(), 200, 60); + m.cold_frame(); + bench("tui/BigResize", |i| m.resize(150 + (i % 100) as u16, 40 + (i % 30) as u16)); + let mut m = BenchModel::new(tree.clone(), 200, 60); + m.zoom_to_child_named(b"flat"); + bench("tui/BigFlatDirCold", |_| m.cold_frame()); + let mut m = BenchModel::new(tree.clone(), 200, 60); + m.cold_frame(); + let keys = ["right", "down", "left", "up"]; + bench("tui/BigNavigate", |i| m.key(keys[i as usize % 4])); + for q in ["size > 1MB", "*.dat AND size > 500k", "path contains sub3"] { + let mut m = BenchModel::new(tree.clone(), 200, 60); + bench(&format!("tui/BigFilter/{q}"), |_| m.apply_filter(q)); + } +} diff --git a/rust/src/app.rs b/rust/src/app.rs new file mode 100644 index 0000000..54cc0da --- /dev/null +++ b/rust/src/app.rs @@ -0,0 +1,216 @@ +//! Wires the command-line modes together: interactive TUI, non-interactive +//! reports, snapshots and comparisons. TUI and CLI share the same scanner +//! and inventory. + +use crate::cancel::Cancel; +use crate::config::Settings; +use crate::inventory::{Kind, SizeMode, Tree, FLAG_HARDLINK_DUP}; +use crate::scan::{self, Scanner}; +use crate::{duplicate, export, snapshot, textutil, tui, Error}; +use std::io::{IsTerminal, Write}; +use std::path::Path; +use std::sync::{Arc, RwLock}; +use std::time::{Duration, Instant}; + +/// The fully-resolved command line. +pub struct Config { + pub paths: Vec, + pub scan_opts: scan::Options, + pub no_ui: bool, + pub json: bool, + pub csv: bool, + pub depth: i64, + pub top: usize, + pub largest_files: usize, + pub save: String, + pub load: String, + pub compare: Option<(String, String)>, + pub duplicates: bool, + pub read_only: bool, + pub theme: String, + pub size_mode: SizeMode, + pub mouse: bool, + pub color: String, + pub settings: Settings, +} + +impl Config { + pub fn new(settings: Settings) -> Config { + Config { + paths: Vec::new(), scan_opts: Default::default(), no_ui: false, json: false, csv: false, depth: -1, + top: 0, largest_files: 0, save: String::new(), load: String::new(), compare: None, duplicates: false, + read_only: false, theme: settings.theme.clone(), size_mode: SizeMode::Allocated, mouse: settings.mouse, + color: "auto".into(), settings, + } + } + + /// Whether any non-interactive output was requested. + pub fn report(&self) -> bool { + self.no_ui || self.json || self.csv || self.top > 0 || self.largest_files > 0 || self.duplicates + } +} + +/// Executes the configured mode. +pub fn run(cfg: Config) -> Result<(), Error> { + if cfg.compare.is_some() { + return run_compare(cfg); + } + if !cfg.report() { + return run_tui(cfg); + } + run_report(cfg) +} + +/// Begins a scan (or loads a snapshot). The scanner is None when the tree +/// came from a snapshot. +fn start_source(cancel: Cancel, load: &str, paths: &[String], opts: &scan::Options) -> Result<(Option, Arc>), Error> { + if !load.is_empty() { + return Ok((None, Arc::new(RwLock::new(snapshot::load_file(Path::new(load))?)))); + } + let root = paths.first().map_or(".", |s| s.as_str()); + let s = scan::start(root.as_bytes(), opts.clone(), cancel)?; + let t = s.tree.clone(); + Ok((Some(s), t)) +} + +/// Cancels on SIGINT for report mode, like Go's signal.NotifyContext. +fn interrupt_cancel() -> Cancel { + static CANCEL: std::sync::OnceLock = std::sync::OnceLock::new(); + extern "C" fn on_sigint(_: libc::c_int) { + if let Some(c) = CANCEL.get() { + // Only an atomic store happens here; the channel drop is deferred. + c.cancel_flag_only(); + } + } + let c = CANCEL.get_or_init(Cancel::new).clone(); + unsafe { libc::signal(libc::SIGINT, on_sigint as libc::sighandler_t) }; + c +} + +fn run_report(cfg: Config) -> Result<(), Error> { + let cancel = interrupt_cancel(); + let (s, tree) = start_source(cancel.clone(), &cfg.load, &cfg.paths, &cfg.scan_opts)?; + if let Some(s) = &s { + wait_with_progress(s, &cancel); + } + let mut out = std::io::stdout().lock(); + if cfg.duplicates && !cfg.json && !cfg.csv { + // find takes its own read lock; never nest read locks. + run_duplicates(&mut out, &cancel, &tree)?; + } + let t = tree.read().unwrap(); + if s.is_some() && !cfg.save.is_empty() { + snapshot::save_file(Path::new(&cfg.save), &t)?; + eprintln!("Saved snapshot {}", cfg.save); + } + if cfg.json { + return Ok(export::json(&mut out, &t, t.root(), cfg.depth)?); + } + if cfg.csv { + return Ok(export::csv(&mut out, &t, t.root(), cfg.depth)?); + } + if cfg.top > 0 { + let mut ids = t.sorted_children(t.root(), cfg.size_mode); + ids.truncate(cfg.top); + export::table(&mut out, &t, &ids, cfg.size_mode)?; + writeln!(out)?; + } + if cfg.largest_files > 0 { + let m = cfg.size_mode; + let ids = t.top(t.root(), cfg.largest_files, |n| n.kind == Kind::File && n.flags & FLAG_HARDLINK_DUP == 0, |n| n.own(m)); + export::table(&mut out, &t, &ids, m)?; + writeln!(out)?; + } + export::summary(&mut out, &t)?; + Ok(()) +} + +/// Blocks until the scan finishes, printing a progress line to stderr if it +/// is a terminal. +fn wait_with_progress(s: &Scanner, cancel: &Cancel) { + let tty = std::io::stderr().is_terminal(); + let start = Instant::now(); + let mut forwarded = false; + loop { + match s.done().recv_timeout(Duration::from_millis(250)) { + Err(crossbeam_channel::RecvTimeoutError::Timeout) => {} + _ => { + if tty { + eprint!("\r\x1b[K"); + } + return; + } + } + if cancel.is_cancelled() && !forwarded { + cancel.cancel(); // complete the signal-handler cancellation + forwarded = true; + } + if tty { + let p = s.progress(); + let el = start.elapsed(); + eprint!( + "\r\x1b[Kscanning… {} entries {}/s queue {} {}", + textutil::count(p.entries), + textutil::count((p.entries as f64 / el.as_secs_f64()) as i64), + p.pending, + textutil::duration(el) + ); + } + } +} + +fn run_tui(cfg: Config) -> Result<(), Error> { + let (load, paths, opts) = (cfg.load.clone(), cfg.paths.clone(), cfg.scan_opts.clone()); + tui::run(tui::Options { + read_only: cfg.read_only, theme: cfg.theme, size_mode: cfg.size_mode, mouse: cfg.mouse, color: cfg.color, + save: cfg.save, settings: cfg.settings, diff: None, + start: Arc::new(move |c| start_source(c, &load, &paths, &opts)), + }) +} + +fn run_compare(cfg: Config) -> Result<(), Error> { + let (a, b) = cfg.compare.clone().unwrap(); + let old = snapshot::load_file(Path::new(&a)).map_err(|e| format!("{a}: {e}"))?; + let nw = snapshot::load_file(Path::new(&b)).map_err(|e| format!("{b}: {e}"))?; + let d = snapshot::compare(old, &nw); + if !cfg.report() { + let tree = Arc::new(RwLock::new(nw)); + return tui::run(tui::Options { + read_only: true, theme: cfg.theme, size_mode: cfg.size_mode, mouse: cfg.mouse, color: cfg.color, + save: String::new(), settings: cfg.settings, diff: Some(d), + start: Arc::new(move |_| Ok((None, tree.clone()))), + }); + } + let k = if cfg.top > 0 { cfg.top } else { 30 }; + let m = cfg.size_mode; + let mut out = std::io::stdout().lock(); + let (ro, rn) = (d.old.node(0), nw.node(0)); + writeln!(out, "{} → {}", textutil::sanitize(&d.old.stats.from_snapshot), textutil::sanitize(&nw.stats.from_snapshot))?; + writeln!(out, "Total {} → {} ({})\n", textutil::size(ro.total(m)), textutil::size(rn.total(m)), textutil::signed_size(rn.total(m) - ro.total(m)))?; + for c in d.changes(&nw, k, m) { + writeln!(out, "{:<10} {:>14} {}", c.status.name(), textutil::signed_size(c.delta), textutil::sanitize(&c.path))?; + } + Ok(()) +} + +fn run_duplicates(out: &mut dyn Write, cancel: &Cancel, tree: &RwLock) -> Result<(), Error> { + let f = duplicate::Finder::default(); + let groups = f.find(cancel, tree, duplicate::Options { min_size: 1, workers: 0 }).ok_or("cancelled")?; + let t = tree.read().unwrap(); + let mut wasted = 0; + for g in &groups { + wasted += g.wasted(); + let h: String = g.hash[..6].iter().map(|b| format!("{b:02x}")).collect(); + writeln!(out, "{} × {} (sha256 {h}…)", textutil::size(g.size), g.files.len())?; + for &id in &g.files { + writeln!(out, " {}", textutil::sanitize_bytes(&t.path_bytes(id)))?; + } + } + let p = f.progress(); + writeln!( + out, + "\n{} verified duplicate groups, {} reclaimable ({} candidates by size, {} unreadable/changed)\n", + groups.len(), textutil::size(wasted), p.candidates, p.skipped + )?; + Ok(()) +} diff --git a/rust/src/brand.rs b/rust/src/brand.rs new file mode 100644 index 0000000..5747cde --- /dev/null +++ b/rust/src/brand.rs @@ -0,0 +1,11 @@ +//! Product name isolated so it can be changed in one place. + +pub const NAME: &str = "mapsize"; +pub const SNAPSHOT_EXT: &str = ".msz"; +pub const SNAPSHOT_MAGIC: &[u8; 8] = b"MAPSIZE\x00"; + +/// Overridden at release time via the MAPSIZE_VERSION environment variable. +pub const VERSION: &str = match option_env!("MAPSIZE_VERSION") { + Some(v) => v, + None => env!("CARGO_PKG_VERSION"), +}; diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs new file mode 100644 index 0000000..e78a996 --- /dev/null +++ b/rust/src/cancel.rs @@ -0,0 +1,49 @@ +//! A clonable cancellation token, the equivalent of a Go context's Done. + +use crossbeam_channel::{Receiver, Sender}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +pub struct Cancel(Arc); + +struct Inner { + flag: AtomicBool, + tx: Mutex>>, + rx: Receiver<()>, +} + +impl Default for Cancel { + fn default() -> Self { + Self::new() + } +} + +impl Cancel { + pub fn new() -> Cancel { + let (tx, rx) = crossbeam_channel::bounded(0); + Cancel(Arc::new(Inner { flag: AtomicBool::new(false), tx: Mutex::new(Some(tx)), rx })) + } + + pub fn cancel(&self) { + self.0.flag.store(true, Ordering::Relaxed); + self.0.tx.lock().unwrap().take(); + } + + pub fn is_cancelled(&self) -> bool { + self.0.flag.load(Ordering::Relaxed) + } + + /// Becomes ready (disconnected) once cancelled; for use in select. + pub fn done(&self) -> &Receiver<()> { + &self.0.rx + } +} + +impl Cancel { + /// Async-signal-safe half of cancel(): sets the flag only. Pollers must + /// follow up with cancel() to wake select-based waiters. + pub fn cancel_flag_only(&self) { + self.0.flag.store(true, Ordering::Relaxed); + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs new file mode 100644 index 0000000..422838d --- /dev/null +++ b/rust/src/config.rs @@ -0,0 +1,75 @@ +//! User settings from a JSON file in the OS-specific user configuration +//! directory. Knows nothing about rendering. + +use crate::brand; +use serde::Deserialize; +use std::collections::HashMap; +use std::path::PathBuf; + +/// User preferences. Load starts from the defaults and overlays the file. +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct Settings { + pub workers: String, // conservative|balanced|aggressive|N + pub follow_symlinks: String, // none|same-filesystem|all + pub one_file_system: bool, + pub units: String, // iec|si + pub size_mode: String, // allocated|logical + pub theme: String, + pub mouse: bool, + pub excludes: Vec, + /// Maps an extension (without dot) to "#rrggbb". + pub extension_colors: HashMap, + /// Maps a category name to "#rrggbb". + pub category_colors: HashMap, + /// UI refresh rate while scanning. + pub refresh_hz: i64, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + workers: "balanced".into(), + follow_symlinks: "none".into(), + one_file_system: false, + units: "iec".into(), + size_mode: "allocated".into(), + theme: "default".into(), + mouse: true, + excludes: Vec::new(), + extension_colors: HashMap::new(), + category_colors: HashMap::new(), + refresh_hz: 10, + } + } +} + +/// The config file location. +pub fn path() -> Option { + let env = |k: &str| std::env::var_os(k).filter(|v| !v.is_empty()).map(PathBuf::from); + let dir = if cfg!(target_os = "macos") { + env("HOME")?.join("Library/Application Support") + } else { + env("XDG_CONFIG_HOME").or_else(|| env("HOME").map(|h| h.join(".config")))? + }; + Some(dir.join(brand::NAME).join("config.json")) +} + +/// Reads the config file if present. A missing file is not an error. +pub fn load() -> (Settings, Option) { + let Some(p) = path() else { return (Settings::default(), None) }; + let b = match std::fs::read(&p) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return (Settings::default(), None), + Err(e) => return (Settings::default(), Some(e.to_string())), + }; + match serde_json::from_slice::(&b) { + Ok(mut s) => { + if !(1..=60).contains(&s.refresh_hz) { + s.refresh_hz = 10; + } + (s, None) + } + Err(e) => (Settings::default(), Some(e.to_string())), + } +} diff --git a/rust/src/duplicate.rs b/rust/src/duplicate.rs new file mode 100644 index 0000000..745c644 --- /dev/null +++ b/rust/src/duplicate.rs @@ -0,0 +1,315 @@ +//! Finds files with identical content in stages: +//! +//! 1. group by exact size (inventory only, no I/O) +//! 2. hash three 16 KiB samples (start, middle, end) of each candidate +//! 3. full SHA-256 of files whose samples still collide +//! +//! Only stage 3 establishes identity. Files are never declared duplicates by +//! name or size alone. Reading files may update their access times. + +use crate::cancel::Cancel; +use crate::inventory::{Kind, NodeId, Tree, FLAG_HARDLINK_DUP}; +use crate::platform; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{self, Read}; +use std::os::unix::fs::FileExt; +use std::path::PathBuf; +use std::sync::atomic::{AtomicI64, AtomicUsize, Ordering}; +use std::sync::{Mutex, RwLock}; + +/// A set of files verified to have identical content. +#[derive(Clone, Debug)] +pub struct Group { + pub size: i64, + pub hash: [u8; 32], + pub files: Vec, +} + +impl Group { + /// The bytes reclaimable by keeping one copy. + pub fn wasted(&self) -> i64 { + self.size * (self.files.len() as i64 - 1) + } +} + +/// Reports activity. +#[derive(Clone, Default, Debug)] +pub struct Progress { + pub stage: String, + pub candidates: i64, // files sharing a size with another file + pub bytes_to_hash: i64, + pub bytes_hashed: i64, + pub skipped: i64, // unreadable or changed during hashing + pub verified_groups: i64, + pub candidate_groups: i64, +} + +/// Tunes the search. +#[derive(Clone, Copy, Default)] +pub struct Options { + pub min_size: i64, // ignore files smaller than this (default 1 byte) + pub workers: usize, // concurrent readers (default 4) +} + +#[derive(Clone)] +struct Cand { + id: NodeId, + path: PathBuf, + size: i64, +} + +const SAMPLE_SIZE: i64 = 16 << 10; + +type HashFn = fn(&Finder, &Cand) -> Option<[u8; 32]>; + +/// Runs a search and exposes live progress. +#[derive(Default)] +pub struct Finder { + hashed: AtomicI64, + to_hash: AtomicI64, + skipped: AtomicI64, + p: Mutex, +} + +impl Finder { + /// A copy of the current progress. + pub fn progress(&self) -> Progress { + let mut p = self.p.lock().unwrap().clone(); + p.bytes_hashed = self.hashed.load(Ordering::Relaxed); + p.bytes_to_hash = self.to_hash.load(Ordering::Relaxed); + p.skipped = self.skipped.load(Ordering::Relaxed); + p + } + + fn stage(&self, s: &str, f: impl FnOnce(&mut Progress)) { + let mut p = self.p.lock().unwrap(); + p.stage = s.to_string(); + f(&mut p); + } + + /// Searches the tree, taking its own read lock only while collecting + /// candidates. None if cancelled. + pub fn find(&self, cancel: &Cancel, tree: &RwLock, mut o: Options) -> Option> { + o.min_size = o.min_size.max(1); + if o.workers < 1 { + o.workers = 4; + } + self.stage("grouping by size", |_| {}); + let mut groups: Vec> = Vec::new(); + let mut ncand = 0i64; + { + let t = tree.read().unwrap(); + let mut by_size: HashMap> = HashMap::new(); + t.walk(t.root(), |id, n| { + if n.kind == Kind::File && !n.has(FLAG_HARDLINK_DUP) && n.size >= o.min_size { + by_size.entry(n.size).or_default().push(Cand { id, path: PathBuf::new(), size: n.size }); + } + true + }); + for mut g in by_size.into_values() { + if g.len() > 1 { + for c in &mut g { + c.path = t.path(c.id); + } + ncand += g.len() as i64; + groups.push(g); + } + } + } + self.stage("sampling", |p| { + p.candidates = ncand; + p.candidate_groups = groups.len() as i64; + }); + + // Stage 2: sample hashes. + let mut sampled: Vec> = Vec::new(); + for g in groups { + if cancel.is_cancelled() { + return None; + } + if g[0].size <= 3 * SAMPLE_SIZE { + sampled.push(g); // a sample would be the whole file + continue; + } + sampled.extend(self.split(cancel, &g, o.workers, sample_hash).into_values().filter(|v| v.len() > 1)); + } + + // Stage 3: full hashes. + self.to_hash.store(sampled.iter().map(|g| g[0].size * g.len() as i64).sum(), Ordering::Relaxed); + self.stage("hashing", |_| {}); + let mut out = Vec::new(); + for g in &sampled { + if cancel.is_cancelled() { + return None; + } + for (hash, v) in self.split(cancel, g, o.workers, full_hash) { + if v.len() > 1 { + let mut files: Vec = v.iter().map(|c| c.id).collect(); + files.sort_unstable(); + out.push(Group { size: v[0].size, hash, files }); + } + } + } + if cancel.is_cancelled() { + return None; + } + out.sort_by(|a, b| b.wasted().cmp(&a.wasted()).then(a.files[0].cmp(&b.files[0]))); + self.stage("done", |p| p.verified_groups = out.len() as i64); + Some(out) + } + + /// Hashes every candidate with a bounded pool of reader threads and + /// buckets the successes by hash. Stops starting new reads once cancelled. + fn split(&self, cancel: &Cancel, g: &[Cand], workers: usize, f: HashFn) -> HashMap<[u8; 32], Vec> { + let next = AtomicUsize::new(0); + let results = Mutex::new(Vec::with_capacity(g.len())); + std::thread::scope(|s| { + for _ in 0..workers.min(g.len()) { + s.spawn(|| loop { + let i = next.fetch_add(1, Ordering::Relaxed); + if i >= g.len() || cancel.is_cancelled() { + break; + } + match f(self, &g[i]) { + Some(sum) => results.lock().unwrap().push((i, sum)), + None => { + self.skipped.fetch_add(1, Ordering::Relaxed); + } + } + }); + } + }); + let mut res = results.into_inner().unwrap(); + res.sort_unstable_by_key(|r| r.0); + let mut by: HashMap<[u8; 32], Vec> = HashMap::new(); + for (i, sum) in res { + by.entry(sum).or_default().push(g[i].clone()); + } + by + } +} + +/// Opens a candidate (never following symlinks or blocking on FIFOs) and +/// checks it is still a regular file of the inventoried size. +fn open(c: &Cand) -> Option { + let fh = platform::open_content(&c.path).ok()?; + let md = fh.metadata().ok()?; + (md.is_file() && md.len() as i64 == c.size).then_some(fh) +} + +fn sample_hash(_: &Finder, c: &Cand) -> Option<[u8; 32]> { + let fh = open(c)?; + let mut h = Sha256::new(); + let mut buf = vec![0u8; SAMPLE_SIZE as usize]; + for off in [0, c.size / 2 - SAMPLE_SIZE / 2, c.size - SAMPLE_SIZE] { + match fh.read_exact_at(&mut buf, off as u64) { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => {} // shrank; stage 3 rejects it + Err(_) => return None, + } + h.update(&buf); + } + Some(h.finalize().into()) +} + +fn full_hash(f: &Finder, c: &Cand) -> Option<[u8; 32]> { + let mut fh = open(c)?; + let mut h = Sha256::new(); + let mut buf = vec![0u8; 32 << 10]; + let mut n = 0i64; + loop { + match fh.read(&mut buf) { + Ok(0) => break, + Ok(k) => { + h.update(&buf[..k]); + n += k as i64; + f.hashed.fetch_add(k as i64, Ordering::Relaxed); + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(_) => return None, + } + } + (n == c.size).then(|| h.finalize().into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::scan::{self, tests::TempDir}; + use std::os::unix::ffi::OsStrExt; + use std::path::Path; + use std::sync::Arc; + use std::time::Duration; + + fn scan(root: &Path, workers: usize) -> Arc> { + let s = scan::start(root.as_os_str().as_bytes(), scan::Options { workers, ..Default::default() }, Cancel::new()).unwrap(); + s.wait(); + s.tree.clone() + } + + #[test] + fn find() { + let root = TempDir::new("dup"); + let big: Vec = (0..200_000usize).map(|i| (i * 7) as u8).collect(); + let w = |name: &str, b: &[u8]| std::fs::write(root.0.join(name), b).unwrap(); + w("a.bin", &big); + w("b.bin", &big); + // Same size, same samples, different middle-of-nowhere byte: must not match. + let mut c = big.clone(); + c[20_000] ^= 1; + w("c.bin", &c); + w("small1", b"hello"); + w("small2", b"hello"); + w("small3", b"world"); // same size, different content + + let tree = scan(&root.0, 2); + let f = Finder::default(); + let groups = f.find(&Cancel::new(), &tree, Options::default()).unwrap(); + assert_eq!(groups.len(), 2); + let t = tree.read().unwrap(); + let names = |g: &Group| { + let mut v: Vec = g.files.iter().map(|&id| t.node(id).name_str().into_owned()).collect(); + v.sort(); + v + }; + assert_eq!(names(&groups[0]), ["a.bin", "b.bin"], "big group"); + assert_eq!(groups[0].size, 200_000); + assert_eq!(names(&groups[1]), ["small1", "small2"], "small group"); + let p = f.progress(); + assert!(p.candidates == 6 && p.verified_groups == 2, "progress {p:?}"); + } + + /// A candidate replaced by a FIFO (or a symlink to one) after the scan + /// must be skipped, not block the search forever in open(2). + #[test] + fn find_skips_swapped_fifo() { + let root = TempDir::new("dupfifo"); + let other = TempDir::new("dupfifo2"); + for n in ["a", "b", "c"] { + std::fs::write(root.0.join(n), b"same").unwrap(); + } + let tree = scan(&root.0, 1); + let mkfifo = |p: &Path| { + let c = std::ffi::CString::new(p.as_os_str().as_bytes()).unwrap(); + unsafe { libc::mkfifo(c.as_ptr(), 0o644) == 0 } + }; + std::fs::remove_file(root.0.join("b")).unwrap(); + if !mkfifo(&root.0.join("b")) { + eprintln!("mkfifo unsupported; skipping"); + return; + } + let fifo2 = other.0.join("p"); + assert!(mkfifo(&fifo2)); + std::fs::remove_file(root.0.join("c")).unwrap(); + std::os::unix::fs::symlink(&fifo2, root.0.join("c")).unwrap(); + + let f = Arc::new(Finder::default()); + let (tx, rx) = crossbeam_channel::bounded(1); + let (f2, t2) = (f.clone(), tree.clone()); + std::thread::spawn(move || tx.send(f2.find(&Cancel::new(), &t2, Options::default()).is_some())); + assert!(rx.recv_timeout(Duration::from_secs(5)).expect("duplicate search blocked on a FIFO")); + assert_eq!(f.progress().skipped, 2); + } +} diff --git a/rust/src/export.rs b/rust/src/export.rs new file mode 100644 index 0000000..60490bd --- /dev/null +++ b/rust/src/export.rs @@ -0,0 +1,222 @@ +//! Writes inventories as JSON, CSV or plain-text reports. Callers hold the +//! tree's read lock (or own the tree exclusively). + +use crate::brand; +use crate::inventory::{self as inv, Flags, Kind, NodeId, SizeMode, Tree, NO_NODE}; +use crate::textutil; +use std::io::{self, Write}; +use std::time::UNIX_EPOCH; + +const FLAG_NAMES: [(Flags, &str); 12] = [ + (inv::FLAG_HARDLINK_DUP, "hardlink-duplicate"), (inv::FLAG_MOUNT_POINT, "mount-point"), + (inv::FLAG_VIRTUAL_FS, "virtual-fs-skipped"), (inv::FLAG_SKIPPED_FS, "other-fs-skipped"), + (inv::FLAG_SPARSE, "sparse"), (inv::FLAG_ERROR, "error"), (inv::FLAG_INCOMPLETE, "incomplete"), + (inv::FLAG_LOOP, "loop-skipped"), (inv::FLAG_FOLLOWED, "followed-symlink"), + (inv::FLAG_ALLOC_UNKNOWN, "allocated-unknown"), (inv::FLAG_BROKEN_LINK, "broken-symlink"), + (inv::FLAG_HARDLINKED, "hardlinked"), +]; + +/// Human-readable names for a node's flags. +pub fn flag_list(f: Flags) -> Vec<&'static str> { + FLAG_NAMES.iter().filter(|(b, _)| f & b != 0).map(|(_, n)| *n).collect() +} + +fn kind_name(k: Kind) -> &'static str { + ["file", "dir", "symlink", "other"][k as usize] +} + +fn json_str(s: &str) -> String { + serde_json::to_string(s).unwrap() +} + +/// Writes the tree beneath root as nested JSON, down to max_depth levels +/// (negative means unlimited). +pub fn json(w: &mut dyn Write, t: &Tree, root: NodeId, max_depth: i64) -> io::Result<()> { + let mut bw = io::BufWriter::new(w); + let st = &t.stats; + let errors: serde_json::Map = st + .err_counts + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(k, &c)| (inv::ErrKind::from_u8(k as u8).name().to_string(), c.into())) + .collect(); + let started = st.start.duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos() as i64); + let head = serde_json::json!({ + "generator": format!("{} {}", brand::NAME, brand::VERSION), + "root": t.path_string(root), + "complete": !st.incomplete(), + "cancelled": st.cancelled, + "started": textutil::rfc3339_utc(started), + "elapsed_s": st.elapsed().as_secs_f64(), + "errors": errors, + "excluded": st.excluded, + "excludes": st.excludes, + "hardlink_duplicates": st.hardlink_dups, + }); + let hb = serde_json::to_string(&head)?; + bw.write_all(&hb.as_bytes()[..hb.len() - 1])?; + bw.write_all(b",\"tree\":")?; + write_node(&mut bw, t, root, max_depth)?; + bw.write_all(b"}\n")?; + bw.flush() +} + +/// Streams one node; children are written recursively so memory stays +/// proportional to depth, not tree size. +fn write_node(bw: &mut impl Write, t: &Tree, id: NodeId, depth: i64) -> io::Result<()> { + let n = t.node(id); + write!(bw, "{{\"name\":{},\"type\":\"{}\",\"size\":{},\"allocated\":{}", json_str(&n.name_str()), kind_name(n.kind), n.tot_size, n.tot_alloc)?; + for (k, v) in [("files", n.files), ("dirs", n.dirs), ("errors", n.errors)] { + if v != 0 { + write!(bw, ",\"{k}\":{v}")?; + } + } + if n.mtime != 0 { + write!(bw, ",\"modified\":\"{}\"", textutil::rfc3339_utc(n.mtime))?; + } + let flags = flag_list(n.flags); + if !flags.is_empty() { + write!(bw, ",\"flags\":{}", serde_json::to_string(&flags)?)?; + } + if n.first_child == NO_NODE || depth == 0 { + return bw.write_all(b"}"); + } + bw.write_all(b",\"children\":[")?; + for (i, c) in t.sorted_children(id, SizeMode::Allocated).into_iter().enumerate() { + if i > 0 { + bw.write_all(b",")?; + } + write_node(bw, t, c, depth - 1)?; + } + bw.write_all(b"]}") +} + +fn csv_field(s: &str) -> String { + if s.contains([',', '"', '\n', '\r']) || s.starts_with(' ') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } +} + +/// Writes one row per node beneath root (depth-limited). Paths are +/// sanitised so the output is safe to print to a terminal. +pub fn csv(w: &mut dyn Write, t: &Tree, root: NodeId, max_depth: i64) -> io::Result<()> { + let mut bw = io::BufWriter::new(w); + bw.write_all(b"path,type,size,allocated,files,dirs,errors,modified,flags\n")?; + fn rec(bw: &mut impl Write, t: &Tree, id: NodeId, path: &[u8], depth: i64) -> io::Result<()> { + let n = t.node(id); + let modified = if n.mtime != 0 { textutil::rfc3339_utc(n.mtime) } else { String::new() }; + writeln!( + bw, + "{},{},{},{},{},{},{},{},{}", + csv_field(&textutil::sanitize_bytes(path)), + kind_name(n.kind), + n.tot_size, n.tot_alloc, n.files, n.dirs, n.errors, modified, + flag_list(n.flags).join("|") + )?; + if depth == 0 { + return Ok(()); + } + for c in t.sorted_children(id, SizeMode::Allocated) { + rec(bw, t, c, &crate::platform::join_path(path, &t.node(c).name), depth - 1)?; + } + Ok(()) + } + rec(&mut bw, t, root, &t.path_bytes(root), max_depth)?; + bw.flush() +} + +/// Writes "size % files path" rows for the given nodes. +pub fn table(w: &mut dyn Write, t: &Tree, ids: &[NodeId], m: SizeMode) -> io::Result<()> { + let total = t.node(t.root()).total(m); + writeln!(w, "{:>12} {:>6} {:>12} PATH", "SIZE", "%", "FILES")?; + for &id in ids { + let n = t.node(id); + let files = if n.is_dir() { textutil::count(n.files as i64) } else { String::new() }; + writeln!( + w, + "{:>12} {:>6} {:>12} {}", + textutil::size(n.total(m)), + textutil::percent(n.total(m), total), + files, + textutil::sanitize_bytes(&t.path_bytes(id)) + )?; + } + Ok(()) +} + +/// Writes the scan summary, making incompleteness explicit. +pub fn summary(w: &mut dyn Write, t: &Tree) -> io::Result<()> { + let st = &t.stats; + let r = t.node(t.root()); + writeln!(w, "Root {}", textutil::sanitize_bytes(&st.root))?; + writeln!(w, "Files {}", textutil::count(st.files + st.symlinks + st.others))?; + writeln!(w, "Directories {}", textutil::count(st.dirs))?; + writeln!(w, "Logical size {} ({} bytes)", textutil::size(r.tot_size), r.tot_size)?; + writeln!(w, "Allocated size {} ({} bytes)", textutil::size(r.tot_alloc), r.tot_alloc)?; + writeln!(w, "Elapsed {}", textutil::go_duration(st.elapsed()))?; + for (k, &c) in st.err_counts.iter().enumerate() { + if c > 0 { + writeln!(w, "{:<15} {}", inv::ErrKind::from_u8(k as u8).name(), textutil::count(c))?; + } + } + if st.excluded > 0 { + writeln!(w, "Excluded {} entries ({})", textutil::count(st.excluded), textutil::sanitize(&st.excludes.join(", ")))?; + } + if st.skipped_mounts > 0 { + writeln!(w, "Other FS skipped {}", textutil::count(st.skipped_mounts))?; + } + if st.virtual_skipped > 0 { + writeln!(w, "Virtual FS skipped {}", textutil::count(st.virtual_skipped))?; + } + if st.hardlink_dups > 0 { + writeln!(w, "Hard-link dups {} (counted once)", textutil::count(st.hardlink_dups))?; + } + if st.incomplete() { + writeln!(w, "WARNING: totals are incomplete — some data could not be inspected.")?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hostile_tree() -> Tree { + let mut t = Tree::new(b"/r", Kind::Dir); + let d = t.add(0, b"esc\x1b[2Jdir", Kind::Dir); + let f = t.add(d, b"nl\nname\x07.txt", Kind::File); + t.node_mut(f).size = 10; + t.node_mut(f).alloc = 4096; + t.stats.complete = true; + t.recompute(); + t + } + + #[test] + fn outputs_are_terminal_safe() { + let t = hostile_tree(); + for out in [ + { let mut b = Vec::new(); json(&mut b, &t, 0, -1).unwrap(); b }, + { let mut b = Vec::new(); csv(&mut b, &t, 0, -1).unwrap(); b }, + { let mut b = Vec::new(); table(&mut b, &t, &[1, 2], SizeMode::Allocated).unwrap(); b }, + ] { + assert!(!out.iter().any(|&c| c == 0x1b || c == 0x07), "{}", String::from_utf8_lossy(&out)); + } + let mut b = Vec::new(); + json(&mut b, &t, 0, -1).unwrap(); + let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); + assert_eq!(v["tree"]["children"][0]["children"][0]["size"], 10); + } + + #[test] + fn json_depth_limit() { + let t = hostile_tree(); + let mut b = Vec::new(); + json(&mut b, &t, 0, 1).unwrap(); + let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); + assert!(v["tree"]["children"][0].get("children").is_none()); + } +} diff --git a/rust/src/filter/apply.rs b/rust/src/filter/apply.rs new file mode 100644 index 0000000..6fa4200 --- /dev/null +++ b/rust/src/filter/apply.rs @@ -0,0 +1,79 @@ +use super::eval::Ctx; +use super::parser::Query; +use crate::cancel::Cancel; +use crate::inventory::{NodeId, SizeMode, Tree, FLAG_HARDLINK_DUP}; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Per-node filtered sizes: the bytes of matching content beneath each node. +/// A matching directory contributes its whole subtree (once, even if +/// descendants match too). matched marks every node satisfying the query. +/// Nodes added after the result was computed (live scan) have no entry and +/// are treated as not matching until the filter is re-applied. +pub struct FilterResult { + pub query: Arc, + pub mode: SizeMode, + pub sizes: Vec, // per NodeId; len = tree size at evaluation time + pub matched: Vec, // node itself matched + pub count: i64, // number of matching nodes + pub total: i64, + pub elapsed: Duration, +} + +impl FilterResult { + /// The filtered size of id (0 if out of range). + pub fn size(&self, id: NodeId) -> i64 { + self.sizes.get(id as usize).copied().unwrap_or(0) + } + + /// Reports whether id itself matched. + pub fn is_match(&self, id: NodeId) -> bool { + self.matched.get(id as usize).copied().unwrap_or(false) + } +} + +/// Evaluates q over the whole tree. The caller holds the read lock. It relies +/// on parents having smaller IDs than their children. None if cancelled. +pub fn apply(cancel: &Cancel, t: &Tree, q: Arc, m: SizeMode) -> Option { + let start = Instant::now(); + let n = t.len(); + let mut sizes = vec![0i64; n]; + let mut matched = vec![false; n]; + let mut covered = vec![false; n]; + let mut count = 0; + let now = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos() as i64); + let mut c = Ctx::new(t, 0, now); + if q.uses_path { + c.dir_paths = Some(vec![String::new(); n]); + } + for i in 0..n { + if i & 0xffff == 0 && cancel.is_cancelled() { + return None; + } + let id = i as NodeId; + let nd = t.node(id); + c.reset(id); + if c.dir_paths.is_some() && nd.is_dir() { + c.fill_path(); + let p = c.path.clone(); + c.dir_paths.as_mut().unwrap()[i] = p; + } + if i > 0 && q.matches(&mut c) { + matched[i] = true; + covered[i] = true; + count += 1; + } + if i > 0 && covered[nd.parent as usize] { + covered[i] = true; + } + if covered[i] && nd.flags & FLAG_HARDLINK_DUP == 0 { + sizes[i] = nd.own(m); + } + } + for i in (1..n).rev() { + let p = t.node(i as NodeId).parent as usize; + sizes[p] = sizes[p].saturating_add(sizes[i]); + } + let total = sizes.first().copied().unwrap_or(0); + Some(FilterResult { query: q, mode: m, sizes, matched, count, total, elapsed: start.elapsed() }) +} diff --git a/rust/src/filter/eval.rs b/rust/src/filter/eval.rs new file mode 100644 index 0000000..1ce677c --- /dev/null +++ b/rust/src/filter/eval.rs @@ -0,0 +1,159 @@ +use super::parser::{Field, Query}; +use crate::inventory::{Flags, Node, NodeId, Tree}; +use crate::platform; +use regex::Regex; +use std::borrow::Cow; + +/// The evaluation context for one node. The tree must be read-locked. +pub struct Ctx<'a> { + pub tree: &'a Tree, + pub id: NodeId, + pub node: &'a Node, + pub now: i64, // unix nanoseconds + // The lower-cased path of the node once has_path is set; the buffers are + // reused across nodes so the per-node cost is one copy, not allocations. + pub(super) path: String, + pub(super) has_path: bool, + name: String, + // dir_paths caches lower-cased directory paths (by NodeId; empty for + // files) during apply, which visits parents before children, so a node's + // path is one concatenation. + pub(super) dir_paths: Option>, +} + +/// Appends name lower-cased, without allocating for ASCII names. +fn push_lower(buf: &mut String, name: &[u8]) { + if name.is_ascii() { + buf.extend(name.iter().map(|b| b.to_ascii_lowercase() as char)); + } else { + buf.push_str(&String::from_utf8_lossy(name).to_lowercase()); + } +} + +impl<'a> Ctx<'a> { + pub fn new(tree: &'a Tree, id: NodeId, now: i64) -> Ctx<'a> { + Ctx { tree, id, node: tree.node(id), now, path: String::new(), has_path: false, name: String::new(), dir_paths: None } + } + + /// Moves the context to another node. + pub(super) fn reset(&mut self, id: NodeId) { + self.id = id; + self.node = self.tree.node(id); + self.has_path = false; + } + + fn str(&mut self, f: Field) -> Cow<'_, str> { + let n = self.node; + match f { + Field::Name => { + self.name.clear(); + push_lower(&mut self.name, &n.name); + self.name.as_str().into() + } + Field::Path => { + if !self.has_path { + self.fill_path(); + } + self.path.as_str().into() + } + Field::Ext => self.tree.ext_name(n).into(), + Field::Type => n.kind.name().to_lowercase().into(), + Field::Category if n.is_dir() => "directory".into(), + Field::Category => n.cat.name().to_lowercase().into(), + Field::Owner => platform::user_name(n.uid).to_lowercase().into(), + Field::Group => platform::group_name(n.gid).to_lowercase().into(), + _ => "".into(), + } + } + + fn num(&self, f: Field) -> i64 { + let n = self.node; + match f { + Field::Size => n.tot_size, + Field::Allocated => n.tot_alloc, + Field::Age => self.now.saturating_sub(n.mtime), + Field::Modified => n.mtime, + Field::Files => n.files as i64, + _ => 0, + } + } + + /// Computes the lower-cased path into self.path. + pub(super) fn fill_path(&mut self) { + self.has_path = true; + self.path.clear(); + if let Some(pp) = self.dir_paths.as_ref().and_then(|dp| dp.get(self.node.parent as usize)).filter(|p| !p.is_empty()) { + self.path.push_str(pp); + if !pp.ends_with('/') { + self.path.push('/'); + } + push_lower(&mut self.path, &self.node.name); + return; + } + self.path.push_str(&self.tree.path_string(self.id).to_lowercase()); + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum NumOp { + Eq, + Ne, + Gt, + Ge, + Lt, + Le, +} + +/// A compiled query expression. +pub(super) enum Expr { + And(Box, Box), + Or(Box, Box), + Not(Box), + Contains(Field, String), + Glob(Field, String), + Re(Field, Regex), + Eq(Field, Vec), + Flag(Flags), // any of these flags set + Num(Field, NumOp, Vec), + False, // malformed bare glob +} + +impl Expr { + fn eval(&self, c: &mut Ctx) -> bool { + match self { + Expr::And(l, r) => l.eval(c) && r.eval(c), + Expr::Or(l, r) => l.eval(c) || r.eval(c), + Expr::Not(n) => !n.eval(c), + Expr::Contains(f, sub) => c.str(*f).contains(sub.as_str()), + Expr::Glob(f, pat) => platform::glob_match(pat.as_bytes(), c.str(*f).as_bytes()), + Expr::Re(f, re) => re.is_match(&c.str(*f)), + Expr::Eq(f, set) => { + let v = c.str(*f); + set.iter().any(|s| *s == v) + } + Expr::Flag(mask) => c.node.flags & mask != 0, + Expr::Num(f, op, vals) => { + if matches!(f, Field::Age | Field::Modified) && c.node.mtime == 0 { + return false; // unknown time never matches + } + let v = c.num(*f); + vals.iter().any(|&x| match op { + NumOp::Eq => v == x, + NumOp::Ne => v != x, + NumOp::Gt => v > x, + NumOp::Ge => v >= x, + NumOp::Lt => v < x, + NumOp::Le => v <= x, + }) + } + Expr::False => false, + } + } +} + +impl Query { + /// Evaluates the query against one node. + pub fn matches(&self, c: &mut Ctx) -> bool { + self.root.eval(c) + } +} diff --git a/rust/src/filter/lexer.rs b/rust/src/filter/lexer.rs new file mode 100644 index 0000000..f39f5f7 --- /dev/null +++ b/rust/src/filter/lexer.rs @@ -0,0 +1,95 @@ +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum TokKind { + Eof, + Word, + Str, + Op, + LParen, + RParen, + Comma, +} + +#[derive(Clone, Debug)] +pub(super) struct Token { + pub kind: TokKind, + pub val: String, + pub pos: usize, // byte offset +} + +fn tok(kind: TokKind, val: &str, pos: usize) -> Token { + Token { kind, val: val.to_string(), pos } +} + +/// ASCII white space as Go's unicode.IsSpace sees it. +fn is_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r') +} + +pub(super) fn lex(s: &str) -> Result, String> { + use TokKind::*; + let b = s.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let c = b[i]; + match c { + // All of is_space, not just " \t\n\r": a word can't start on a + // space byte, so anything else here would never advance. + _ if is_space(c) => i += 1, + b'(' => { + out.push(tok(LParen, "(", i)); + i += 1; + } + b')' => { + out.push(tok(RParen, ")", i)); + i += 1; + } + b',' => { + out.push(tok(Comma, ",", i)); + i += 1; + } + b'"' | b'\'' => { + let start = i; + i += 1; + let mut v = Vec::new(); + while i < b.len() && b[i] != c { + if b[i] == b'\\' && i + 1 < b.len() { + i += 1; + } + v.push(b[i]); + i += 1; + } + if i >= b.len() { + return Err(format!("unterminated string at {}", start + 1)); + } + i += 1; + // Only ASCII bytes were dropped, so v is still valid UTF-8. + out.push(Token { kind: Str, val: String::from_utf8_lossy(&v).into_owned(), pos: start }); + } + b'=' | b'!' | b'<' | b'>' | b'~' | b'&' | b'|' => { + let start = i; + let n = b.get(i + 1).copied(); + let two = n == Some(b'=') || (c == b'&' && n == Some(b'&')) || (c == b'|' && n == Some(b'|')); + let op = &s[i..i + 1 + two as usize]; + i += op.len(); + match op { + "&&" => out.push(tok(Word, "AND", start)), + "||" => out.push(tok(Word, "OR", start)), + "!" => out.push(tok(Word, "NOT", start)), + "&" | "|" => return Err(format!("unexpected {op:?} at {}", start + 1)), + "==" => out.push(tok(Op, "=", start)), + _ => out.push(tok(Op, op, start)), + } + } + _ => { + let start = i; + while i < b.len() && !is_space(b[i]) && !b"()=!<>~,\"'&|".contains(&b[i]) { + i += 1; + } + out.push(tok(Word, &s[start..i], start)); + } + } + } + out.push(tok(Eof, "", s.len())); + Ok(out) +} diff --git a/rust/src/filter/mod.rs b/rust/src/filter/mod.rs new file mode 100644 index 0000000..77dcfb2 --- /dev/null +++ b/rust/src/filter/mod.rs @@ -0,0 +1,127 @@ +//! The search/filter query language: +//! +//! ```text +//! query := or +//! or := and { OR and } +//! and := unary { [AND] unary } (juxtaposition means AND) +//! unary := NOT unary | "(" query ")" | cmp | word +//! cmp := field op value +//! op := = | != | > | >= | < | <= | contains | matches | in | ~ +//! value := word | "quoted string" | "(" value { "," value } ")" +//! ``` +//! +//! A bare word matches the name: substring (case-insensitive), or a glob if it +//! contains * ? or [. Parsing is independent of the UI and fully testable. + +mod apply; +mod eval; +mod lexer; +mod parser; + +pub use apply::{apply, FilterResult}; +pub use eval::Ctx; +pub use parser::{parse, parse_age, parse_size, Field, Query, FIELDS}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::cancel::Cancel; + use crate::inventory::{category_for, ext, Kind, NodeId, SizeMode, Tree}; + use std::collections::HashSet; + use std::sync::Arc; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + const HOUR: Duration = Duration::from_secs(3600); + const DAY: Duration = Duration::from_secs(86400); + + fn tree() -> Tree { + let mut t = Tree::new(b"/srv", Kind::Dir); + let vms = t.add(0, b"vms", Kind::Dir); + let cache = t.add(0, b"cache", Kind::Dir); + let mut add = |p: NodeId, name: &str, size: i64, age: Duration| { + let id = t.add(p, name.as_bytes(), Kind::File); + let n = t.node_mut(id); + n.size = size; + n.alloc = size; + n.mtime = (SystemTime::now() - age).duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64; + n.cat = category_for(&ext(name.as_bytes()), false, false); + }; + add(vms, "ubuntu.qcow2", 80 << 30, 400 * DAY); + add(vms, "win.vmdk", 40 << 30, HOUR); + add(vms, "notes.txt", 1000, HOUR); + add(cache, "blob.iso", 700 << 20, 10 * DAY); + add(cache, "x.tmp", 10, HOUR); + t.recompute(); + t + } + + fn names(tr: &Tree, q: &str) -> HashSet { + let pq = parse(q, SystemTime::now()).unwrap_or_else(|e| panic!("parse({q:?}): {e}")); + let r = apply(&Cancel::new(), tr, Arc::new(pq), SizeMode::Logical).unwrap(); + (0..tr.len() as NodeId).filter(|&i| r.is_match(i)).map(|i| tr.node(i).name_str().into_owned()).collect() + } + + #[test] + fn queries() { + let tr = tree(); + let cases: &[(&str, &[&str])] = &[ + ("ubuntu", &["ubuntu.qcow2"]), + ("*.iso", &["blob.iso"]), + ("size > 1GB", &["vms", "ubuntu.qcow2", "win.vmdk"]), + ("size > 1GB AND type = file", &["ubuntu.qcow2", "win.vmdk"]), + ("ext = qcow2", &["ubuntu.qcow2"]), + ("size > 500MB AND ext IN (iso,qcow2,vmdk)", &["ubuntu.qcow2", "win.vmdk", "blob.iso"]), + ("path contains cache", &["cache", "blob.iso", "x.tmp"]), + ("age > 365d", &["ubuntu.qcow2"]), + ("NOT type = dir AND size < 1k", &["notes.txt", "x.tmp"]), + ("(ext = iso OR ext = tmp) type=file", &["blob.iso", "x.tmp"]), + ("category = vm", &["ubuntu.qcow2", "win.vmdk", "blob.iso"]), + ("name matches '^w.*k$'", &["win.vmdk"]), + ("ext != qcow2 && type = file && size > 1GB", &["win.vmdk"]), + ("name = 'notes.txt'", &["notes.txt"]), + ]; + for (q, want) in cases { + let want: HashSet = want.iter().map(|s| s.to_string()).collect(); + assert_eq!(names(&tr, q), want, "{q:?}"); + } + } + + #[test] + fn filtered_sizes_no_double_count() { + let tr = tree(); + let q = parse("name = vms OR ext = qcow2", SystemTime::now()).unwrap(); + let r = apply(&Cancel::new(), &tr, Arc::new(q), SizeMode::Logical).unwrap(); + assert_eq!(r.total, tr.node(1).tot_size, "matching dir must not double count matching child"); + } + + #[test] + fn parse_errors() { + for q in [ + "", "size >", "size > banana", "(ext = iso", "ext = iso)", "size contains 5", + "name matches '['", "type = blob", "'unterminated", "a & b", "category = nope", + ] { + assert!(parse(q, SystemTime::now()).is_err(), "parse({q:?}) should fail"); + } + } + + #[test] + fn dates_globs_and_non_ascii() { + let tr = tree(); + assert_eq!(names(&tr, "modified < 2000-01-01"), HashSet::new()); + assert_eq!(names(&tr, "modified > '2000-01-01 9:30'").len(), 5); + assert_eq!(names(&tr, "modified > 2000-01-01T00:00:00.5+01:00").len(), 5); + assert_eq!(names(&tr, "modified > 30d").len(), 4); + for bad in ["modified > 2024-02-30", "modified > 2024-01-31T10:00", "ext = 'a['"] { + assert!(parse(bad, SystemTime::now()).is_ok() == bad.ends_with("10:00"), "{bad}"); + } + assert!(names(&tr, "[").is_empty()); // malformed bare glob matches nothing + assert!(names(&tr, "voilà\u{a0}x").is_empty()); // Go's lexer hangs on byte 0xA0 + } + + #[test] + fn parse_size_units() { + for (input, want) in [("1GB", 1_000_000_000), ("1GiB", 1 << 30), ("1.5k", 1536), ("500MB", 500_000_000), ("10", 10)] { + assert_eq!(parse_size(input), Ok(want), "{input}"); + } + } +} diff --git a/rust/src/filter/parser.rs b/rust/src/filter/parser.rs new file mode 100644 index 0000000..ef0bf3d --- /dev/null +++ b/rust/src/filter/parser.rs @@ -0,0 +1,531 @@ +use super::eval::{Expr, NumOp}; +use super::lexer::{lex, TokKind, Token}; +use crate::inventory::{self as inv, Flags}; +use regex::Regex; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// A filterable attribute. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Field { + Name, + Path, + Ext, + Type, + Category, + Size, + Allocated, + Age, + Modified, + Owner, + Group, + Files, + Flag, +} + +fn field_by_name(s: &str) -> Option { + use Field::*; + Some(match s { + "name" => Name, + "path" => Path, + "ext" | "extension" => Ext, + "type" | "kind" => Type, + "category" | "cat" => Category, + "size" => Size, + "allocated" | "alloc" => Allocated, + "age" => Age, + "modified" | "mtime" => Modified, + "owner" | "user" => Owner, + "group" => Group, + "files" => Files, + "flag" | "is" => Flag, + _ => return None, + }) +} + +/// Field names for help text. +pub const FIELDS: &str = "name path ext type category size allocated age modified owner group files flag"; + +/// A compiled filter. +pub struct Query { + src: String, + pub(super) root: Expr, + pub uses_path: bool, +} + +impl std::fmt::Display for Query { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.src) + } +} + +struct Parser { + toks: Vec, + i: usize, + uses_path: bool, + now: SystemTime, +} + +/// Compiles a query. now anchors relative ages (pass SystemTime::now()). +pub fn parse(s: &str, now: SystemTime) -> Result { + let toks = lex(s)?; + let mut p = Parser { toks, i: 0, uses_path: false, now }; + if p.peek().kind == TokKind::Eof { + return Err("empty query".into()); + } + let root = p.or()?; + let t = p.peek(); + if t.kind != TokKind::Eof { + return Err(format!("unexpected {:?} at {}", t.val, t.pos + 1)); + } + Ok(Query { src: s.to_string(), root, uses_path: p.uses_path }) +} + +fn is_kw(t: &Token, kw: &str) -> bool { + t.kind == TokKind::Word && t.val.eq_ignore_ascii_case(kw) +} + +fn is_value(t: &Token) -> bool { + t.kind == TokKind::Word || t.kind == TokKind::Str +} + +impl Parser { + fn peek(&self) -> &Token { + &self.toks[self.i] + } + + fn next(&mut self) -> Token { + let t = self.toks[self.i].clone(); + if t.kind != TokKind::Eof { + self.i += 1; + } + t + } + + fn or(&mut self) -> Result { + let mut l = self.and()?; + while is_kw(self.peek(), "OR") { + self.next(); + let r = self.and()?; + l = Expr::Or(Box::new(l), Box::new(r)); + } + Ok(l) + } + + fn and(&mut self) -> Result { + let mut l = self.unary()?; + loop { + let t = self.peek(); + if is_kw(t, "AND") { + self.next(); + } else if matches!(t.kind, TokKind::Eof | TokKind::RParen) || is_kw(t, "OR") { + return Ok(l); + } + let r = self.unary()?; + l = Expr::And(Box::new(l), Box::new(r)); + } + } + + fn unary(&mut self) -> Result { + let t = self.peek().clone(); + if is_kw(&t, "NOT") { + self.next(); + return Ok(Expr::Not(Box::new(self.unary()?))); + } + if t.kind == TokKind::LParen { + self.next(); + let n = self.or()?; + if self.next().kind != TokKind::RParen { + return Err(format!("missing ) for ( at {}", t.pos + 1)); + } + return Ok(n); + } + if is_value(&t) { + self.next(); + if let Some(f) = field_by_name(&t.val.to_lowercase()) { + if t.kind == TokKind::Word && self.is_op(self.peek()) { + let op = self.next(); + return self.cmp(f, op); + } + } + return Ok(name_match(&t.val)); + } + Err(format!("unexpected {:?} at {}", t.val, t.pos + 1)) + } + + fn is_op(&self, t: &Token) -> bool { + t.kind == TokKind::Op || is_kw(t, "contains") || is_kw(t, "matches") || is_kw(t, "in") + } + + fn values(&mut self) -> Result, String> { + if self.peek().kind != TokKind::LParen { + let t = self.next(); + if !is_value(&t) { + return Err(format!("expected value at {}", t.pos + 1)); + } + return Ok(vec![t]); + } + self.next(); + let mut out = Vec::new(); + loop { + let t = self.next(); + if !is_value(&t) { + return Err(format!("expected value at {}", t.pos + 1)); + } + out.push(t); + let n = self.next(); + match n.kind { + TokKind::Comma => {} + TokKind::RParen => return Ok(out), + _ => return Err(format!("expected , or ) at {}", n.pos + 1)), + } + } + } + + fn cmp(&mut self, f: Field, op_tok: Token) -> Result { + let mut op = op_tok.val.to_lowercase(); + if op == "~" { + op = "matches".into(); + } + let vals = self.values()?; + if op != "in" && vals.len() != 1 { + return Err(format!("operator {op} takes one value")); + } + if f == Field::Path { + self.uses_path = true; + } + if matches!(f, Field::Size | Field::Allocated | Field::Age | Field::Modified | Field::Files) { + let nop = match op.as_str() { + "=" | "in" => NumOp::Eq, + "!=" => NumOp::Ne, + ">" => NumOp::Gt, + ">=" => NumOp::Ge, + "<" => NumOp::Lt, + "<=" => NumOp::Le, + _ => return Err(format!("{op} cannot be used with numeric field")), + }; + let nums = vals.iter().map(|v| self.number(f, &v.val)).collect::, _>>()?; + return Ok(Expr::Num(f, nop, nums)); + } + // String-like fields. + match op.as_str() { + "contains" => Ok(Expr::Contains(f, vals[0].val.to_lowercase())), + "matches" => match Regex::new(&format!("(?i){}", vals[0].val)) { + Ok(re) => Ok(Expr::Re(f, re)), + Err(e) => Err(format!("bad regular expression: {e}")), + }, + "=" | "!=" | "in" => { + let mut set = Vec::new(); + for v in &vals { + let mut s = v.val.to_lowercase(); + match f { + Field::Ext => s = s.strip_prefix('.').unwrap_or(&s).to_string(), + Field::Category => match inv::parse_category(&s) { + Some(c) => s = c.name().to_lowercase(), + None => return Err(format!("unknown category {:?}", v.val)), + }, + Field::Type => s = norm_type(&s)?.to_string(), + Field::Flag if flag_bit(&s).is_none() => { + return Err(format!( + "unknown flag {:?} (sparse, hardlink, error, mount, incomplete, followed, broken)", + v.val + )) + } + _ => {} + } + set.push(s); + } + let n = if f == Field::Flag { + Expr::Flag(set.iter().filter_map(|s| flag_bit(s)).fold(0, |a, b| a | b)) + } else if set.len() == 1 && has_glob(&set[0]) { + if !glob_ok(set[0].as_bytes()) { + return Err(format!("bad pattern {:?}", vals[0].val)); + } + Expr::Glob(f, set.pop().unwrap()) + } else { + Expr::Eq(f, set) + }; + Ok(if op == "!=" { Expr::Not(Box::new(n)) } else { n }) + } + _ => Err(format!("operator {op} not valid for {}", op_tok.val)), + } + } + + /// Parses sizes ("1.5GiB", "500MB", "10k"), ages ("365d", "2w") and dates + /// ("2024-01-31"). Ages and dates are converted to modification times + /// (unix nanoseconds) relative to self.now. + fn number(&self, f: Field, s: &str) -> Result { + match f { + Field::Size | Field::Allocated => parse_size(s), + Field::Files => s.parse::().map_err(|e| format!("bad number {s:?}: {e}")), + Field::Age => parse_age(s).map(nanos), + Field::Modified => { + if let Some(t) = parse_date(s) { + return Ok(t); + } + if let Ok(d) = parse_age(s) { + let now = self.now.duration_since(UNIX_EPOCH).map(nanos).unwrap_or(0); + return Ok(now.saturating_sub(nanos(d))); + } + Err(format!("bad date {s:?} (use YYYY-MM-DD or an age like 30d)")) + } + _ => Err("not numeric".into()), + } + } +} + +fn nanos(d: Duration) -> i64 { + d.as_nanos().min(i64::MAX as u128) as i64 +} + +fn has_glob(s: &str) -> bool { + s.contains(['*', '?', '[']) +} + +pub(super) fn name_match(v: &str) -> Expr { + let lv = v.to_lowercase(); + if has_glob(v) { + // Go's filepath.Match never matches a malformed pattern. + return if glob_ok(lv.as_bytes()) { Expr::Glob(Field::Name, lv) } else { Expr::False }; + } + Expr::Contains(Field::Name, lv) +} + +/// Reports whether pattern is well formed by Go's filepath.Match rules +/// (fnmatch, which does the matching, accepts anything). +fn glob_ok(p: &[u8]) -> bool { + // One possibly escaped character of a range; None if malformed. + let esc = |mut i: usize| -> Option { + match p.get(i)? { + b'-' | b']' => return None, + b'\\' => { + i += 1; + p.get(i)?; + } + _ => {} + } + Some(i + 1) + }; + let mut i = 0; + while i < p.len() { + match p[i] { + b'\\' => { + if i + 1 >= p.len() { + return false; + } + i += 2; + } + b'[' => { + i += 1; + if p.get(i) == Some(&b'^') { + i += 1; + } + let mut n = 0; + loop { + if p.get(i) == Some(&b']') && n > 0 { + i += 1; + break; + } + let Some(j) = esc(i) else { return false }; + i = j; + if p.get(i) == Some(&b'-') { + let Some(j) = esc(i + 1) else { return false }; + i = j; + } + n += 1; + } + } + _ => i += 1, + } + } + true +} + +fn norm_type(s: &str) -> Result<&'static str, String> { + match s { + "f" | "file" => Ok("file"), + "d" | "dir" | "directory" => Ok("directory"), + "l" | "link" | "symlink" => Ok("symlink"), + "other" | "special" => Ok("special"), + _ => Err(format!("unknown type {s:?} (file, dir, symlink, other)")), + } +} + +fn flag_bit(s: &str) -> Option { + Some(match s { + "sparse" => inv::FLAG_SPARSE, + "hardlink" | "hardlinked" => inv::FLAG_HARDLINKED, + "error" => inv::FLAG_ERROR, + "mount" => inv::FLAG_MOUNT_POINT, + "incomplete" => inv::FLAG_INCOMPLETE, + "followed" => inv::FLAG_FOLLOWED, + "broken" => inv::FLAG_BROKEN_LINK, + _ => return None, + }) +} + +/// Splits a leading run of digits and dots from s. +fn split_num(s: &str) -> (&str, &str) { + let i = s.bytes().position(|c| !(c.is_ascii_digit() || c == b'.')).unwrap_or(s.len()); + s.split_at(i) +} + +/// Parses a size with optional unit. kB/MB/GB/TB are SI (powers of 1000); +/// KiB/MiB/GiB/TiB are IEC (1024). Bare K/M/G/T follow the display setting +/// (textutil::set_si). +pub fn parse_size(s: &str) -> Result { + let (num, unit_raw) = split_num(s); + let Ok(v) = num.parse::() else { return Err(format!("bad size {s:?}")) }; + let bare: f64 = if crate::textutil::si() { 1000.0 } else { 1024.0 }; + let m = match unit_raw.trim().to_lowercase().as_str() { + "" | "b" => 1.0, + "k" => bare, + "m" => bare.powi(2), + "g" => bare.powi(3), + "t" => bare.powi(4), + "p" => bare.powi(5), + "kb" => 1e3, + "mb" => 1e6, + "gb" => 1e9, + "tb" => 1e12, + "pb" => 1e15, + "kib" => (1u64 << 10) as f64, + "mib" => (1u64 << 20) as f64, + "gib" => (1u64 << 30) as f64, + "tib" => (1u64 << 40) as f64, + "pib" => (1u64 << 50) as f64, + _ => return Err(format!("unknown size unit {unit_raw:?}")), + }; + let r = v * m; + if r > 9e18 { + return Err(format!("size {s:?} too large")); + } + Ok(r as i64) +} + +/// Parses durations like 90s, 30min, 12h, 7d, 2w, 6mo, 1y. +pub fn parse_age(s: &str) -> Result { + let (num, unit) = split_num(s); + let Ok(v) = num.parse::() else { return Err(format!("bad age {s:?}")) }; + const DAY: f64 = 86400.0; + let secs = match unit.to_lowercase().as_str() { + "s" | "sec" => 1.0, + "min" => 60.0, + "h" => 3600.0, + "d" | "" => DAY, + "w" => 7.0 * DAY, + "m" | "mo" => 30.0 * DAY, + "y" => 365.0 * DAY, + _ => return Err(format!("unknown age unit {unit:?} (s, min, h, d, w, mo, y)")), + }; + // Saturates where Go's time.Duration would overflow. + Ok(Duration::from_nanos((v * secs * 1e9) as u64)) +} + +/// Parses "2006-01-02", "2006-01-02T15:04", "2006-01-02 15:04" (local time) +/// or RFC 3339, as Go's time.ParseInLocation with time.Local would, into unix +/// nanoseconds. +fn parse_date(s: &str) -> Option { + let b = s.as_bytes(); + let fixed = |i: usize, n: usize| -> Option { + let d = b.get(i..i + n)?; + d.iter().all(u8::is_ascii_digit).then(|| d.iter().fold(0, |a, &c| a * 10 + (c - b'0') as i64)) + }; + let (y, mo, d) = (fixed(0, 4)?, fixed(5, 2)?, fixed(8, 2)?); + if b.get(4) != Some(&b'-') || b.get(7) != Some(&b'-') || !(1..=12).contains(&mo) || d < 1 || d > days_in(y, mo) { + return None; + } + if b.len() == 10 { + return local_nanos(y, mo, d, 0, 0); + } + let sep = *b.get(10)?; + if sep != b'T' && sep != b' ' { + return None; + } + // Go's "15" hour accepts one or two digits. + let mut i = 11; + let hl = if fixed(i, 2).is_some() { 2 } else { 1 }; + let h = fixed(i, hl)?; + i += hl; + if b.get(i) != Some(&b':') || h > 23 { + return None; + } + let mi = fixed(i + 1, 2)?; + i += 3; + if mi > 59 { + return None; + } + if i == b.len() { + return local_nanos(y, mo, d, h, mi); + } + // RFC 3339: :05[.frac](Z|±07:00) + if sep != b'T' || b.get(i) != Some(&b':') { + return None; + } + let sec = fixed(i + 1, 2)?; + i += 3; + if sec > 59 { + return None; + } + let mut ns = 0i64; + if matches!(b.get(i), Some(b'.' | b',')) { + let n = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); + if n == 0 { + return None; + } + for k in 0..9 { + ns = ns * 10 + b.get(i + 1 + k).filter(|_| k < n).map_or(0, |c| (c - b'0') as i64); + } + i += 1 + n; + } + let off = match b.get(i)? { + b'Z' => { + i += 1; + 0 + } + &c @ (b'+' | b'-') => { + let (oh, om) = (fixed(i + 1, 2)?, fixed(i + 4, 2)?); + if b.get(i + 3) != Some(&b':') || oh > 23 || om > 59 { + return None; + } + i += 6; + (oh * 3600 + om * 60) * if c == b'-' { -1 } else { 1 } + } + _ => return None, + }; + if i != b.len() { + return None; + } + let secs = days_from_civil(y, mo, d) * 86400 + h * 3600 + mi * 60 + sec - off; + Some(secs * 1_000_000_000 + ns) +} + +fn days_in(y: i64, m: i64) -> i64 { + match m { + 2 if y % 4 == 0 && (y % 100 != 0 || y % 400 == 0) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +fn local_nanos(y: i64, mo: i64, d: i64, h: i64, mi: i64) -> Option { + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + tm.tm_year = (y - 1900) as i32; + tm.tm_mon = (mo - 1) as i32; + tm.tm_mday = d as i32; + tm.tm_hour = h as i32; + tm.tm_min = mi as i32; + tm.tm_isdst = -1; + let t = unsafe { libc::mktime(&mut tm) }; + (t != -1).then(|| t as i64 * 1_000_000_000) +} diff --git a/rust/src/inventory/category.rs b/rust/src/inventory/category.rs new file mode 100644 index 0000000..9b1a2a8 --- /dev/null +++ b/rust/src/inventory/category.rs @@ -0,0 +1,134 @@ +use std::collections::HashMap; +use std::sync::OnceLock; + +/// Coarse file classification used for colouring and stats. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] +#[repr(u8)] +pub enum Category { + #[default] + Other, + Image, + Video, + Audio, + Archive, + DiskImage, + Database, + Code, + Document, + Executable, + Log, + Cache, +} + +pub const NUM_CATEGORIES: usize = 12; + +const CATEGORY_NAMES: [&str; NUM_CATEGORIES] = [ + "Other", "Image", "Video", "Audio", "Archive", "Disk image", "Database", + "Source code", "Document", "Executable", "Log", "Cache", +]; + +const ALL: [Category; NUM_CATEGORIES] = [ + Category::Other, Category::Image, Category::Video, Category::Audio, Category::Archive, + Category::DiskImage, Category::Database, Category::Code, Category::Document, + Category::Executable, Category::Log, Category::Cache, +]; + +impl Category { + pub fn from_u8(v: u8) -> Category { + ALL.get(v as usize).copied().unwrap_or(Category::Other) + } + pub fn all() -> &'static [Category; NUM_CATEGORIES] { + &ALL + } + pub fn name(self) -> &'static str { + CATEGORY_NAMES[self as usize] + } +} + +impl std::fmt::Display for Category { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.name()) + } +} + +/// Resolves a category by (case-insensitive) name or prefix. +pub fn parse_category(s: &str) -> Option { + let s = s.trim().to_lowercase(); + for (i, n) in CATEGORY_NAMES.iter().enumerate() { + let ln = n.to_lowercase(); + if ln == s || ln.replace(' ', "") == s || ln.starts_with(&s) && s.len() >= 3 { + return Some(ALL[i]); + } + } + match s.as_str() { + "vm" | "virtual machine" | "disk" => Some(Category::DiskImage), + "db" => Some(Category::Database), + "src" | "source" => Some(Category::Code), + "exe" | "binary" => Some(Category::Executable), + _ => None, + } +} + +fn ext_table() -> &'static HashMap<&'static str, Category> { + static T: OnceLock> = OnceLock::new(); + T.get_or_init(|| { + let mut m = HashMap::new(); + let mut add = |c: Category, exts: &'static str| { + for e in exts.split_whitespace() { + m.insert(e, c); + } + }; + add(Category::Image, "jpg jpeg png gif bmp tif tiff webp heic heif raw cr2 cr3 nef arw dng svg ico psd xcf avif jxl orf rw2"); + add(Category::Video, "mp4 mkv mov avi wmv flv webm m4v mpg mpeg ts m2ts vob 3gp ogv mts"); + add(Category::Audio, "mp3 flac wav ogg oga opus m4a aac wma aiff aif alac mid midi"); + add(Category::Archive, "zip tar gz tgz bz2 tbz xz txz zst zstd 7z rar lz lz4 lzma cab deb rpm apk jar war cpio pkg snap whl gem"); + add(Category::DiskImage, "iso img qcow qcow2 vmdk vdi vhd vhdx hdd ova ovf dmg raw vmem vmsn nvram sav"); + add(Category::Database, "db sqlite sqlite3 db3 mdb accdb frm ibd myd myi ldf mdf ndf dbf wal kdbx realm lmdb rdb aof"); + add(Category::Code, "c h cc cpp cxx hpp hh go rs py pyc js mjs cjs ts tsx jsx java kt kts scala rb php pl pm sh bash zsh fish lua swift m mm cs fs vb r jl hs ml ex exs erl clj dart sql css scss sass less html htm vue svelte json yaml yml toml xml proto gradle cmake mk ipynb o a lib obj class"); + add(Category::Document, "pdf doc docx odt rtf txt md rst tex xls xlsx ods csv tsv ppt pptx odp epub mobi djvu pages numbers key org adoc"); + add(Category::Executable, "exe dll so dylib bin msi app elf ko sys efi appimage com"); + add(Category::Log, "log journal"); + add(Category::Cache, "cache tmp temp swp bak old part crdownload"); + m + }) +} + +/// Classifies a file by its (lower-case) extension and by whether it lives +/// beneath a cache or log directory. +pub fn category_for(ext: &str, in_cache: bool, in_log: bool) -> Category { + let c = ext_table().get(ext).copied(); + match c { + _ if in_cache && matches!(c, None | Some(Category::Other) | Some(Category::Code)) => Category::Cache, + _ if in_log && matches!(c, None | Some(Category::Archive) | Some(Category::Other)) => Category::Log, + Some(c) => c, + None => Category::Other, + } +} + +/// Reports whether a directory path contains a cache or log component. It is +/// evaluated once per directory, not per file. +pub fn path_hints(path: &[u8]) -> (bool, bool) { + let (mut in_cache, mut in_log) = (false, false); + for part in path.split(|&b| b == b'/' || b == b'\\') { + if part.is_empty() || part.len() > 12 { + continue; + } + match part.to_ascii_lowercase().as_slice() { + b"cache" | b".cache" | b"caches" | b"__pycache__" | b".npm" | b".gradle" | b"inetcache" | b"temp" | b"tmp" => in_cache = true, + b"log" | b"logs" | b".logs" | b"journal" => in_log = true, + _ => {} + } + } + (in_cache, in_log) +} + +/// Returns the lower-case extension of name without the dot ("" if none, or +/// if the name is a dotfile like ".bashrc"). +pub fn ext(name: &[u8]) -> String { + match name.iter().rposition(|&b| b == b'.') { + Some(i) if i > 0 && i != name.len() - 1 && name.len() - i <= 16 => { + String::from_utf8_lossy(&name[i + 1..]).to_lowercase() + } + _ => String::new(), + } +} diff --git a/rust/src/inventory/mod.rs b/rust/src/inventory/mod.rs new file mode 100644 index 0000000..4905241 --- /dev/null +++ b/rust/src/inventory/mod.rs @@ -0,0 +1,481 @@ +//! The in-memory filesystem model: an append-only arena of nodes addressed +//! by stable NodeIds, with incrementally maintained directory aggregates. +//! +//! Concurrency: a Tree has exactly one writer (the scan controller or a +//! snapshot loader), shared as `Arc>`. Nodes are never removed, +//! so a NodeId stays valid for the Tree's lifetime. + +mod category; +mod query; +mod stats; + +pub use category::*; +pub use query::*; +pub use stats::*; + +use std::collections::HashMap; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::PathBuf; + +/// Addresses a node within one Tree. The root is always 0. +pub type NodeId = u32; + +/// The nil NodeId. +pub const NO_NODE: NodeId = u32::MAX; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum Kind { + #[default] + File, + Dir, + Symlink, + Other, // devices, sockets, fifos, reparse points +} + +impl Kind { + pub fn from_u8(v: u8) -> Kind { + [Kind::File, Kind::Dir, Kind::Symlink, Kind::Other][(v & 3) as usize] + } + pub fn name(self) -> &'static str { + ["File", "Directory", "Symlink", "Special"][self as usize] + } +} + +/// Per-node facts (bit set). +pub type Flags = u16; +pub const FLAG_HARDLINK_DUP: Flags = 1 << 0; // additional link to an inode counted elsewhere +pub const FLAG_MOUNT_POINT: Flags = 1 << 1; // directory on a different device than its parent +pub const FLAG_VIRTUAL_FS: Flags = 1 << 2; // virtual filesystem, not descended +pub const FLAG_SKIPPED_FS: Flags = 1 << 3; // other filesystem, not descended (--one-file-system) +pub const FLAG_SPARSE: Flags = 1 << 4; // allocated < logical (sparse or compressed) +pub const FLAG_ERROR: Flags = 1 << 5; // stat or read failed +pub const FLAG_INCOMPLETE: Flags = 1 << 6; // directory contents not (fully) read +pub const FLAG_SCANNED: Flags = 1 << 7; // directory fully read +pub const FLAG_LOOP: Flags = 1 << 8; // directory already visited via another path +pub const FLAG_FOLLOWED: Flags = 1 << 9; // reached through a followed symlink +pub const FLAG_ALLOC_UNKNOWN: Flags = 1 << 10; // platform cannot report allocated size +pub const FLAG_BROKEN_LINK: Flags = 1 << 11; // symlink target does not exist +pub const FLAG_HARDLINKED: Flags = 1 << 12; // regular file with link count > 1 +pub const FLAG_DELETED: Flags = 1 << 13; // moved to trash from the UI + +/// One filesystem object. Sizes are own sizes; tot_* fields are aggregates +/// including the node itself and (for directories) everything beneath it. +#[derive(Clone, Debug)] +pub struct Node { + pub parent: NodeId, + pub first_child: NodeId, + pub next_sibling: NodeId, + dir: u32, // index into dir side table (directories only) + + /// Single path component (raw bytes); the root holds the full root path. + pub name: Box<[u8]>, + + pub size: i64, // own logical bytes + pub alloc: i64, // own allocated bytes + pub tot_size: i64, // aggregates (hard-link duplicates contribute 0) + pub tot_alloc: i64, + pub mtime: i64, // unix nanoseconds + + pub files: u32, // recursive counts beneath (dirs excludes self) + pub dirs: u32, + pub errors: u32, // recursive error count beneath and including self + pub mode: u32, // Go fs.FileMode bit layout (see platform::mode) + pub uid: u32, + pub gid: u32, + pub nlink: u32, + + pub ext: u16, + pub flags: Flags, + pub kind: Kind, + pub cat: Category, +} + +impl Node { + pub fn is_dir(&self) -> bool { + self.kind == Kind::Dir + } + pub fn has(&self, f: Flags) -> bool { + self.flags & f != 0 + } + /// The name as (lossy) UTF-8, unsanitised. + pub fn name_str(&self) -> std::borrow::Cow<'_, str> { + String::from_utf8_lossy(&self.name) + } +} + +/// An aggregate change applied to a node and all its ancestors. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct Delta { + pub size: i64, + pub alloc: i64, + pub files: i64, + pub dirs: i64, + pub errors: i64, + pub cat: [i64; NUM_CATEGORIES], +} + +const CHUNK_BITS: u32 = 16; +const CHUNK_SIZE: usize = 1 << CHUNK_BITS; + +/// A growable array that never moves existing elements, so growth never +/// doubles peak memory. +#[derive(Clone)] +struct Chunked { + c: Vec>, + n: usize, +} + +impl Chunked { + fn new() -> Self { + Chunked { c: Vec::new(), n: 0 } + } + fn add(&mut self, v: T) -> u32 { + if self.n & (CHUNK_SIZE - 1) == 0 && self.n >> CHUNK_BITS == self.c.len() { + self.c.push(Vec::with_capacity(CHUNK_SIZE)); + } + self.c[self.n >> CHUNK_BITS].push(v); + self.n += 1; + (self.n - 1) as u32 + } + #[inline] + fn at(&self, i: u32) -> &T { + &self.c[(i >> CHUNK_BITS) as usize][i as usize & (CHUNK_SIZE - 1)] + } + #[inline] + fn at_mut(&mut self, i: u32) -> &mut T { + &mut self.c[(i >> CHUNK_BITS) as usize][i as usize & (CHUNK_SIZE - 1)] + } +} + +/// The inventory for one scan root. +#[derive(Clone)] +pub struct Tree { + nodes: Chunked, + dirs: Chunked<[i64; NUM_CATEGORIES]>, + exts: Vec, + ext_idx: HashMap, + pub stats: Stats, +} + +impl Tree { + /// Creates a tree whose root has the given full path and kind. + pub fn new(root_path: &[u8], kind: Kind) -> Tree { + let mut t = Tree { + nodes: Chunked::new(), + dirs: Chunked::new(), + exts: vec![String::new()], + ext_idx: HashMap::from([(String::new(), 0)]), + stats: Stats::default(), + }; + t.stats.root = root_path.to_vec(); + t.stats.start = std::time::SystemTime::now(); + t.add(NO_NODE, root_path, kind); + t + } + + pub fn len(&self) -> usize { + self.nodes.n + } + pub fn is_empty(&self) -> bool { + self.nodes.n == 0 + } + #[inline] + pub fn node(&self, id: NodeId) -> &Node { + self.nodes.at(id) + } + #[inline] + pub fn node_mut(&mut self, id: NodeId) -> &mut Node { + self.nodes.at_mut(id) + } + pub fn root(&self) -> NodeId { + 0 + } + + /// Appends a child node and links it under parent. It does not touch + /// aggregates; use propagate. + pub fn add(&mut self, parent: NodeId, name: &[u8], kind: Kind) -> NodeId { + let mut n = Node { + parent, + first_child: NO_NODE, + next_sibling: NO_NODE, + dir: 0, + name: name.into(), + size: 0, alloc: 0, tot_size: 0, tot_alloc: 0, mtime: 0, + files: 0, dirs: 0, errors: 0, mode: 0, uid: 0, gid: 0, nlink: 0, + ext: 0, flags: 0, kind, cat: Category::Other, + }; + if kind == Kind::Dir { + n.dir = self.dirs.add([0; NUM_CATEGORIES]); + } else { + n.ext = self.intern_ext(&ext(name)); + } + let id = self.nodes.add(n); + if parent != NO_NODE { + let first = self.node(parent).first_child; + self.node_mut(id).next_sibling = first; + self.node_mut(parent).first_child = id; + } + id + } + + /// Interns an extension string; used by loaders rebuilding a tree. + pub fn intern_ext(&mut self, e: &str) -> u16 { + if let Some(&i) = self.ext_idx.get(e) { + return i; + } + if self.exts.len() >= (1 << 16) - 1 { + return 0; + } + self.exts.push(e.to_string()); + let i = (self.exts.len() - 1) as u16; + self.ext_idx.insert(e.to_string(), i); + i + } + + /// The extension string for a node ("" if none). + pub fn ext_name(&self, n: &Node) -> &str { + &self.exts[n.ext as usize] + } + pub fn ext_count(&self) -> usize { + self.exts.len() + } + pub fn ext_by_index(&self, i: usize) -> &str { + &self.exts[i] + } + + /// Per-category allocated bytes beneath a directory. + pub fn cat_sizes(&self, id: NodeId) -> Option<&[i64; NUM_CATEGORIES]> { + let n = self.node(id); + (n.kind == Kind::Dir).then(|| self.dirs.at(n.dir)) + } + + /// Adds d to id and every ancestor. Cost is O(depth); callers batch many + /// entries into one Delta. + pub fn propagate(&mut self, mut id: NodeId, d: &Delta) { + while id != NO_NODE { + let n = self.nodes.at_mut(id); + n.tot_size += d.size; + n.tot_alloc += d.alloc; + n.files = add_u32(n.files, d.files); + n.dirs = add_u32(n.dirs, d.dirs); + n.errors = add_u32(n.errors, d.errors); + let (is_dir, dir, parent) = (n.kind == Kind::Dir, n.dir, n.parent); + if is_dir { + let cs = self.dirs.at_mut(dir); + for (c, v) in cs.iter_mut().zip(d.cat) { + *c += v; + } + } + id = parent; + } + } + + /// Iterates the children of id (in reverse insertion order). + pub fn children(&self, id: NodeId) -> Children<'_> { + Children { t: self, cur: self.node(id).first_child } + } + + pub fn child_count(&self, id: NodeId) -> usize { + self.children(id).count() + } + + /// Reconstructs the full path of id as raw bytes. + pub fn path_bytes(&self, mut id: NodeId) -> Vec { + let mut parts: Vec<&[u8]> = Vec::new(); + while id != NO_NODE { + let n = self.node(id); + parts.push(&n.name); + id = n.parent; + } + let mut out = Vec::with_capacity(parts.iter().map(|p| p.len() + 1).sum()); + for p in parts.iter().rev() { + if !out.is_empty() && out.last() != Some(&b'/') { + out.push(b'/'); + } + out.extend_from_slice(p); + } + out + } + + /// Reconstructs the full path of id. + pub fn path(&self, id: NodeId) -> PathBuf { + PathBuf::from(OsStr::from_bytes(&self.path_bytes(id))) + } + + /// The full path as (lossy) UTF-8, unsanitised. + pub fn path_string(&self, id: NodeId) -> String { + String::from_utf8_lossy(&self.path_bytes(id)).into_owned() + } + + /// The chain root..id inclusive. + pub fn ancestors(&self, mut id: NodeId) -> Vec { + let mut out = Vec::new(); + while id != NO_NODE { + out.push(id); + id = self.node(id).parent; + } + out.reverse(); + out + } + + /// Reports whether a is id or an ancestor of id. + pub fn is_ancestor(&self, a: NodeId, mut id: NodeId) -> bool { + while id != NO_NODE { + if id == a { + return true; + } + id = self.node(id).parent; + } + false + } + + /// Number of ancestors of id. + pub fn depth(&self, id: NodeId) -> usize { + let mut d = 0; + let mut id = self.node(id).parent; + while id != NO_NODE { + d += 1; + id = self.node(id).parent; + } + d + } + + /// Rebuilds all aggregates bottom-up. Relies on a parent's ID being + /// smaller than its children's, which both the scanner and the snapshot + /// loader guarantee. Used after loading a snapshot. + pub fn recompute(&mut self) { + for i in 0..self.len() as NodeId { + let n = self.nodes.at_mut(i); + n.tot_size = 0; + n.tot_alloc = 0; + n.files = 0; + n.dirs = 0; + n.errors = (n.flags & FLAG_ERROR != 0) as u32; + if n.flags & FLAG_HARDLINK_DUP == 0 { + n.tot_size = n.size; + n.tot_alloc = n.alloc; + } + if n.kind == Kind::Dir { + let d = n.dir; + *self.dirs.at_mut(d) = [0; NUM_CATEGORIES]; + } + } + // Sums saturate: snapshot data is untrusted and must not wrap to + // negative totals. + for i in (1..self.len() as NodeId).rev() { + let n = self.nodes.at(i).clone_shallow(); + let p = self.nodes.at_mut(n.parent); + p.tot_size = sat_add(p.tot_size, n.tot_size); + p.tot_alloc = sat_add(p.tot_alloc, n.tot_alloc); + p.errors = add_u32(p.errors, n.errors as i64); + let pdir = p.dir; + if n.kind == Kind::Dir { + p.files = add_u32(p.files, n.files as i64); + p.dirs = add_u32(p.dirs, n.dirs as i64 + 1); + let child = *self.dirs.at(n.dir); + let cs = self.dirs.at_mut(pdir); + for (c, v) in cs.iter_mut().zip(child) { + *c = sat_add(*c, v); + } + } else { + p.files = add_u32(p.files, 1); + if n.flags & FLAG_HARDLINK_DUP == 0 { + let cs = self.dirs.at_mut(pdir); + cs[n.cat as usize] = sat_add(cs[n.cat as usize], n.alloc); + } + } + } + } +} + +/// The numeric fields recompute needs, without cloning the name. +struct Shallow { + parent: NodeId, + dir: u32, + tot_size: i64, + tot_alloc: i64, + alloc: i64, + files: u32, + dirs: u32, + errors: u32, + flags: Flags, + kind: Kind, + cat: Category, +} + +impl Node { + fn clone_shallow(&self) -> Shallow { + Shallow { + parent: self.parent, dir: self.dir, tot_size: self.tot_size, tot_alloc: self.tot_alloc, + alloc: self.alloc, files: self.files, dirs: self.dirs, errors: self.errors, + flags: self.flags, kind: self.kind, cat: self.cat, + } + } +} + +pub struct Children<'a> { + t: &'a Tree, + cur: NodeId, +} + +impl<'a> Iterator for Children<'a> { + type Item = (NodeId, &'a Node); + fn next(&mut self) -> Option { + if self.cur == NO_NODE { + return None; + } + let id = self.cur; + let n = self.t.node(id); + self.cur = n.next_sibling; + Some((id, n)) + } +} + +/// Adds non-negative i64s, saturating at the maximum. +pub fn sat_add(a: i64, b: i64) -> i64 { + if b > 0 && a > i64::MAX - b { + return i64::MAX; + } + a + b +} + +fn add_u32(a: u32, d: i64) -> u32 { + (a as i64 + d).clamp(0, u32::MAX as i64) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn topk_and_recompute() { + let mut tr = Tree::new(b"/r", Kind::Dir); + let d = tr.add(0, b"d", Kind::Dir); + for i in 0..100 { + let id = tr.add(d, b"f", Kind::File); + let n = tr.node_mut(id); + n.size = i; + n.alloc = i; + } + tr.recompute(); + assert_eq!(tr.node(0).tot_size, 4950); + assert_eq!((tr.node(0).files, tr.node(0).dirs), (100, 1)); + let top = tr.top(0, 3, |n| n.kind == Kind::File, |n| n.size); + assert_eq!(top.len(), 3); + assert_eq!(tr.node(top[0]).size, 99); + assert_eq!(tr.node(top[2]).size, 97); + assert_eq!(tr.path_string(top[0]), "/r/d/f"); + } + + #[test] + fn ext_and_category() { + for (input, want) in [("a.QCOW2", "qcow2"), (".bashrc", ""), ("noext", ""), ("x.tar.gz", "gz"), ("trail.", "")] { + assert_eq!(ext(input.as_bytes()), want, "{input}"); + } + assert_eq!(category_for("qcow2", false, false), Category::DiskImage); + assert_eq!(category_for("bin", true, false), Category::Executable); + assert_eq!(category_for("", true, false), Category::Cache); + } +} diff --git a/rust/src/inventory/query.rs b/rust/src/inventory/query.rs new file mode 100644 index 0000000..cb02600 --- /dev/null +++ b/rust/src/inventory/query.rs @@ -0,0 +1,141 @@ +use super::*; +use std::cmp::Reverse; +use std::collections::BinaryHeap; + +/// Which size drives layout and sorting. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum SizeMode { + #[default] + Allocated, // disk usage + Logical, // apparent size +} + +impl SizeMode { + pub fn name(self) -> &'static str { + match self { + SizeMode::Logical => "logical", + SizeMode::Allocated => "allocated", + } + } +} + +impl Node { + /// The aggregate size in the given mode. + #[inline] + pub fn total(&self, m: SizeMode) -> i64 { + match m { + SizeMode::Logical => self.tot_size, + SizeMode::Allocated => self.tot_alloc, + } + } + /// The node's own size in the given mode. + #[inline] + pub fn own(&self, m: SizeMode) -> i64 { + match m { + SizeMode::Logical => self.size, + SizeMode::Allocated => self.alloc, + } + } +} + +/// Keeps the k entries with the largest keys (ties: smaller id wins); O(n log k). +pub struct TopK { + k: usize, + // Min-heap on (key, Reverse(id)): the root is the weakest entry. + h: BinaryHeap)>>, +} + +impl TopK { + pub fn new(k: usize) -> TopK { + TopK { k, h: BinaryHeap::with_capacity(k.min(1 << 16)) } + } + + pub fn offer(&mut self, id: NodeId, key: i64) { + if self.k == 0 { + return; + } + let e = Reverse((key, Reverse(id))); + if self.h.len() < self.k { + self.h.push(e); + } else if let Some(mut top) = self.h.peek_mut() { + if e < *top { + *top = e; + } + } + } + + /// The collected IDs, largest key first (ties by ID). + pub fn sorted(self) -> Vec { + // into_sorted_vec is ascending in Reverse order = descending key. + self.h.into_sorted_vec().into_iter().map(|Reverse((_, Reverse(id)))| id).collect() + } +} + +impl Tree { + /// Visits id and all its descendants depth-first (iteratively). fn + /// returns false to skip a node's children. + pub fn walk(&self, id: NodeId, mut f: impl FnMut(NodeId, &Node) -> bool) { + let mut stack = vec![id]; + while let Some(cur) = stack.pop() { + let n = self.node(cur); + if !f(cur, n) { + continue; + } + stack.extend(self.children(cur).map(|(c, _)| c)); + } + } + + /// The k nodes beneath (and including) root with the largest key, + /// considering only nodes accepted by keep. + pub fn top(&self, root: NodeId, k: usize, keep: impl Fn(&Node) -> bool, key: impl Fn(&Node) -> i64) -> Vec { + let mut tk = TopK::new(k); + self.walk(root, |id, n| { + if keep(n) { + tk.offer(id, key(n)); + } + true + }); + tk.sorted() + } + + /// The children of id ordered by size descending (ties by ID). + pub fn sorted_children(&self, id: NodeId, m: SizeMode) -> Vec { + let mut out: Vec<(i64, NodeId)> = self.children(id).map(|(c, n)| (n.total(m), c)).collect(); + out.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + out.into_iter().map(|(_, c)| c).collect() + } + + /// Aggregates file statistics by extension beneath root. + pub fn ext_stats(&self, root: NodeId) -> Vec { + let mut stats = vec![ExtStat::default(); self.ext_count()]; + self.walk(root, |_, n| { + if n.kind == Kind::File && n.flags & FLAG_HARDLINK_DUP == 0 { + let s = &mut stats[n.ext as usize]; + s.count += 1; + s.size += n.size; + s.alloc += n.alloc; + } + true + }); + stats + .into_iter() + .enumerate() + .filter(|(_, s)| s.count > 0) + .map(|(i, mut s)| { + s.ext = self.exts[i].clone(); + s.cat = category_for(&s.ext, false, false); + s + }) + .collect() + } +} + +/// Summarises one extension. +#[derive(Clone, Default, Debug)] +pub struct ExtStat { + pub ext: String, + pub cat: Category, + pub count: i64, + pub size: i64, // logical + pub alloc: i64, +} diff --git a/rust/src/inventory/stats.rs b/rust/src/inventory/stats.rs new file mode 100644 index 0000000..a5d0d2d --- /dev/null +++ b/rust/src/inventory/stats.rs @@ -0,0 +1,124 @@ +use super::NodeId; +use std::io; +use std::time::{Duration, SystemTime}; + +/// Classifies scan errors for reporting. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum ErrKind { + Permission, + Vanished, + Io, + Loop, + #[default] + Other, +} + +pub const NUM_ERR_KINDS: usize = 5; + +impl ErrKind { + pub fn from_u8(v: u8) -> ErrKind { + [ErrKind::Permission, ErrKind::Vanished, ErrKind::Io, ErrKind::Loop, ErrKind::Other] + .get(v as usize) + .copied() + .unwrap_or(ErrKind::Other) + } + pub fn name(self) -> &'static str { + ["Permission denied", "Vanished during scan", "I/O error", "Symlink loop", "Other error"][self as usize] + } +} + +/// Maps an OS error to an ErrKind. +pub fn classify(err: &io::Error) -> ErrKind { + match err.raw_os_error() { + Some(libc::EACCES) | Some(libc::EPERM) => ErrKind::Permission, + Some(libc::ENOENT) => ErrKind::Vanished, + Some(libc::ELOOP) => ErrKind::Loop, + Some(libc::EIO) => ErrKind::Io, + _ => match err.kind() { + io::ErrorKind::PermissionDenied => ErrKind::Permission, + io::ErrorKind::NotFound => ErrKind::Vanished, + _ => ErrKind::Other, + }, + } +} + +/// One recorded scan error. +#[derive(Clone, Debug, Default)] +pub struct ErrorRecord { + pub node: NodeId, // node the error is attached to (the directory for listing failures) + pub name: Vec, // entry name when no node was created (e.g. vanished) + pub kind: ErrKind, + pub msg: String, +} + +/// Caps the detailed error list; counts are always exact. +pub const MAX_ERROR_RECORDS: usize = 100_000; + +/// Describes a scan as a whole. +#[derive(Clone, Debug)] +pub struct Stats { + pub root: Vec, + pub start: SystemTime, + pub end: Option, + pub complete: bool, // scan finished (possibly with errors) + pub cancelled: bool, + + pub files: i64, + pub dirs: i64, + pub symlinks: i64, + pub others: i64, + + pub err_counts: [i64; NUM_ERR_KINDS], + pub errors: Vec, + pub excluded: i64, + pub skipped_mounts: i64, + pub virtual_skipped: i64, + pub loops_skipped: i64, + pub broken_links: i64, + pub hardlink_dups: i64, + pub unscanned: i64, // directories never read because the scan was cancelled + + pub excludes: Vec, + pub one_file_system: bool, + pub follow: String, + pub workers: usize, + pub from_snapshot: String, +} + +impl Default for Stats { + fn default() -> Self { + Stats { + root: Vec::new(), start: SystemTime::now(), end: None, complete: false, cancelled: false, + files: 0, dirs: 0, symlinks: 0, others: 0, err_counts: [0; NUM_ERR_KINDS], errors: Vec::new(), + excluded: 0, skipped_mounts: 0, virtual_skipped: 0, loops_skipped: 0, broken_links: 0, + hardlink_dups: 0, unscanned: 0, excludes: Vec::new(), one_file_system: false, + follow: String::new(), workers: 0, from_snapshot: String::new(), + } + } +} + +impl Stats { + pub fn total_errors(&self) -> i64 { + self.err_counts.iter().sum() + } + + /// Records an error (the detailed list is capped). + pub fn add_error(&mut self, r: ErrorRecord) { + self.err_counts[r.kind as usize] += 1; + if self.errors.len() < MAX_ERROR_RECORDS { + self.errors.push(r); + } + } + + /// The scan duration so far (or total once finished). + pub fn elapsed(&self) -> Duration { + let end = self.end.unwrap_or_else(SystemTime::now); + end.duration_since(self.start).unwrap_or_default() + } + + /// Whether totals may under-count real usage. + pub fn incomplete(&self) -> bool { + !self.complete || self.cancelled || self.total_errors() > 0 || self.unscanned > 0 + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..2a8cf38 --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,19 @@ +//! mapsize: interactive terminal disk usage analyser (Rust port). + +pub mod app; +pub mod brand; +pub mod cancel; +pub mod config; +pub mod duplicate; +pub mod export; +pub mod filter; +pub mod inventory; +pub mod platform; +pub mod scan; +pub mod snapshot; +pub mod textutil; +pub mod treemap; +pub mod tui; + +/// Boxed error used across mode boundaries. +pub type Error = Box; diff --git a/rust/src/main.rs b/rust/src/main.rs new file mode 100644 index 0000000..c32a9ac --- /dev/null +++ b/rust/src/main.rs @@ -0,0 +1,189 @@ +//! mapsize: interactive terminal disk usage analyser. + +use mapsize::app::{self, Config}; +use mapsize::inventory::SizeMode; +use mapsize::{brand, config, scan, snapshot, textutil}; +use std::path::Path; + +/// Flags taking a value (Go's flag package: `-f v`, `-f=v`, `--f v`, `--f=v`). +const VALUE_FLAGS: &[&str] = &[ + "workers", "follow-symlinks", "depth", "top", "largest-files", "save", "snapshot", "load", "scan", + "theme", "color", "log", "log-level", "exclude", +]; +/// Boolean flags (`-f`, `-f=false`). +const BOOL_FLAGS: &[&str] = &["one-file-system", "x", "read-only", "no-ui", "json", "csv", "duplicates", "compare", "si", "apparent", "no-mouse", "version"]; + +fn usage() { + let n = brand::NAME; + let e = brand::SNAPSHOT_EXT; + eprint!( + "{n} — interactive terminal disk usage analyser + +Usage: + {n} [flags] [PATH] scan PATH (default .) interactively + {n} [flags] SNAPSHOT{e} open a saved snapshot + {n} --compare OLD NEW compare two snapshots + {n} PATH --top 50 | --json | --csv | --no-ui --save FILE + +Flags: + --workers MODE concurrency: conservative, balanced, aggressive or a number + --follow-symlinks MODE none, same-filesystem or all + -x, --one-file-system do not cross filesystem boundaries + --exclude PATTERN exclude PATTERN (glob on name, or path prefix if it contains a separator); repeatable + --read-only disable all modifying actions + --no-ui scan without the interactive interface and print a summary + --json, --csv print the inventory as JSON / CSV + --depth N maximum depth for --json/--csv (-1 = unlimited) + --top N print the N largest entries of the root + --largest-files N print the N largest files + --duplicates find duplicate files (non-interactive) + --save FILE save a snapshot to FILE after scanning (alias --snapshot) + --load FILE open a saved snapshot FILE instead of scanning + --scan PATH path to scan (alternative to a positional argument) + --compare compare two snapshots: --compare OLD NEW + --si use SI units (kB, MB) instead of IEC (KiB, MiB) + --apparent size by logical (apparent) size instead of disk usage + --theme NAME colour theme: default, dark, high-contrast, mono + --color MODE colour support: auto, truecolor, 256, 16, none + --no-mouse disable mouse support + --log FILE write a debug log to FILE + --log-level LEVEL log level: error, warn, info, debug, trace + --version print version and exit +" + ); +} + +fn main() { + std::process::exit(run(std::env::args_os().skip(1).map(|a| a.to_string_lossy().into_owned()).collect())); +} + +fn fail(msg: impl std::fmt::Display) -> i32 { + eprintln!("{}: {msg}", brand::NAME); + 2 +} + +fn run(args: Vec) -> i32 { + let (settings, cfg_err) = config::load(); + let mut vals: std::collections::HashMap<&str, String> = std::collections::HashMap::new(); + let mut bools: std::collections::HashMap<&str, bool> = std::collections::HashMap::new(); + let mut excludes = settings.excludes.clone(); + let mut positional = Vec::new(); + let mut it = args.into_iter(); + while let Some(a) = it.next() { + if a == "--" { + positional.extend(it.by_ref()); + break; + } + let Some(flag) = a.strip_prefix("--").or_else(|| a.strip_prefix('-')).filter(|f| !f.is_empty() && a != "-") else { + positional.push(a); + continue; + }; + let (name, inline) = match flag.split_once('=') { + Some((n, v)) => (n, Some(v.to_string())), + None => (flag, None), + }; + if name == "h" || name == "help" { + usage(); + return 0; + } + if let Some(&n) = BOOL_FLAGS.iter().find(|&&f| f == name) { + match inline.as_deref().map(str::parse::) { + None => bools.insert(n, true), + Some(Ok(v)) => bools.insert(n, v), + Some(Err(_)) => return fail(format!("invalid boolean value {:?} for -{n}", inline.unwrap())), + }; + } else if let Some(&n) = VALUE_FLAGS.iter().find(|&&f| f == name) { + let Some(v) = inline.or_else(|| it.next()) else { return fail(format!("flag needs an argument: -{n}")) }; + if n == "exclude" { + excludes.push(v); + } else { + vals.insert(n, v); + } + } else { + usage(); + return fail(format!("flag provided but not defined: -{name}")); + } + } + let b = |n: &str| bools.get(n).copied(); + let v = |n: &str| vals.get(n).cloned(); + let int = |n: &str, d: i64| -> Result { + v(n).map_or(Ok(d), |s| s.parse().map_err(|_| format!("invalid value {s:?} for flag -{n}: parse error"))) + }; + + if b("version") == Some(true) { + println!("{} {} ({}/{})", brand::NAME, brand::VERSION, std::env::consts::OS, std::env::consts::ARCH); + return 0; + } + if let Some(e) = cfg_err { + eprintln!("{}: warning: config: {e} (using defaults)", brand::NAME); + } + textutil::set_si(b("si").unwrap_or(settings.units == "si")); + if let Some(l) = v("log-level") { + if !["error", "warn", "info", "debug", "trace"].contains(&l.as_str()) { + return fail(format!("invalid --log-level {l:?}")); + } + } + // ponytail: --log is accepted for CLI compatibility; the Rust port has no debug logging yet. + + let (depth, top, largest) = match (int("depth", -1), int("top", 0), int("largest-files", 0)) { + (Ok(d), Ok(t), Ok(l)) => (d, t.max(0) as usize, l.max(0) as usize), + (Err(e), _, _) | (_, Err(e), _) | (_, _, Err(e)) => return fail(e), + }; + let mut cfg = Config { + no_ui: b("no-ui").unwrap_or(false), + json: b("json").unwrap_or(false), + csv: b("csv").unwrap_or(false), + depth, + top, + largest_files: largest, + duplicates: b("duplicates").unwrap_or(false), + read_only: b("read-only").unwrap_or(false), + theme: v("theme").unwrap_or_else(|| settings.theme.clone()), + mouse: !b("no-mouse").unwrap_or(!settings.mouse), + color: v("color").unwrap_or_else(|| "auto".into()), + load: v("load").unwrap_or_default(), + save: v("save").or_else(|| v("snapshot")).unwrap_or_default(), + size_mode: if b("apparent").unwrap_or(settings.size_mode == "logical") { SizeMode::Logical } else { SizeMode::Allocated }, + ..Config::new(settings.clone()) + }; + if let Some(p) = v("scan") { + positional.insert(0, p); + } + if b("compare") == Some(true) { + if positional.len() != 2 { + return fail("--compare needs exactly two snapshot files"); + } + cfg.compare = Some((positional[0].clone(), positional[1].clone())); + } else { + if positional.len() > 1 { + return fail("only one path may be scanned at a time"); + } + if positional.len() == 1 && cfg.load.is_empty() && snapshot::is_snapshot(Path::new(&positional[0])) { + cfg.load = positional.remove(0); + } else { + cfg.paths = positional; + } + } + let workers = match scan::workers(&v("workers").unwrap_or_else(|| settings.workers.clone())) { + Ok(w) => w, + Err(e) => return fail(e), + }; + let follow = match scan::parse_follow(&v("follow-symlinks").unwrap_or_else(|| settings.follow_symlinks.clone())) { + Ok(f) => f, + Err(e) => return fail(e), + }; + cfg.scan_opts = scan::Options { + workers, + follow, + one_file_system: b("one-file-system").unwrap_or(settings.one_file_system) || b("x").unwrap_or(false), + excludes, + ..Default::default() + }; + match app::run(cfg) { + Ok(()) => 0, + Err(e) => { + eprintln!("{}: {}", brand::NAME, textutil::sanitize(&e.to_string())); + 1 + } + } +} diff --git a/rust/src/platform.rs b/rust/src/platform.rs new file mode 100644 index 0000000..d52c8de --- /dev/null +++ b/rust/src/platform.rs @@ -0,0 +1,885 @@ +//! OS-specific filesystem metadata and file operations (Unix; Linux first). +//! Nothing else in the program calls libc for filesystem work. + +use std::collections::HashMap; +use std::ffi::{CStr, CString, OsStr}; +use std::fs::File; +use std::io::{self, Write}; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; + +/// File mode bits in Go's io/fs.FileMode layout. Kept so snapshots stay +/// byte-compatible with the Go implementation. +pub mod mode { + pub const DIR: u32 = 1 << 31; + pub const APPEND: u32 = 1 << 30; + pub const EXCLUSIVE: u32 = 1 << 29; + pub const TEMPORARY: u32 = 1 << 28; + pub const SYMLINK: u32 = 1 << 27; + pub const DEVICE: u32 = 1 << 26; + pub const NAMED_PIPE: u32 = 1 << 25; + pub const SOCKET: u32 = 1 << 24; + pub const SETUID: u32 = 1 << 23; + pub const SETGID: u32 = 1 << 22; + pub const CHAR_DEVICE: u32 = 1 << 21; + pub const STICKY: u32 = 1 << 20; + pub const IRREGULAR: u32 = 1 << 19; + pub const TYPE: u32 = DIR | SYMLINK | NAMED_PIPE | SOCKET | DEVICE | CHAR_DEVICE | IRREGULAR; + pub const PERM: u32 = 0o777; + + pub fn is_dir(m: u32) -> bool { + m & DIR != 0 + } + pub fn is_regular(m: u32) -> bool { + m & TYPE == 0 + } + pub fn is_symlink(m: u32) -> bool { + m & SYMLINK != 0 + } + + /// Formats like Go's FileMode.String, e.g. "drwxr-xr-x". + pub fn string(m: u32) -> String { + const STR: &[u8] = b"dalTLDpSugct?"; + let mut out = String::with_capacity(12); + for (i, &c) in STR.iter().enumerate() { + if m & (1 << (31 - i)) != 0 { + out.push(c as char); + } + } + if out.is_empty() { + out.push('-'); + } + for (i, c) in "rwxrwxrwx".chars().enumerate() { + out.push(if m & (1 << (8 - i)) != 0 { c } else { '-' }); + } + out + } +} + +/// Converts a Unix st_mode to the Go FileMode layout. +pub fn mode_from_unix(m: u32) -> u32 { + let mut out = m & 0o777; + out |= match m & libc::S_IFMT { + libc::S_IFDIR => mode::DIR, + libc::S_IFLNK => mode::SYMLINK, + libc::S_IFIFO => mode::NAMED_PIPE, + libc::S_IFSOCK => mode::SOCKET, + libc::S_IFCHR => mode::DEVICE | mode::CHAR_DEVICE, + libc::S_IFBLK => mode::DEVICE, + _ => 0, + }; + if m & libc::S_ISUID != 0 { + out |= mode::SETUID; + } + if m & libc::S_ISGID != 0 { + out |= mode::SETGID; + } + if m & libc::S_ISVTX != 0 { + out |= mode::STICKY; + } + out +} + +/// Metadata of one directory entry. +#[derive(Debug, Default)] +pub struct Meta { + pub name: Vec, + pub mode: u32, // Go FileMode layout + pub size: i64, + pub alloc: i64, + pub alloc_known: bool, + pub mtime: i64, // unix nanoseconds + pub btime: i64, // birth time, 0 if unknown + pub uid: u32, + pub gid: u32, + pub nlink: u32, + pub dev: u64, + pub ino: u64, + pub has_ino: bool, + pub err: Option, // stat failed; only name is valid +} + +fn cstr(b: &[u8]) -> io::Result { + CString::new(b).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL)) +} + +fn cvt(r: libc::c_int) -> io::Result { + if r < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(r) + } +} + +fn open_dir(path: &[u8]) -> io::Result { + let c = cstr(path)?; + let fd = cvt(unsafe { libc::open(c.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC) })?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) +} + +#[cfg(target_os = "linux")] +static NO_STATX: AtomicBool = AtomicBool::new(false); + +#[cfg(target_os = "linux")] +fn stat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + if !NO_STATX.load(Ordering::Relaxed) { + let mut st: libc::statx = unsafe { std::mem::zeroed() }; + let mask = libc::STATX_BASIC_STATS | libc::STATX_BTIME; + let r = unsafe { libc::statx(dirfd, name.as_ptr(), flags | libc::AT_STATX_DONT_SYNC, mask, &mut st) }; + if r == 0 { + let mut m = Meta { + mode: mode_from_unix(st.stx_mode as u32), + size: st.stx_size as i64, + alloc: st.stx_blocks as i64 * 512, + alloc_known: true, + mtime: st.stx_mtime.tv_sec * 1_000_000_000 + st.stx_mtime.tv_nsec as i64, + uid: st.stx_uid, + gid: st.stx_gid, + nlink: st.stx_nlink, + dev: libc::makedev(st.stx_dev_major, st.stx_dev_minor), + ino: st.stx_ino, + has_ino: true, + ..Default::default() + }; + if st.stx_mask & libc::STATX_BTIME != 0 { + m.btime = st.stx_btime.tv_sec * 1_000_000_000 + st.stx_btime.tv_nsec as i64; + } + return Ok(m); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(libc::ENOSYS) { + return Err(err); + } + NO_STATX.store(true, Ordering::Relaxed); + } + fstat_at(dirfd, name, flags) +} + +#[cfg(not(target_os = "linux"))] +fn stat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + fstat_at(dirfd, name, flags) +} + +fn fstat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + cvt(unsafe { libc::fstatat(dirfd, name.as_ptr(), &mut st, flags) })?; + Ok(from_stat(&st)) +} + +#[allow(clippy::unnecessary_cast)] +fn from_stat(st: &libc::stat) -> Meta { + Meta { + mode: mode_from_unix(st.st_mode as u32), + size: st.st_size as i64, + alloc: st.st_blocks as i64 * 512, + alloc_known: true, + mtime: st.st_mtime as i64 * 1_000_000_000 + st.st_mtime_nsec as i64, + uid: st.st_uid, + gid: st.st_gid, + nlink: st.st_nlink as u32, + dev: st.st_dev as u64, + ino: st.st_ino as u64, + has_ino: true, + ..Default::default() + } +} + +/// An open directory stream (readdir over the fd we also stat relative to). +struct DirStream { + dir: *mut libc::DIR, +} + +impl Drop for DirStream { + fn drop(&mut self) { + unsafe { libc::closedir(self.dir) }; + } +} + +/// Streams the entries of a directory in chunks of up to `chunk`, stat-ing +/// each entry relative to the directory fd (no repeated path resolution). +/// Returning an error from f stops the read and returns it. +pub fn read_dir>(path: &[u8], chunk: usize, mut f: impl FnMut(Vec) -> Result<(), E>) -> Result<(), E> { + let fd = open_dir(path)?; + let raw = fd.as_raw_fd(); + let dir = unsafe { libc::fdopendir(raw) }; + if dir.is_null() { + return Err(io::Error::last_os_error().into()); + } + std::mem::forget(fd); // owned by the DIR stream now + let ds = DirStream { dir }; + let mut out = Vec::with_capacity(chunk); + loop { + unsafe { *libc::__errno_location() = 0 }; + let ent = unsafe { libc::readdir(ds.dir) }; + if ent.is_null() { + let e = io::Error::last_os_error(); + if e.raw_os_error().unwrap_or(0) != 0 { + return Err(e.into()); + } + break; + } + let name = unsafe { CStr::from_ptr((*ent).d_name.as_ptr()) }; + let nb = name.to_bytes(); + if nb == b"." || nb == b".." { + continue; + } + let mut m = stat_at(raw, name, libc::AT_SYMLINK_NOFOLLOW).unwrap_or_else(|e| Meta { err: Some(e), ..Default::default() }); + m.name = nb.to_vec(); + out.push(m); + if out.len() >= chunk { + f(std::mem::replace(&mut out, Vec::with_capacity(chunk)))?; + } + } + if !out.is_empty() { + f(out)?; + } + Ok(()) +} + +fn stat_path(path: &[u8], follow: bool) -> io::Result { + let c = cstr(path)?; + let flags = if follow { 0 } else { libc::AT_SYMLINK_NOFOLLOW }; + let mut m = stat_at(libc::AT_FDCWD, &c, flags)?; + m.name = base_name(path).to_vec(); + Ok(m) +} + +/// Metadata for path without following a final symlink. +pub fn lstat(path: &[u8]) -> io::Result { + stat_path(path, false) +} + +/// Metadata for path, following symlinks. +pub fn stat(path: &[u8]) -> io::Result { + stat_path(path, true) +} + +/// Joins a directory and an entry name without cleaning. +pub fn join_path(dir: &[u8], name: &[u8]) -> Vec { + let mut p = Vec::with_capacity(dir.len() + name.len() + 1); + p.extend_from_slice(dir); + if dir.last() != Some(&b'/') { + p.push(b'/'); + } + p.extend_from_slice(name); + p +} + +fn base_name(path: &[u8]) -> &[u8] { + let mut end = path.len(); + while end > 1 && path[end - 1] == b'/' { + end -= 1; + } + let p = &path[..end]; + match p.iter().rposition(|&b| b == b'/') { + Some(i) if p.len() > 1 => &p[i + 1..], + _ => p, + } +} + +static NAMES: Mutex, HashMap)>> = Mutex::new(None); + +fn lookup_name(group: bool, id: u32) -> String { + let mut g = NAMES.lock().unwrap(); + let (users, groups) = g.get_or_insert_with(Default::default); + let cache = if group { groups } else { users }; + if let Some(n) = cache.get(&id) { + return n.clone(); + } + let mut buf = vec![0u8; 16384]; + let name = unsafe { + if group { + let mut gr: libc::group = std::mem::zeroed(); + let mut res = std::ptr::null_mut(); + libc::getgrgid_r(id, &mut gr, buf.as_mut_ptr().cast(), buf.len(), &mut res); + (!res.is_null()).then(|| CStr::from_ptr(gr.gr_name).to_string_lossy().into_owned()) + } else { + let mut pw: libc::passwd = std::mem::zeroed(); + let mut res = std::ptr::null_mut(); + libc::getpwuid_r(id, &mut pw, buf.as_mut_ptr().cast(), buf.len(), &mut res); + (!res.is_null()).then(|| CStr::from_ptr(pw.pw_name).to_string_lossy().into_owned()) + } + } + .unwrap_or_else(|| id.to_string()); + cache.insert(id, name.clone()); + name +} + +/// Resolves a uid to a name (cached; falls back to the number). +pub fn user_name(uid: u32) -> String { + lookup_name(false, uid) +} + +/// Resolves a gid to a name (cached; falls back to the number). +pub fn group_name(gid: u32) -> String { + lookup_name(true, gid) +} + +/// Linux pseudo-filesystems whose contents are not disk usage. +#[cfg(target_os = "linux")] +const VIRTUAL_FS: [i64; 19] = [ + 0x9fa0, // proc + 0x62656572, // sysfs + 0x1cd1, // devpts + 0x27e0eb, // cgroup + 0x63677270, // cgroup2 + 0x64626720, // debugfs + 0x74726163, // tracefs + 0x73636673, // securityfs + 0x6165676c, // pstore + 0xcafe4a11, // bpf + 0x62656570, // configfs + 0x65735543, // fusectl + 0x19800202, // mqueue + 0xde5e81e4, // efivarfs + 0x42494e4d, // binfmt_misc + 0xf97cff8c, // selinuxfs + 0x958458f6, // hugetlbfs + 0x6e736673, // nsfs + 0x50495045, // pipefs +]; + +/// Whether path is the root of a pseudo filesystem (proc, sysfs, cgroup, …) +/// that should not be scanned as disk usage. +#[cfg(target_os = "linux")] +pub fn is_virtual_fs(path: &[u8]) -> bool { + let Ok(c) = cstr(path) else { return false }; + let mut st: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(c.as_ptr(), &mut st) } != 0 { + return false; + } + VIRTUAL_FS.contains(&(st.f_type as i64)) +} + +#[cfg(not(target_os = "linux"))] +pub fn is_virtual_fs(_: &[u8]) -> bool { + false +} + +/// What the scan recorded for an item about to be trashed or deleted. The +/// operation refuses an item whose owner or file type no longer matches. +#[derive(Clone, Copy, Debug)] +pub struct Expect { + pub uid: u32, + pub mode: u32, // only the type bits are compared +} + +const CHANGED: &str = "item changed since the scan (owner or type differs); rescan first"; + +fn other(msg: impl Into) -> io::Error { + io::Error::other(msg.into()) +} + +/// Guards helper invocations: an absolute path can never be parsed as an +/// option by the helper program. +fn check_abs(path: &Path) -> io::Result<()> { + if !path.is_absolute() { + return Err(other("refusing relative path")); + } + std::fs::symlink_metadata(path).map(|_| ()) +} + +/// Opens a file whose content is about to be read. It does not follow a +/// final symlink, and O_NONBLOCK keeps a FIFO swapped in since the scan from +/// blocking open(2) forever; callers still check the file type. +pub fn open_content(path: &Path) -> io::Result { + std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK).open(path) +} + +/// Runs a helper program with an argument vector (never a shell), detached +/// from the terminal so it cannot disturb the TUI. +fn start_detached(name: &str, arg: &Path) -> io::Result<()> { + use std::process::{Command, Stdio}; + let mut child = Command::new(name).arg(arg).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn()?; + std::thread::spawn(move || child.wait()); + Ok(()) +} + +/// The user-facing name of the trash. +pub const TRASH_NAME: &str = "trash"; + +/// Opens the item's directory in the desktop file manager. +pub fn reveal(path: &Path) -> io::Result<()> { + check_abs(path)?; + let md = std::fs::symlink_metadata(path)?; + let target = if md.is_dir() { path } else { path.parent().unwrap_or(path) }; + let opener = if cfg!(target_os = "macos") { "open" } else { "xdg-open" }; + start_detached(opener, target) +} + +/// Whether there is no graphical session (no X or Wayland display), so there +/// is no file manager to open and no desktop trash anyone will empty. +pub fn headless() -> bool { + if cfg!(target_os = "macos") { + return false; + } + std::env::var_os("DISPLAY").is_none_or(|v| v.is_empty()) && std::env::var_os("WAYLAND_DISPLAY").is_none_or(|v| v.is_empty()) +} + +fn split_parent(path: &Path) -> io::Result<(&Path, &OsStr)> { + match (path.parent(), path.file_name()) { + (Some(d), Some(b)) => Ok((d, b)), + _ => Err(other("refusing to operate on a filesystem root")), + } +} + +fn check_expect(dirfd: RawFd, base: &CStr, want: Expect) -> io::Result { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + cvt(unsafe { libc::fstatat(dirfd, base.as_ptr(), &mut st, libc::AT_SYMLINK_NOFOLLOW) })?; + if st.st_uid != want.uid || mode_from_unix(st.st_mode as u32) & mode::TYPE != want.mode & mode::TYPE { + return Err(other(CHANGED)); + } + Ok(st) +} + +/// Permanently removes path (recursively for a directory) if it still +/// matches want. The check and the removal both go through descriptors +/// opened from the parent directory without following symlinks, so a +/// symlink swapped in since the scan cannot redirect the removal outside it. +pub fn delete(path: &Path, want: Expect) -> io::Result<()> { + check_abs(path)?; + let (dir, base) = split_parent(path)?; + let dfd = open_dir(dir.as_os_str().as_bytes())?; + let cbase = cstr(base.as_bytes())?; + let st = check_expect(dfd.as_raw_fd(), &cbase, want)?; + remove_at(dfd.as_raw_fd(), &cbase, st.st_mode & libc::S_IFMT == libc::S_IFDIR) +} + +fn remove_at(dirfd: RawFd, name: &CStr, is_dir: bool) -> io::Result<()> { + if !is_dir { + return cvt(unsafe { libc::unlinkat(dirfd, name.as_ptr(), 0) }).map(|_| ()); + } + let fd = cvt(unsafe { + libc::openat(dirfd, name.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + })?; + let sub = unsafe { OwnedFd::from_raw_fd(fd) }; + // Collect names first: removing while iterating a DIR stream is unspecified. + let mut names = Vec::new(); + let dup = cvt(unsafe { libc::dup(sub.as_raw_fd()) })?; + let dir = unsafe { libc::fdopendir(dup) }; + if dir.is_null() { + unsafe { libc::close(dup) }; + return Err(io::Error::last_os_error()); + } + let ds = DirStream { dir }; + loop { + let ent = unsafe { libc::readdir(ds.dir) }; + if ent.is_null() { + break; + } + let n = unsafe { CStr::from_ptr((*ent).d_name.as_ptr()) }; + if n.to_bytes() != b"." && n.to_bytes() != b".." { + let t = unsafe { (*ent).d_type }; + let is_dir = match t { + libc::DT_UNKNOWN => fstat_at(sub.as_raw_fd(), n, libc::AT_SYMLINK_NOFOLLOW).is_ok_and(|m| mode::is_dir(m.mode)), + t => t == libc::DT_DIR, + }; + names.push((n.to_owned(), is_dir)); + } + } + drop(ds); + for (n, d) in names { + remove_at(sub.as_raw_fd(), &n, d)?; + } + cvt(unsafe { libc::unlinkat(dirfd, name.as_ptr(), libc::AT_REMOVEDIR) }).map(|_| ()) +} + +/// Moves path to dst if it still matches want. The parent directory is +/// opened once and both the check and the rename are relative to that +/// descriptor, so a parent swapped for a symlink since the scan (or between +/// check and rename) cannot redirect the move to another user's file. +fn rename_checked(path: &Path, dst: &Path, want: Expect) -> io::Result<()> { + let (dir, base) = split_parent(path)?; + let dfd = open_dir(dir.as_os_str().as_bytes())?; + let cbase = cstr(base.as_bytes())?; + check_expect(dfd.as_raw_fd(), &cbase, want)?; + let cdst = cstr(dst.as_os_str().as_bytes())?; + cvt(unsafe { libc::renameat(dfd.as_raw_fd(), cbase.as_ptr(), libc::AT_FDCWD, cdst.as_ptr()) }).map(|_| ()) +} + +/// Creates dir (mode 0700) if missing; an existing entry must be a real +/// directory (not a symlink) owned by us. With strict (trash on a shared +/// filesystem) it must also be inaccessible to others, as the freedesktop.org +/// spec requires. +fn ensure_private_dir(dir: &Path, strict: bool) -> io::Result<()> { + use std::os::unix::fs::DirBuilderExt; + if let Err(e) = std::fs::DirBuilder::new().mode(0o700).create(dir) { + if e.kind() != io::ErrorKind::AlreadyExists { + return Err(e); + } + } + let md = std::fs::symlink_metadata(dir)?; + if !md.is_dir() { + return Err(other("not a directory (symlink?)")); + } + if md.uid() != unsafe { libc::getuid() } { + return Err(other("owned by another user")); + } + if strict && md.mode() & 0o077 != 0 { + return Err(other("accessible to other users")); + } + Ok(()) +} + +/// The top directory of the filesystem containing path. +fn mount_top(path: &Path, dev: u64) -> PathBuf { + let mut cur = path.parent().unwrap_or(path).to_path_buf(); + loop { + let Some(parent) = cur.parent() else { return cur }; + match std::fs::metadata(parent) { + Ok(m) if m.dev() == dev => cur = parent.to_path_buf(), + _ => return cur, + } + } +} + +fn percent_encode_path(p: &[u8]) -> String { + let mut s = String::with_capacity(p.len()); + for &b in p { + if b.is_ascii_alphanumeric() || b"-._~/!$&'()*+,;=:@".contains(&b) { + s.push(b as char); + } else { + s.push_str(&format!("%{b:02X}")); + } + } + s +} + +fn local_timestamp() -> String { + let mut t: libc::time_t = unsafe { libc::time(std::ptr::null_mut()) }; + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + unsafe { libc::localtime_r(&mut t, &mut tm) }; + format!("{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec) +} + +/// Moves path to the freedesktop.org trash: the home trash when on the same +/// filesystem, otherwise $topdir/.Trash-$uid on the item's filesystem. It +/// never copies across filesystems and never deletes. +pub fn trash(path: &Path, want: Expect) -> io::Result<()> { + check_abs(path)?; + let dev = std::fs::symlink_metadata(path)?.dev(); + let data_home = match std::env::var_os("XDG_DATA_HOME").filter(|v| !v.is_empty()) { + Some(d) => PathBuf::from(d), + None => { + let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).ok_or_else(|| other("$HOME is not defined"))?; + PathBuf::from(home).join(".local/share") + } + }; + let home = data_home.join("Trash"); + let (trash, info_path): (PathBuf, Vec) = match std::fs::metadata(&data_home) { + Ok(m) if m.dev() == dev => (home.clone(), path.as_os_str().as_bytes().to_vec()), + _ => { + let top = mount_top(path, dev); + let rel = path.strip_prefix(&top).map_err(|e| other(e.to_string()))?; + (top.join(format!(".Trash-{}", unsafe { libc::getuid() })), rel.as_os_str().as_bytes().to_vec()) + } + }; + let shared = trash != home; + if !shared { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&trash)?; + } else if let Err(e) = ensure_private_dir(&trash, true) { + // On a shared filesystem another user could plant a symlink or a + // world-writable directory here; refuse rather than follow it. + return Err(other(format!("trash directory {} is unsafe: {e}", trash.display()))); + } + let (files, info) = (trash.join("files"), trash.join("info")); + for d in [&files, &info] { + ensure_private_dir(d, shared).map_err(|e| other(format!("trash directory {} is unsafe: {e}", d.display())))?; + } + let base = path.file_name().ok_or_else(|| other("no file name"))?; + for i in 1..10000 { + let mut name = base.to_os_string(); + if i > 1 { + name.push(format!(".{i}")); + } + let dst = files.join(&name); + if std::fs::symlink_metadata(&dst).is_ok() { + continue; // never overwrite an earlier trashed item (even without .trashinfo) + } + let mut info_name = name.clone(); + info_name.push(".trashinfo"); + let ip = info.join(info_name); + let mut f = match std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&ip) { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + }; + let w = write!(f, "[Trash Info]\nPath={}\nDeletionDate={}\n", percent_encode_path(&info_path), local_timestamp()).and_then(|_| f.sync_all()); + drop(f); + if let Err(e) = w.and_then(|_| rename_checked(path, &dst, want)) { + let _ = std::fs::remove_file(&ip); + return Err(e); + } + return Ok(()); + } + Err(other("trash: too many items with the same name")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mode_string_matches_go() { + assert_eq!(mode::string(mode::DIR | 0o755), "drwxr-xr-x"); + assert_eq!(mode::string(0o644), "-rw-r--r--"); + assert_eq!(mode::string(mode::SYMLINK | 0o777), "Lrwxrwxrwx"); + } + + #[test] + fn delete_refuses_changed_type() { + let d = std::env::temp_dir().join(format!("mapsize-del-{}", std::process::id())); + std::fs::create_dir_all(d.join("sub/inner")).unwrap(); + std::fs::write(d.join("sub/inner/f"), b"x").unwrap(); + let uid = unsafe { libc::getuid() }; + assert!(delete(&d.join("sub"), Expect { uid, mode: 0 }).is_err()); + delete(&d.join("sub"), Expect { uid, mode: mode::DIR }).unwrap(); + assert!(!d.join("sub").exists()); + std::fs::remove_dir_all(&d).unwrap(); + } +} + +/// Shell-style glob match, a port of Go's filepath.Match: `*` and `?` do not +/// match '/', `[^...]` negates, `\\` escapes. A malformed pattern matches +/// nothing. +pub fn glob_match(pattern: &[u8], name: &[u8]) -> bool { + // Fast path for the common "*.ext" shape: * cannot cross '/'. + if let Some(suffix) = pattern.strip_prefix(b"*") { + if !suffix.iter().any(|b| b"*?[\\".contains(b)) { + return name.ends_with(suffix) && !name[..name.len() - suffix.len()].contains(&b'/'); + } + } + glob(pattern, name).unwrap_or(false) +} + +struct BadPattern; + +fn decode_rune(s: &[u8]) -> (u32, usize) { + match s.utf8_chunks().next() { + Some(c) if !c.valid().is_empty() => { + let ch = c.valid().chars().next().unwrap(); + (ch as u32, ch.len_utf8()) + } + _ => (0xFFFD, 1), + } +} + +fn glob(mut pattern: &[u8], mut name: &[u8]) -> Result { + 'pattern: while !pattern.is_empty() { + let (star, chunk, rest) = scan_chunk(pattern); + pattern = rest; + if star && chunk.is_empty() { + // Trailing * matches the rest of the string unless it has a /. + return Ok(!name.contains(&b'/')); + } + let r = match_chunk(chunk, name); + if let Ok(Some(t)) = r { + // The last chunk must exhaust the name. + if t.is_empty() || !pattern.is_empty() { + name = t; + continue; + } + } + r?; + if star { + // Look for a match skipping i+1 bytes; cannot skip /. + let mut i = 0; + while i < name.len() && name[i] != b'/' { + if let Some(t) = match_chunk(chunk, &name[i + 1..])? { + if !(pattern.is_empty() && !t.is_empty()) { + name = t; + continue 'pattern; + } + } + i += 1; + } + } + // Before failing, check the rest of the pattern is well-formed. + while !pattern.is_empty() { + let (_, chunk, rest) = scan_chunk(pattern); + pattern = rest; + match_chunk(chunk, b"")?; + } + return Ok(false); + } + Ok(name.is_empty()) +} + +fn scan_chunk(mut pattern: &[u8]) -> (bool, &[u8], &[u8]) { + let mut star = false; + while pattern.first() == Some(&b'*') { + pattern = &pattern[1..]; + star = true; + } + let mut in_range = false; + let mut i = 0; + while i < pattern.len() { + match pattern[i] { + b'\\' if i + 1 < pattern.len() => i += 1, + b'[' => in_range = true, + b']' => in_range = false, + b'*' if !in_range => break, + _ => {} + } + i += 1; + } + (star, &pattern[..i], &pattern[i..]) +} + +/// Matches one chunk against the start of s, returning the rest of s. +fn match_chunk<'a>(mut chunk: &[u8], mut s: &'a [u8]) -> Result, BadPattern> { + // After a failure, keep parsing chunk to validate it, but stop reading s. + let mut failed = false; + while !chunk.is_empty() { + if !failed && s.is_empty() { + failed = true; + } + match chunk[0] { + b'[' => { + let mut r = 0; + if !failed { + let (c, n) = decode_rune(s); + r = c; + s = &s[n..]; + } + chunk = &chunk[1..]; + let negated = chunk.first() == Some(&b'^'); + if negated { + chunk = &chunk[1..]; + } + let (mut matched, mut nrange) = (false, 0); + loop { + if chunk.first() == Some(&b']') && nrange > 0 { + chunk = &chunk[1..]; + break; + } + let (lo, rest) = get_esc(chunk)?; + chunk = rest; + let mut hi = lo; + if chunk[0] == b'-' { + (hi, chunk) = get_esc(&chunk[1..])?; + } + if lo <= r && r <= hi { + matched = true; + } + nrange += 1; + } + if matched == negated { + failed = true; + } + } + b'?' => { + if !failed { + if s[0] == b'/' { + failed = true; + } + s = &s[decode_rune(s).1..]; + } + chunk = &chunk[1..]; + } + c => { + let c = if c == b'\\' { + chunk = &chunk[1..]; + *chunk.first().ok_or(BadPattern)? + } else { + c + }; + if !failed { + if c != s[0] { + failed = true; + } + s = &s[1..]; + } + chunk = &chunk[1..]; + } + } + } + Ok((!failed).then_some(s)) +} + +fn get_esc(mut chunk: &[u8]) -> Result<(u32, &[u8]), BadPattern> { + if chunk.is_empty() || chunk[0] == b'-' || chunk[0] == b']' { + return Err(BadPattern); + } + if chunk[0] == b'\\' { + chunk = &chunk[1..]; + if chunk.is_empty() { + return Err(BadPattern); + } + } + let (r, n) = decode_rune(chunk); + if (r == 0xFFFD && n == 1) || chunk.len() == n { + return Err(BadPattern); + } + Ok((r, &chunk[n..])) +} + +/// Lexically cleans a path like Go's filepath.Clean. +pub fn clean_path(p: &[u8]) -> Vec { + if p.is_empty() { + return b".".to_vec(); + } + let rooted = p[0] == b'/'; + let mut parts: Vec<&[u8]> = Vec::new(); + for c in p.split(|&b| b == b'/') { + match c { + b"" | b"." => {} + b".." => { + if parts.last().is_some_and(|l| *l != b"..") { + parts.pop(); + } else if !rooted { + parts.push(c); + } + } + _ => parts.push(c), + } + } + let mut out = Vec::with_capacity(p.len()); + if rooted { + out.push(b'/'); + } + out.extend_from_slice(&parts.join(&b'/')); + if out.is_empty() { + out.push(b'.'); + } + out +} + +/// Makes a path absolute (relative to the working directory) and cleans it. +pub fn abs_path(p: &[u8]) -> io::Result> { + if p.first() == Some(&b'/') { + return Ok(clean_path(p)); + } + let cwd = std::env::current_dir()?; + Ok(clean_path(&join_path(cwd.as_os_str().as_bytes(), p))) +} + +#[cfg(test)] +mod path_tests { + use super::*; + + #[test] + fn clean_and_glob() { + assert_eq!(clean_path(b"/a/b/../c/./"), b"/a/c"); + assert_eq!(clean_path(b"/.."), b"/"); + assert_eq!(clean_path(b"a/../../b"), b"../b"); + assert!(glob_match(b"*.log", b"x.log")); + assert!(!glob_match(b"*.log", b"d/x.log")); + assert!(glob_match(b"/tmp/*/x", b"/tmp/a/x")); + assert!(!glob_match(b"[", b"[")); + assert!(glob_match(b"a[^b-d]?", "ax\u{e9}".as_bytes())); + assert!(!glob_match(b"a[^b-d]c", b"abc")); + assert!(glob_match(b"\\*x", b"*x")); + assert!(glob_match(b"*.dat", b"f1.dat")); + assert!(!glob_match(b"*.dat", b"f1.dax")); + assert!(glob_match(b"a*b*c", b"axxbyyc")); + assert!(glob_match(b"*/x", b"a/x") && !glob_match(b"*/x", b"a/b/x")); + assert!(glob_match(b"*", b"abc") && !glob_match(b"*", b"a/c")); + assert!(!glob_match(b"*.dat", b"x.dat/y.da") && !glob_match(b"*.dat", b"d/x.dat")); + } +} diff --git a/rust/src/scan.rs b/rust/src/scan.rs new file mode 100644 index 0000000..f117a7d --- /dev/null +++ b/rust/src/scan.rs @@ -0,0 +1,775 @@ +//! The bounded concurrent filesystem scanner. +//! +//! Architecture (see ARCHITECTURE.md): a single controller thread owns the +//! write side of the Tree and a FIFO of pending directories. A fixed pool of +//! workers performs directory I/O and sends result chunks back. The +//! controller's select loop offers jobs and accepts results at the same +//! time, so bounded channels can never deadlock: workers never enqueue work +//! themselves. + +use crate::cancel::Cancel; +use crate::inventory::{self as inv, Delta, ErrorRecord, Kind, Node, NodeId, Tree}; +use crate::platform::{self, Meta}; +use crossbeam_channel::{Receiver, Select, Sender}; +use std::collections::HashSet; +use std::io; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::SystemTime; + +/// Symlink traversal. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum FollowMode { + #[default] + None, + SameFs, + All, +} + +impl FollowMode { + pub fn name(self) -> &'static str { + ["none", "same-filesystem", "all"][self as usize] + } +} + +/// Parses a --follow-symlinks value. +pub fn parse_follow(s: &str) -> Result { + match s { + "none" | "" => Ok(FollowMode::None), + "same-filesystem" | "same-fs" => Ok(FollowMode::SameFs), + "all" => Ok(FollowMode::All), + _ => Err(format!("invalid --follow-symlinks {s:?} (want none, same-filesystem or all)")), + } +} + +/// Resolves a concurrency mode name or number to a worker count. +pub fn workers(mode: &str) -> Result { + let cpu = std::thread::available_parallelism().map_or(1, |n| n.get()); + match mode { + "conservative" => Ok(2), + "balanced" | "" => Ok((2 * cpu).clamp(2, 8)), + "aggressive" => Ok((8 * cpu).clamp(4, 64)), + _ => match mode.trim().parse::() { + Ok(n) if (1..=1024).contains(&n) => Ok(n), + _ => Err(format!("invalid workers {mode:?} (conservative, balanced, aggressive or 1-1024)")), + }, + } +} + +/// Configures a scan. Zero tuning knobs mean default. +#[derive(Clone, Debug, Default)] +pub struct Options { + pub workers: usize, + pub follow: FollowMode, + pub one_file_system: bool, + pub excludes: Vec, + + pub chunk_size: usize, // entries per result chunk + pub job_buffer: usize, // capacity of the job channel + pub result_buf: usize, // capacity of the result channel + pub apply_batch: usize, // max results applied per write-lock hold +} + +impl Options { + fn defaults(&mut self) { + if self.workers == 0 { + self.workers = workers("balanced").unwrap(); + } + if self.chunk_size == 0 { + self.chunk_size = 1024; + } + if self.job_buffer == 0 { + self.job_buffer = self.workers * 2; + } + if self.result_buf == 0 { + self.result_buf = self.workers * 4; + } + if self.apply_batch == 0 { + self.apply_batch = 64; + } + } +} + +/// A cheap, lock-free view of scan activity. +#[derive(Clone, Copy, Debug, Default)] +pub struct Progress { + pub entries: i64, + pub pending: i64, // directories discovered but not yet dispatched + pub inflight: i64, // directories being read + pub workers: usize, +} + +#[derive(Default)] +struct Counters { + entries: AtomicI64, + pending: AtomicI64, + inflight: AtomicI64, +} + +/// One running scan. `tree` is safe to read under its read lock while the +/// scan runs. +pub struct Scanner { + pub tree: Arc>, + counters: Arc, + workers: usize, + done: Receiver<()>, +} + +impl Scanner { + /// Ready (disconnected) once the scan has finished or been cancelled and + /// drained; for use in select. + pub fn done(&self) -> &Receiver<()> { + &self.done + } + + pub fn is_done(&self) -> bool { + matches!(self.done.try_recv(), Err(crossbeam_channel::TryRecvError::Disconnected)) + } + + /// Blocks until the scan is done. + pub fn wait(&self) { + let _ = self.done.recv(); + } + + /// Live counters without locking the tree. + pub fn progress(&self) -> Progress { + Progress { + entries: self.counters.entries.load(Ordering::Relaxed), + pending: self.counters.pending.load(Ordering::Relaxed), + inflight: self.counters.inflight.load(Ordering::Relaxed), + workers: self.workers, + } + } +} + +struct Job { + id: NodeId, + path: Vec, + dev: u64, + // hints derived once per directory + in_cache: bool, + in_log: bool, +} + +enum ReadErr { + Io(io::Error), + Cancelled, +} + +impl From for ReadErr { + fn from(e: io::Error) -> Self { + ReadErr::Io(e) + } +} + +struct ScanResult { + job: Arc, + entries: Vec, + is_final: bool, + err: Option, +} + +type InodeKey = (u64, u64); + +fn kind_of(m: &Meta) -> Kind { + if platform::mode::is_dir(m.mode) { + Kind::Dir + } else if platform::mode::is_regular(m.mode) { + Kind::File + } else if platform::mode::is_symlink(m.mode) { + Kind::Symlink + } else { + Kind::Other + } +} + +fn set_meta(n: &mut Node, m: &Meta) { + n.size = m.size; + n.alloc = m.alloc; + n.mtime = m.mtime; + n.mode = m.mode; + n.uid = m.uid; + n.gid = m.gid; + n.nlink = m.nlink; + if !m.alloc_known { + n.flags |= inv::FLAG_ALLOC_UNKNOWN; + } +} + +/// Runs f, aborting the process if it panics: a scanner bug must never +/// leave the controller waiting forever. The panic hook (which the TUI uses +/// to restore the terminal) runs first. +fn guard(f: impl FnOnce()) { + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).is_err() { + std::process::abort(); + } +} + +/// Stats root, creates the tree and begins scanning in the background. +pub fn start(root: &[u8], mut opts: Options, cancel: Cancel) -> io::Result { + opts.defaults(); + // Path-style exclusions are matched against absolute paths. + for p in &mut opts.excludes { + if p.contains(['/', '\\']) { + if let Ok(a) = platform::abs_path(p.as_bytes()) { + *p = String::from_utf8_lossy(&a).into_owned(); + } + } + } + let abs = platform::abs_path(root)?; + let m = platform::stat(&abs)?; + let kind = kind_of(&m); + let mut t = Tree::new(&abs, kind); + t.stats.workers = opts.workers; + t.stats.excludes = opts.excludes.clone(); + t.stats.one_file_system = opts.one_file_system; + t.stats.follow = opts.follow.name().to_string(); + let r = t.node_mut(0); + set_meta(r, &m); + r.tot_size = m.size; + r.tot_alloc = m.alloc; + let (done_tx, done) = crossbeam_channel::bounded::<()>(0); + let counters = Arc::new(Counters::default()); + if kind != Kind::Dir { + t.stats.files = 1; + t.stats.complete = true; + t.stats.end = Some(SystemTime::now()); + drop(done_tx); + return Ok(Scanner { tree: Arc::new(RwLock::new(t)), counters, workers: opts.workers, done }); + } + t.stats.dirs = 1; + // Loop detection needs inode identity; without it symlinks are not + // followed at all. + if opts.follow != FollowMode::None && !m.has_ino { + opts.follow = FollowMode::None; + t.stats.follow = "none (no inode identity on this platform)".into(); + } + let visited = (opts.follow != FollowMode::None).then(|| HashSet::from([(m.dev, m.ino)])); + let path_excludes = opts.excludes.iter().map(|p| (p.contains(['/', '\\']), p.as_bytes().to_vec())).collect(); + let tree = Arc::new(RwLock::new(t)); + let workers = opts.workers; + counters.pending.store(1, Ordering::Relaxed); + let c = Controller { + tree: tree.clone(), + counters: counters.clone(), + excludes: path_excludes, + opts, + pending: vec![(0, m.dev)], + head: 0, + links: HashSet::new(), + visited, + }; + std::thread::Builder::new().name("scan-controller".into()).spawn(move || { + guard(|| c.run(cancel)); + drop(done_tx); + })?; + Ok(Scanner { tree, counters, workers, done }) +} + +/// Bounds how long the controller holds the write lock, so the UI's read +/// lock is never starved for more than a few milliseconds. +const MAX_ENTRIES_PER_LOCK: usize = 8192; + +struct Controller { + tree: Arc>, + counters: Arc, + opts: Options, + excludes: Vec<(bool, Vec)>, // (is path pattern, pattern) + pending: Vec<(NodeId, u64)>, // FIFO of (dir, dev); head advances, compacted occasionally + head: usize, + links: HashSet, + visited: Option>, // only when following symlinks +} + +fn worker(cancel: &Cancel, jobs: Receiver>, results: Sender, chunk: usize) { + for j in jobs { + let err = if cancel.is_cancelled() { + Some(ReadErr::Cancelled) + } else { + platform::read_dir(&j.path, chunk, |ms| { + if cancel.is_cancelled() { + return Err(ReadErr::Cancelled); + } + results.send(ScanResult { job: j.clone(), entries: ms, is_final: false, err: None }).map_err(|_| ReadErr::Cancelled) + }) + .err() + }; + let _ = results.send(ScanResult { job: j, entries: Vec::new(), is_final: true, err }); + } +} + +impl Controller { + fn run(mut self, cancel: Cancel) { + let (jobs_tx, jobs_rx) = crossbeam_channel::bounded::>(self.opts.job_buffer); + let (res_tx, res_rx) = crossbeam_channel::bounded::(self.opts.result_buf); + let mut handles = Vec::with_capacity(self.opts.workers); + for i in 0..self.opts.workers { + let (jobs, results, cancel, chunk) = (jobs_rx.clone(), res_tx.clone(), cancel.clone(), self.opts.chunk_size); + handles.push( + std::thread::Builder::new() + .name(format!("scan-worker-{i}")) + .spawn(move || guard(|| worker(&cancel, jobs, results, chunk))) + .expect("spawn scan worker"), + ); + } + drop((jobs_rx, res_tx)); + + let mut inflight = 0i64; + let mut cancelled = false; + let mut next: Option> = None; + let mut batch: Vec = Vec::with_capacity(self.opts.apply_batch); + loop { + if next.is_none() && !cancelled && self.head < self.pending.len() { + next = Some(self.make_job(self.pending[self.head])); + } + if inflight == 0 && next.is_none() { + break; + } + let mut sel = Select::new(); + let send_i = next.as_ref().map(|_| sel.send(&jobs_tx)); + let recv_i = sel.recv(&res_rx); + let cancel_i = (!cancelled).then(|| sel.recv(cancel.done())); + let op = sel.select(); + let i = op.index(); + if Some(i) == send_i { + op.send(&jobs_tx, next.take().unwrap()).expect("workers alive"); + self.head += 1; + if self.head > 4096 && self.head * 2 > self.pending.len() { + self.pending.drain(..self.head); + self.head = 0; + } + inflight += 1; + } else if i == recv_i { + let r = op.recv(&res_rx).expect("workers alive"); + let mut n = r.entries.len(); + batch.push(r); + while batch.len() < self.opts.apply_batch && n < MAX_ENTRIES_PER_LOCK { + match res_rx.try_recv() { + Ok(r) => { + n += r.entries.len(); + batch.push(r); + } + Err(_) => break, + } + } + let tree = self.tree.clone(); + let mut t = tree.write().unwrap(); + for r in batch.drain(..) { + if self.apply(&mut t, r) { + inflight -= 1; + } + } + } else { + debug_assert_eq!(Some(i), cancel_i); + let _ = op.recv(cancel.done()); + cancelled = true; + next = None; + } + self.counters.inflight.store(inflight, Ordering::Relaxed); + self.counters.pending.store((self.pending.len() - self.head) as i64, Ordering::Relaxed); + } + drop(jobs_tx); + for h in handles { + let _ = h.join(); + } + + let mut t = self.tree.write().unwrap(); + if cancelled { + t.stats.cancelled = true; + t.stats.unscanned += (self.pending.len() - self.head) as i64; + for &(id, _) in &self.pending[self.head..] { + t.node_mut(id).flags |= inv::FLAG_INCOMPLETE; + } + } + t.stats.complete = true; + t.stats.end = Some(SystemTime::now()); + drop(t); + self.counters.pending.store(0, Ordering::Relaxed); + self.counters.inflight.store(0, Ordering::Relaxed); + } + + fn make_job(&self, (id, dev): (NodeId, u64)) -> Arc { + let path = self.tree.read().unwrap().path_bytes(id); + let (in_cache, in_log) = inv::path_hints(&path); + Arc::new(Job { id, path, dev, in_cache, in_log }) + } + + /// Merges one result into the tree. Reports whether the result completed + /// its directory. Called with the tree write-locked. + fn apply(&mut self, t: &mut Tree, r: ScanResult) -> bool { + let j = &*r.job; + let mut d = Delta::default(); + self.counters.entries.fetch_add(r.entries.len() as i64, Ordering::Relaxed); + for m in r.entries { + self.add_entry(t, j, m, &mut d); + } + if r.is_final { + let dn = t.node_mut(j.id); + match r.err { + None => dn.flags |= inv::FLAG_SCANNED, + Some(ReadErr::Cancelled) => { + dn.flags |= inv::FLAG_INCOMPLETE; + t.stats.unscanned += 1; + } + Some(ReadErr::Io(e)) => { + dn.flags |= inv::FLAG_ERROR | inv::FLAG_INCOMPLETE; + t.stats.add_error(ErrorRecord { node: j.id, kind: inv::classify(&e), msg: format!("open: {}", err_msg(&e)), ..Default::default() }); + d.errors += 1; + } + } + } + if d != Delta::default() { + t.propagate(j.id, &d); + } + r.is_final + } + + fn excluded(&self, path: &[u8], name: &[u8]) -> bool { + self.excludes.iter().any(|(is_path, p)| { + if *is_path { + path == p.as_slice() || path.starts_with(p) && path.get(p.len()) == Some(&b'/') || platform::glob_match(p, path) + } else { + platform::glob_match(p, name) + } + }) + } + + fn add_entry(&mut self, t: &mut Tree, j: &Job, m: Meta, d: &mut Delta) { + if let Some(e) = &m.err { + let k = inv::classify(e); + if k == inv::ErrKind::Vanished { + t.stats.add_error(ErrorRecord { node: j.id, name: m.name, kind: k, msg: "vanished between listing and stat".into() }); + d.errors += 1; + return; + } + let id = t.add(j.id, &m.name, Kind::Other); + let n = t.node_mut(id); + n.flags |= inv::FLAG_ERROR; + n.errors = 1; + t.stats.add_error(ErrorRecord { node: id, kind: k, msg: format!("stat: {}", err_msg(e)), ..Default::default() }); + d.errors += 1; + d.files += 1; + t.stats.others += 1; + return; + } + let mut path = Vec::new(); + if !self.excludes.is_empty() { + path = platform::join_path(&j.path, &m.name); + if self.excluded(&path, &m.name) { + t.stats.excluded += 1; + return; + } + } + let mut kind = kind_of(&m); + let mut target = None; + let mut broken = false; + if kind == Kind::Symlink { + if path.is_empty() { + path = platform::join_path(&j.path, &m.name); + } + match platform::stat(&path) { + Err(_) => { + t.stats.broken_links += 1; + broken = true; + } + Ok(tm) if platform::mode::is_dir(tm.mode) && tm.has_ino && (self.opts.follow == FollowMode::All || self.opts.follow == FollowMode::SameFs && tm.dev == j.dev) => { + target = Some(tm); + kind = Kind::Dir; + } + Ok(_) => {} + } + } + + let id = t.add(j.id, &m.name, kind); + let n = t.node_mut(id); + set_meta(n, &m); + n.tot_size = n.size; + n.tot_alloc = n.alloc; + let ext = n.ext; + let mut flags = if broken { inv::FLAG_BROKEN_LINK } else { 0 }; + let st = &mut t.stats; + + match kind { + Kind::Dir => { + st.dirs += 1; + d.dirs += 1; + d.size += m.size; + d.alloc += m.alloc; + let mut dev = m.dev; + let mut key = (m.dev, m.ino); + if let Some(tm) = &target { + flags |= inv::FLAG_FOLLOWED; + dev = tm.dev; + key = (tm.dev, tm.ino); + } + let descend = 'check: { + if let Some(v) = &mut self.visited { + if !v.insert(key) { + flags |= inv::FLAG_LOOP; + st.loops_skipped += 1; + break 'check false; + } + } + if m.has_ino && dev != j.dev { + flags |= inv::FLAG_MOUNT_POINT; + if self.opts.one_file_system { + flags |= inv::FLAG_SKIPPED_FS; + st.skipped_mounts += 1; + break 'check false; + } + if path.is_empty() { + path = platform::join_path(&j.path, &m.name); + } + if platform::is_virtual_fs(&path) { + flags |= inv::FLAG_VIRTUAL_FS; + st.virtual_skipped += 1; + break 'check false; + } + } + true + }; + if descend { + self.pending.push((id, dev)); + } + } + Kind::File => { + st.files += 1; + d.files += 1; + let dup = 'check: { + if m.nlink > 1 && m.has_ino { + flags |= inv::FLAG_HARDLINKED; + if !self.links.insert((m.dev, m.ino)) { + flags |= inv::FLAG_HARDLINK_DUP; + st.hardlink_dups += 1; + break 'check true; + } + } + false + }; + if !dup { + if m.alloc_known && m.alloc < m.size && m.size - m.alloc >= 4096 { + flags |= inv::FLAG_SPARSE; + } + let cat = inv::category_for(t.ext_by_index(ext as usize), j.in_cache, j.in_log); + t.node_mut(id).cat = cat; + d.size += m.size; + d.alloc += m.alloc; + d.cat[cat as usize] += m.alloc; + } else { + // Duplicates contribute zero to aggregates. + let n = t.node_mut(id); + n.tot_size = 0; + n.tot_alloc = 0; + } + } + _ => { + if kind == Kind::Symlink { + st.symlinks += 1; + } else { + st.others += 1; + } + d.files += 1; + d.size += m.size; + d.alloc += m.alloc; + } + } + t.node_mut(id).flags |= flags; + } +} + +/// The OS error text without Rust's " (os error N)" suffix, matching Go. +fn err_msg(e: &io::Error) -> String { + let s = e.to_string(); + match s.rfind(" (os error ") { + Some(i) => s[..i].to_lowercase(), + None => s, + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::os::unix::ffi::OsStrExt; + use std::path::{Path, PathBuf}; + + pub struct TempDir(pub PathBuf); + impl TempDir { + pub fn new(tag: &str) -> TempDir { + static N: AtomicI64 = AtomicI64::new(0); + let p = std::env::temp_dir().join(format!("mapsize-{tag}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } + } + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// Creates a fan-out tree: (files, dirs, bytes). + pub fn mk_tree(dir: &Path, fanout: usize, depth: usize, files: usize) -> (i64, i64, i64) { + fn rec(dir: &Path, d: usize, fanout: usize, depth: usize, files: usize, acc: &mut (i64, i64, i64)) { + for i in 0..files { + let sz = (i + 1) * 100; + std::fs::write(dir.join(format!("f{i}.txt")), vec![0u8; sz]).unwrap(); + acc.0 += 1; + acc.2 += sz as i64; + } + if d == depth { + return; + } + for i in 0..fanout { + let sub = dir.join(format!("d{i}")); + std::fs::create_dir(&sub).unwrap(); + acc.1 += 1; + rec(&sub, d + 1, fanout, depth, files, acc); + } + } + let mut acc = (0, 0, 0); + rec(dir, 0, fanout, depth, files, &mut acc); + acc + } + + pub fn run_scan(root: &Path, o: Options) -> Tree { + let s = start(root.as_os_str().as_bytes(), o, Cancel::new()).unwrap(); + s.done().recv_timeout(std::time::Duration::from_secs(60)).expect_err("scan did not finish (deadlock?)"); + let t = s.tree.read().unwrap().clone(); + t + } + + fn totals(t: &Tree) -> Vec<[i64; 4]> { + (0..t.len() as NodeId).map(|i| t.node(i)).map(|n| [n.tot_size, n.tot_alloc, n.files as i64, n.dirs as i64]).collect() + } + + #[test] + fn counts_and_aggregates() { + let root = TempDir::new("scan"); + let (nf, nd, bytes) = mk_tree(&root.0, 3, 3, 4); + let mut t = run_scan(&root.0, Options { workers: 4, ..Default::default() }); + let r = t.node(0); + assert_eq!((r.files as i64, r.dirs as i64), (nf, nd)); + let dir_bytes: i64 = (0..t.len() as NodeId).map(|i| t.node(i)).filter(|n| n.is_dir()).map(|n| n.size).sum(); + assert_eq!(r.tot_size, bytes + dir_bytes); + assert!(t.stats.complete && !t.stats.incomplete()); + for i in 0..t.len() as NodeId { + let n = t.node(i); + if n.is_dir() { + let sum: i64 = n.size + t.children(i).map(|(_, c)| c.tot_size).sum::(); + assert_eq!(sum, n.tot_size, "{}", t.path_string(i)); + } + } + let before = totals(&t); + t.recompute(); + assert_eq!(before, totals(&t), "incremental aggregates differ from recompute"); + } + + #[test] + fn backpressure_no_deadlock() { + let root = TempDir::new("bp"); + let (nf, nd, _) = mk_tree(&root.0, 6, 4, 3); + for w in [1, 2, 16] { + let t = run_scan(&root.0, Options { workers: w, job_buffer: 1, result_buf: 1, chunk_size: 1, apply_batch: 1, ..Default::default() }); + assert_eq!((t.node(0).files as i64, t.node(0).dirs as i64), (nf, nd), "workers={w}"); + } + } + + #[test] + fn cancel_stops() { + let root = TempDir::new("cancel"); + mk_tree(&root.0, 8, 3, 2); + let c = Cancel::new(); + let s = start(root.0.as_os_str().as_bytes(), Options { workers: 2, job_buffer: 1, result_buf: 1, chunk_size: 1, ..Default::default() }, c.clone()).unwrap(); + c.cancel(); + s.done().recv_timeout(std::time::Duration::from_secs(10)).expect_err("cancel did not stop scan"); + let t = s.tree.read().unwrap(); + assert!(t.stats.cancelled && t.stats.incomplete()); + } + + #[test] + fn symlink_loops() { + let root = TempDir::new("loop"); + let a = root.0.join("a"); + std::fs::create_dir_all(a.join("b")).unwrap(); + std::fs::write(a.join("b/x"), vec![0u8; 5000]).unwrap(); + std::os::unix::fs::symlink("..", a.join("b/up")).unwrap(); + std::os::unix::fs::symlink("../a", a.join("self")).unwrap(); + std::os::unix::fs::symlink("nowhere", root.0.join("broken")).unwrap(); + for f in [FollowMode::None, FollowMode::SameFs, FollowMode::All] { + let t = run_scan(&root.0, Options { workers: 4, follow: f, ..Default::default() }); + let r = t.node(0); + assert!(r.files > 0 && r.tot_size > 5000, "{f:?}"); + assert_eq!(t.stats.broken_links, 1); + assert!(f == FollowMode::None || t.stats.loops_skipped > 0); + let count = (0..t.len() as NodeId).filter(|&i| &*t.node(i).name == b"x").count(); + assert_eq!(count, 1, "{f:?}"); + } + } + + #[test] + fn hardlinks_counted_once() { + let root = TempDir::new("hl"); + let p = root.0.join("orig"); + std::fs::write(&p, vec![0u8; 100000]).unwrap(); + std::fs::create_dir(root.0.join("d")).unwrap(); + std::fs::hard_link(&p, root.0.join("d/link")).unwrap(); + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + assert_eq!(t.stats.hardlink_dups, 1); + assert!(t.node(0).tot_size < 200000); + } + + #[test] + fn permission_errors_surface() { + if unsafe { libc::geteuid() } == 0 { + return; + } + use std::os::unix::fs::PermissionsExt; + let root = TempDir::new("perm"); + let locked = root.0.join("locked"); + std::fs::create_dir(&locked).unwrap(); + std::fs::write(locked.join("f"), b"x").unwrap(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0)).unwrap(); + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(t.stats.err_counts[inv::ErrKind::Permission as usize], 1); + assert!(t.stats.incomplete()); + assert_eq!(t.node(0).errors, 1); + } + + #[test] + fn excludes() { + let root = TempDir::new("excl"); + std::fs::create_dir_all(root.0.join("keep")).unwrap(); + std::fs::create_dir_all(root.0.join("skip")).unwrap(); + std::fs::write(root.0.join("keep/a.tmp"), b"x").unwrap(); + std::fs::write(root.0.join("keep/b.txt"), b"x").unwrap(); + std::fs::write(root.0.join("skip/c.txt"), b"x").unwrap(); + let skip = root.0.join("skip").to_string_lossy().into_owned(); + let t = run_scan(&root.0, Options { workers: 2, excludes: vec!["*.tmp".into(), skip], ..Default::default() }); + assert_eq!((t.stats.excluded, t.node(0).files), (2, 1)); + } + + #[test] + fn hostile_names_preserved() { + let root = TempDir::new("names"); + let names: [&[u8]; 6] = ["日本語.txt".as_bytes(), "emoji-🎉.png".as_bytes(), b"esc\x1b[31mred", b"nl\nname", b"sp ace", b"bad\xffutf8"]; + for n in names { + std::fs::write(root.0.join(std::ffi::OsStr::from_bytes(n)), b"x").unwrap(); + } + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + let got: HashSet<&[u8]> = t.children(0).map(|(_, n)| &*n.name).collect(); + for n in names { + assert!(got.contains(n), "missing {n:?}"); + } + } + + #[test] + fn single_file_root() { + let root = TempDir::new("single"); + let p = root.0.join("file"); + std::fs::write(&p, vec![0u8; 1234]).unwrap(); + let t = run_scan(&p, Options::default()); + assert_eq!(t.node(0).tot_size, 1234); + assert_eq!(t.node(0).kind, Kind::File); + } +} diff --git a/rust/src/snapshot/compare.rs b/rust/src/snapshot/compare.rs new file mode 100644 index 0000000..08989a7 --- /dev/null +++ b/rust/src/snapshot/compare.rs @@ -0,0 +1,160 @@ +use crate::inventory::{sat_add, NodeId, SizeMode, Tree, NO_NODE}; +use std::collections::HashMap; +use std::fmt; + +/// Relates two inventories by path. It owns the old tree; methods that need +/// the new one take it as a parameter. The new tree must not be mutated +/// while a Diff is in use (snapshots and finished scans satisfy this). +pub struct Diff { + pub old: Tree, + /// Maps a new NodeId to the matching old NodeId, or NO_NODE if the entry + /// is new. + pub old_of: Vec, + /// The topmost old nodes with no counterpart in the new tree. + pub removed: Vec, + /// Per new directory, the sizes of removed children ([allocated, + /// logical]) so growth colouring can account for them. + removed_under: HashMap, +} + +/// Matches nodes by name, directory by directory, from the roots. +pub fn compare(old: Tree, new: &Tree) -> Diff { + let mut old_of = vec![NO_NODE; new.len()]; + let mut removed = Vec::new(); + let mut removed_under: HashMap = HashMap::new(); + let mut stack = vec![(old.root(), new.root())]; + while let Some((o, n)) = stack.pop() { + old_of[n as usize] = o; + if !old.node(o).is_dir() || !new.node(n).is_dir() { + continue; + } + let mut by_name: HashMap<&[u8], NodeId> = old.children(o).map(|(id, c)| (&c.name[..], id)).collect(); + for (id, c) in new.children(n) { + if let Some(oid) = by_name.remove(&c.name[..]) { + stack.push((oid, id)); + } + } + for oid in by_name.into_values() { + removed.push(oid); + let on = old.node(oid); + let r = removed_under.entry(n).or_default(); + r[0] = sat_add(r[0], on.tot_alloc); + r[1] = sat_add(r[1], on.tot_size); + } + } + removed.sort_unstable(); + Diff { old, old_of, removed, removed_under } +} + +/// Classifies a node. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub enum Status { + Unchanged, + Added, + Removed, + Grew, + Shrank, +} + +impl Status { + pub fn name(self) -> &'static str { + ["unchanged", "new", "removed", "increased", "decreased"][self as usize] + } +} + +impl fmt::Display for Status { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + f.write_str(self.name()) + } +} + +/// One row in a change report. +#[derive(Clone, Debug)] +pub struct Change { + pub path: String, + pub status: Status, + pub delta: i64, + pub old: i64, + pub new: i64, + pub node: NodeId, // in the new tree (NO_NODE for removed) +} + +impl Diff { + /// The matching old size of a new node, if it existed. + pub fn old_size(&self, id: NodeId, m: SizeMode) -> Option { + let o = self.old_of[id as usize]; + (o != NO_NODE).then(|| self.old.node(o).total(m)) + } + + /// New minus old size for a new node (full size if new). + pub fn delta(&self, new: &Tree, id: NodeId, m: SizeMode) -> i64 { + new.node(id).total(m) - self.old_size(id, m).unwrap_or(0) + } + + /// Classifies a new node. + pub fn status_of(&self, new: &Tree, id: NodeId, m: SizeMode) -> Status { + if self.old_of[id as usize] == NO_NODE { + return Status::Added; + } + match self.delta(new, id, m) { + d if d > 0 => Status::Grew, + d if d < 0 => Status::Shrank, + _ => Status::Unchanged, + } + } + + /// The k most significant changes (all if k is 0). A directory is listed + /// only if its change is not almost entirely explained by a single + /// child, so the report points at /var/lib/postgresql rather than /, + /// /var and /var/lib. + pub fn changes(&self, new: &Tree, k: usize, m: SizeMode) -> Vec { + let mut out = Vec::new(); + for id in 0..new.len() as NodeId { + let delta = self.delta(new, id, m); + if delta == 0 { + continue; + } + let n = new.node(id); + if self.old_of[id as usize] != NO_NODE && n.is_dir() && self.explained_by_child(new, id, delta, m) { + continue; + } + if n.parent != NO_NODE && self.old_of[n.parent as usize] == NO_NODE { + continue; // inside a wholly new directory; the directory is listed + } + out.push(Change { + path: new.path_string(id), + status: self.status_of(new, id, m), + delta, + old: self.old_size(id, m).unwrap_or(0), + new: n.total(m), + node: id, + }); + } + for &oid in &self.removed { + let o = self.old.node(oid).total(m); + out.push(Change { path: self.old.path_string(oid), status: Status::Removed, delta: -o, old: o, new: 0, node: NO_NODE }); + } + out.sort_by(|a, b| b.delta.unsigned_abs().cmp(&a.delta.unsigned_abs()).then_with(|| a.path.cmp(&b.path))); + if k > 0 { + out.truncate(k); + } + out + } + + // Products are widened to i128 so saturated (untrusted) totals cannot wrap. + fn explained_by_child(&self, new: &Tree, id: NodeId, delta: i64, m: SizeMode) -> bool { + let whole = delta.unsigned_abs() as i128 * 8; + let by_child = new.children(id).any(|(c, _)| { + let cd = self.delta(new, c, m); + (cd > 0) == (delta > 0) && cd.unsigned_abs() as i128 * 10 >= whole + }); + by_child + || match self.removed_under.get(&id) { + Some(r) => { + let v = if m == SizeMode::Logical { r[1] } else { r[0] }; + delta < 0 && v as i128 * 10 >= whole + } + None => false, + } + } +} diff --git a/rust/src/snapshot/mod.rs b/rust/src/snapshot/mod.rs new file mode 100644 index 0000000..814ef58 --- /dev/null +++ b/rust/src/snapshot/mod.rs @@ -0,0 +1,989 @@ +//! The versioned, checksummed binary inventory format and snapshot +//! comparison. +//! +//! Layout (all integers little-endian / varint): +//! +//! ```text +//! magic 8 bytes brand::SNAPSHOT_MAGIC +//! version uint16 currently 1 +//! flags uint16 bit 0: body is gzip-compressed (always set by save) +//! body gzip stream of: +//! meta uvarint length + JSON (root, timestamps, counters, options) +//! exts uvarint count, then count × (uvarint length + bytes) +//! nodes uvarint count, then per node: +//! uvarint parentDelta (id-parent; 0 only for the root) +//! byte kind, byte category, uvarint flags +//! uvarint nameLen + name +//! varint size, varint alloc, varint mtime +//! uvarint mode, uid, gid, nlink, ext +//! errors uvarint count, then per record: uvarint node, byte kind, +//! uvarint len + name, uvarint len + message +//! sha256 32 bytes over the uncompressed body +//! ``` +//! +//! Varints are Go's encoding/binary (zig-zag for signed). Nodes are written +//! in ID order; a parent always precedes its children, so a reader can +//! rebuild the tree in one pass and malformed input cannot create cycles. +//! Aggregates are recomputed on load, never trusted. + +mod compare; + +pub use compare::*; + +use crate::brand; +use crate::inventory::{Category, ErrKind, ErrorRecord, Kind, NodeId, Tree, MAX_ERROR_RECORDS, NUM_CATEGORIES}; +use crossbeam_channel::Receiver; +use flate2::{bufread::GzDecoder, write::GzEncoder, Compression}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::fs::{self, File, OpenOptions}; +use std::io::{self, BufReader, BufWriter, Read, Write}; +use std::os::unix::fs::OpenOptionsExt; +use std::path::{Path, PathBuf}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// The current format version. +pub const VERSION: u16 = 1; + +// Limits applied when reading untrusted snapshots. +pub const MAX_NODES: u64 = (1 << 31) - 1; +pub const MAX_DEPTH: u16 = 4096; // far beyond any real path; bounds recursion in exporters +pub const MAX_SIZE: i64 = 1 << 60; // per-node size cap (1 EiB); keeps aggregates from overflowing +/// Caps the decompressed body for load. load_file uses a limit proportional +/// to the file size instead (see body_limit). +pub const DEFAULT_MAX_BODY: i64 = 4 << 30; +pub const MAX_NAME_LEN: u64 = 1 << 16; +pub const MAX_META_LEN: u64 = 16 << 20; +pub const MAX_EXT_LEN: u64 = 64; +pub const MAX_MSG_LEN: u64 = 4096; +const MAX_EXTS: u64 = (1 << 16) - 1; +const FLAG_GZIPPED: u16 = 1; + +#[derive(Serialize, Deserialize, Default)] +#[serde(default)] +struct Meta { + root: String, + created: GoTime, + start: GoTime, + end: GoTime, + complete: bool, + cancelled: bool, + files: i64, + dirs: i64, + symlinks: i64, + others: i64, + err_counts: Option>, + excluded: i64, + skipped_mounts: i64, + virtual_skipped: i64, + loops_skipped: i64, + broken_links: i64, + hardlink_dups: i64, + unscanned: i64, + excludes: Option>, + one_file_system: bool, + follow: String, + workers: i64, + host: String, + generator: String, +} + +/// Reports whether path starts with the snapshot magic. +pub fn is_snapshot(path: &Path) -> bool { + let mut b = [0u8; 8]; + File::open(path).and_then(|mut f| f.read_exact(&mut b)).is_ok() && &b == brand::SNAPSHOT_MAGIC +} + +/// Accumulates the body; flushes it through the hash and gzip in chunks. +struct Writer { + gz: GzEncoder, + h: Sha256, + buf: Vec, +} + +impl Writer { + fn bytes(&mut self, b: &[u8]) -> io::Result<()> { + self.buf.extend_from_slice(b); + if self.buf.len() >= 256 << 10 { + self.flush()?; + } + Ok(()) + } + fn flush(&mut self) -> io::Result<()> { + self.h.update(&self.buf); + self.gz.write_all(&self.buf)?; + self.buf.clear(); + Ok(()) + } + fn uvarint(&mut self, mut v: u64) -> io::Result<()> { + let (mut b, mut i) = ([0u8; 10], 0); + while v >= 0x80 { + b[i] = v as u8 | 0x80; + v >>= 7; + i += 1; + } + b[i] = v as u8; + self.bytes(&b[..=i]) + } + fn varint(&mut self, v: i64) -> io::Result<()> { + self.uvarint(((v << 1) ^ (v >> 63)) as u64) // zig-zag, as encoding/binary.PutVarint + } + fn str(&mut self, s: &[u8]) -> io::Result<()> { + self.uvarint(s.len() as u64)?; + self.bytes(s) + } +} + +/// Writes t to out. The caller holds t's read lock. +pub fn save(out: &mut dyn Write, t: &Tree) -> io::Result<()> { + let mut bw = BufWriter::with_capacity(1 << 20, out); + bw.write_all(brand::SNAPSHOT_MAGIC)?; + bw.write_all(&VERSION.to_le_bytes())?; + bw.write_all(&FLAG_GZIPPED.to_le_bytes())?; + let mut w = Writer { gz: GzEncoder::new(bw, Compression::fast()), h: Sha256::new(), buf: Vec::with_capacity(256 << 10) }; + + let st = &t.stats; + let m = Meta { + root: String::from_utf8_lossy(&st.root).into_owned(), + created: GoTime::from_system(SystemTime::now()), + start: GoTime::from_system(st.start).local(), + end: st.end.map(|e| GoTime::from_system(e).local()).unwrap_or_default(), + complete: st.complete, + cancelled: st.cancelled, + files: st.files, + dirs: st.dirs, + symlinks: st.symlinks, + others: st.others, + err_counts: Some(st.err_counts.to_vec()), + excluded: st.excluded, + skipped_mounts: st.skipped_mounts, + virtual_skipped: st.virtual_skipped, + loops_skipped: st.loops_skipped, + broken_links: st.broken_links, + hardlink_dups: st.hardlink_dups, + unscanned: st.unscanned, + excludes: Some(st.excludes.clone()), + one_file_system: st.one_file_system, + follow: st.follow.clone(), + workers: st.workers as i64, + host: hostname(), + generator: format!("{} {}", brand::NAME, brand::VERSION), + }; + let mb = serde_json::to_vec(&m)?; + w.uvarint(mb.len() as u64)?; + w.bytes(&mb)?; + + w.uvarint(t.ext_count() as u64)?; + for i in 0..t.ext_count() { + w.str(t.ext_by_index(i).as_bytes())?; + } + w.uvarint(t.len() as u64)?; + for id in 0..t.len() as NodeId { + let n = t.node(id); + w.uvarint(if id == 0 { 0 } else { (id - n.parent) as u64 })?; + w.bytes(&[n.kind as u8, n.cat as u8])?; + w.uvarint(n.flags as u64)?; + w.str(&n.name)?; + w.varint(n.size)?; + w.varint(n.alloc)?; + w.varint(n.mtime)?; + for v in [n.mode, n.uid, n.gid, n.nlink, n.ext as u32] { + w.uvarint(v as u64)?; + } + } + w.uvarint(st.errors.len() as u64)?; + for e in &st.errors { + w.uvarint(e.node as u64)?; + w.bytes(&[e.kind as u8])?; + w.str(&e.name)?; + w.str(e.msg.as_bytes())?; + } + w.flush()?; + let mut bw = w.gz.finish()?; + bw.write_all(&w.h.finalize())?; + bw.flush() +} + +/// Writes a snapshot atomically (temp file + rename). +pub fn save_file(path: &Path, t: &Tree) -> io::Result<()> { + let dir = match path.parent() { + Some(d) if !d.as_os_str().is_empty() => d, + _ => Path::new("."), + }; + let (tmp, mut f) = create_temp(dir)?; + let res = save(&mut f, t).and_then(|_| { + drop(f); + fs::rename(&tmp, path) + }); + if res.is_err() { + let _ = fs::remove_file(&tmp); + } + res +} + +/// Creates a new 0600 file named .mapsize-*.tmp in dir (like os.CreateTemp). +fn create_temp(dir: &Path) -> io::Result<(PathBuf, File)> { + let seed = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().subsec_nanos() ^ std::process::id(); + for i in 0..10000u32 { + let p = dir.join(format!(".mapsize-{}.tmp", seed.wrapping_add(i.wrapping_mul(7919)))); + match OpenOptions::new().write(true).create_new(true).mode(0o600).open(&p) { + Ok(f) => return Ok((p, f)), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + } + } + Err(io::Error::new(io::ErrorKind::AlreadyExists, "snapshot: cannot create temporary file")) +} + +fn hostname() -> String { + let mut b = [0u8; 256]; + if unsafe { libc::gethostname(b.as_mut_ptr() as *mut libc::c_char, b.len()) } != 0 { + return String::new(); + } + let n = b.iter().position(|&c| c == 0).unwrap_or(b.len()); + String::from_utf8_lossy(&b[..n]).into_owned() +} + +/// The decompressed body, fed in chunks by the decompressing thread. +struct Body { + rx: Receiver, String>>, + buf: Vec, + pos: usize, + err: Option, +} + +const EOF: &str = "EOF"; +const UNEXPECTED_EOF: &str = "unexpected EOF"; + +impl Body { + fn new(rx: Receiver, String>>) -> Body { + Body { rx, buf: Vec::new(), pos: 0, err: None } + } + + /// Ensures unread bytes are buffered; false at end of stream. + fn fill(&mut self) -> Result { + while self.pos == self.buf.len() { + if let Some(e) = &self.err { + return Err(e.clone()); + } + match self.rx.recv() { + Ok(Ok(b)) => (self.buf, self.pos) = (b, 0), + Ok(Err(e)) => self.err = Some(e), + Err(_) => return Ok(false), + } + } + Ok(true) + } + + /// The next byte; Err("EOF") at end of stream. + #[inline] + fn byte(&mut self) -> Result { + if self.pos == self.buf.len() && !self.fill()? { + return Err(EOF.into()); + } + self.pos += 1; + Ok(self.buf[self.pos - 1]) + } + + /// Fills out completely (errors like io.ReadFull). + fn read_full(&mut self, out: &mut [u8]) -> Result<(), String> { + let mut done = 0; + while done < out.len() { + if !self.fill()? { + return Err(if done == 0 { EOF } else { UNEXPECTED_EOF }.into()); + } + let k = (out.len() - done).min(self.buf.len() - self.pos); + out[done..done + k].copy_from_slice(&self.buf[self.pos..self.pos + k]); + self.pos += k; + done += k; + } + Ok(()) + } + + fn byte1(&mut self) -> Result { + self.byte().map_err(|e| format!("snapshot: truncated: {e}")) + } + + fn uvarint(&mut self, max: u64, what: &str) -> Result { + let v = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?; + if v > max { + return Err(format!("snapshot: {what} {v} exceeds limit {max}")); + } + Ok(v) + } + + fn varint(&mut self, what: &str) -> Result { + let ux = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?; + let v = (ux >> 1) as i64; // zig-zag, as encoding/binary.PutVarint + Ok(if ux & 1 != 0 { !v } else { v }) + } + + fn bytes_n(&mut self, n: u64) -> Result, String> { + let mut b = vec![0u8; n as usize]; + self.read_full(&mut b).map_err(|e| format!("snapshot: truncated: {e}"))?; + Ok(b) + } + + fn str(&mut self, max: u64, what: &str) -> Result, String> { + let n = self.uvarint(max, &format!("{what} length"))?; + self.bytes_n(n) + } +} + +/// encoding/binary.ReadUvarint over the body. +fn read_uvarint(b: &mut Body) -> Result { + let (mut x, mut s) = (0u64, 0u32); + for i in 0..10 { + let c = match b.byte() { + Ok(c) => c, + Err(e) if i > 0 && e == EOF => return Err(UNEXPECTED_EOF.into()), + Err(e) => return Err(e), + }; + if c < 0x80 { + if i == 9 && c > 1 { + break; + } + return Ok(x | (c as u64) << s); + } + x |= ((c & 0x7f) as u64) << s; + s += 7; + } + Err("varint overflows 64 bits".into()) +} + +const BOMB: &str = "snapshot: decompressed size exceeds limit (possible decompression bomb)"; + +/// Allows decompressed bodies up to 64× the compressed file size (real +/// snapshots compress ~6×), with a 64 MiB floor for tiny files. +fn body_limit(file_size: i64) -> i64 { + (64 << 20).max(file_size.saturating_mul(64)) +} + +/// Caps the node count for a file of file_size bytes. Real snapshots spend +/// ~15 compressed bytes per node; a crafted one describes a node in a +/// fraction of a byte, so without this a 1 MB file can demand gigabytes of +/// memory while staying within body_limit. +fn node_limit(file_size: i64) -> u64 { + (1u64 << 20).max(file_size.max(0) as u64) +} + +/// Reads and validates a snapshot, allowing up to DEFAULT_MAX_BODY bytes of +/// decompressed data. +pub fn load(r: impl Read) -> Result { + load_limit(r, DEFAULT_MAX_BODY) +} + +/// load with an explicit decompressed-size limit. +pub fn load_limit(r: impl Read, max_body: i64) -> Result { + load_inner(r, max_body, MAX_NODES).map_err(Into::into) +} + +/// Opens and loads a snapshot file. +pub fn load_file(path: &Path) -> Result { + let f = File::open(path)?; + let (limit, nodes) = match f.metadata() { + Ok(m) => (body_limit(m.len() as i64), node_limit(m.len() as i64)), + Err(_) => (DEFAULT_MAX_BODY, MAX_NODES), + }; + load_inner(f, limit, nodes).map_err(Into::into) +} + +fn load_inner(input: impl Read, max_body: i64, max_nodes: u64) -> Result { + let mut br = BufReader::with_capacity(1 << 20, input); + let mut hdr = [0u8; 12]; + br.read_exact(&mut hdr).map_err(|_| "snapshot: file too short".to_string())?; + if &hdr[..8] != brand::SNAPSHOT_MAGIC { + return Err(format!("snapshot: not a {} snapshot", brand::NAME)); + } + let v = u16::from_le_bytes([hdr[8], hdr[9]]); + if v != VERSION { + return Err(format!("snapshot: unsupported version {v} (this build reads {VERSION})")); + } + if u16::from_le_bytes([hdr[10], hdr[11]]) & FLAG_GZIPPED == 0 { + return Err("snapshot: uncompressed bodies are not supported".into()); + } + // Decompression and hashing run on this thread while a scoped thread + // parses, overlapping the two. The parser only sees end of stream once + // the channel's sender is dropped, i.e. after decompression finished. + let mut h = Sha256::new(); + let res = { + let mut gz = GzDecoder::new(&mut br); // single member, like gz.Multistream(false) + std::thread::scope(|s| { + let (tx, rx) = crossbeam_channel::bounded(4); + let parser = s.spawn(move || parse(&mut Body::new(rx), max_nodes)); + let mut left = max_body; + loop { + if left <= 0 { + let _ = tx.send(Err(BOMB.to_string())); + break; + } + let mut buf = vec![0u8; (256 << 10).min(left as usize)]; + match gz.read(&mut buf) { + Ok(0) => break, + Ok(n) => { + buf.truncate(n); + left -= n as i64; + h.update(&buf); + if tx.send(Ok(buf)).is_err() { + break; // parser bailed out + } + } + Err(e) if e.kind() == io::ErrorKind::Interrupted => {} + Err(e) => { + let _ = tx.send(Err(format!("gzip: {e}"))); + break; + } + } + } + drop(tx); + parser.join().unwrap_or_else(|p| std::panic::resume_unwind(p)) + }) + }; + let (mut t, m) = res?; + // The checksum follows the gzip stream. + let mut sum = [0u8; 32]; + br.read_exact(&mut sum).map_err(|_| "snapshot: missing checksum".to_string())?; + if sum[..] != h.finalize()[..] { + return Err("snapshot: checksum mismatch (file corrupt)".into()); + } + + t.recompute(); + let st = &mut t.stats; + st.start = m.start.to_system(); + st.end = (!m.end.is_zero()).then(|| m.end.to_system()); + st.complete = true; + st.cancelled = m.cancelled; + (st.files, st.dirs, st.symlinks, st.others) = (m.files, m.dirs, m.symlinks, m.others); + for (d, s) in st.err_counts.iter_mut().zip(m.err_counts.unwrap_or_default()) { + *d = s; + } + (st.excluded, st.skipped_mounts, st.virtual_skipped) = (m.excluded, m.skipped_mounts, m.virtual_skipped); + (st.loops_skipped, st.broken_links, st.hardlink_dups, st.unscanned) = + (m.loops_skipped, m.broken_links, m.hardlink_dups, m.unscanned); + st.excludes = m.excludes.unwrap_or_default(); + st.one_file_system = m.one_file_system; + st.follow = m.follow; + st.workers = m.workers.max(0) as usize; + st.from_snapshot = format!("{} ({}, {})", m.created.local_minutes(), m.host, m.generator); + Ok(t) +} + +/// Parses the decompressed body up to (and including) its end. +fn parse(r: &mut Body, max_nodes: u64) -> Result<(Tree, Meta), String> { + let n = r.uvarint(MAX_META_LEN, "metadata length")?; + let mb = r.bytes_n(n)?; + let m: Meta = serde_json::from_slice(&mb).map_err(|e| format!("snapshot: metadata: {e}"))?; + let mut t = Tree::new(m.root.as_bytes(), Kind::Dir); + + let n_ext = r.uvarint(MAX_EXTS, "extension count")?; + let mut exts = Vec::with_capacity(n_ext.min(1024) as usize); + for _ in 0..n_ext { + exts.push(String::from_utf8_lossy(&r.str(MAX_EXT_LEN, "extension")?).into_owned()); + } + let n_nodes = r.uvarint(max_nodes, "node count")?; + if n_nodes == 0 { + return Err("snapshot: no root node".into()); + } + let mut depth: Vec = vec![0]; // per node, for the depth limit; parents precede children + for i in 0..n_nodes { + let pd = r.uvarint(MAX_NODES, "parent")?; + let kind = r.byte1()?; + let cat = r.byte1()?; + let flags = r.uvarint((1 << 16) - 1, "flags")? as u16; + let name = r.str(MAX_NAME_LEN, "name")?; + let (size, alloc, mtime) = (r.varint("size")?, r.varint("alloc")?, r.varint("mtime")?); + let mode = r.uvarint(u32::MAX as u64, "mode")? as u32; + let uid = r.uvarint(u32::MAX as u64, "uid")? as u32; + let gid = r.uvarint(u32::MAX as u64, "gid")? as u32; + let nlink = r.uvarint(u32::MAX as u64, "nlink")? as u32; + let ext = r.uvarint(n_ext, "ext")?; + if kind > Kind::Other as u8 || cat as usize >= NUM_CATEGORIES { + return Err(format!("snapshot: node {i}: invalid kind/category")); + } + let kind = Kind::from_u8(kind); + if size < 0 || alloc < 0 || size > MAX_SIZE || alloc > MAX_SIZE { + return Err(format!("snapshot: node {i}: size out of range")); + } + let id = if i == 0 { + if pd != 0 || kind != Kind::Dir && n_nodes > 1 { + return Err("snapshot: invalid root".into()); + } + t.node_mut(0).kind = kind; + 0 + } else { + if pd == 0 || pd > i { + return Err(format!("snapshot: node {i}: invalid parent reference")); + } + let parent = (i - pd) as NodeId; + if t.node(parent).kind != Kind::Dir { + return Err(format!("snapshot: node {i}: parent is not a directory")); + } + if !valid_name(&name) { + return Err(format!("snapshot: node {i}: invalid name {:?}", String::from_utf8_lossy(&name))); + } + let d = depth[parent as usize] + 1; + if d > MAX_DEPTH { + return Err(format!("snapshot: node {i}: deeper than {MAX_DEPTH} levels")); + } + depth.push(d); + t.add(parent, &name, kind) + }; + let n = t.node_mut(id); + n.cat = Category::from_u8(cat); + n.flags = flags; + (n.size, n.alloc, n.mtime) = (size, alloc, mtime); + (n.mode, n.uid, n.gid, n.nlink) = (mode, uid, gid, nlink); + if kind != Kind::Dir && (ext as usize) < exts.len() { + let e = t.intern_ext(&exts[ext as usize]); + t.node_mut(id).ext = e; + } + } + let n_err = r.uvarint(MAX_ERROR_RECORDS as u64, "error count")?; + for _ in 0..n_err { + let node = r.uvarint(n_nodes - 1, "error node")? as NodeId; + let kind = ErrKind::from_u8(r.byte1()?); // out-of-range kinds become Other + let name = r.str(MAX_NAME_LEN, "error name")?; + let msg = String::from_utf8_lossy(&r.str(MAX_MSG_LEN, "error message")?).into_owned(); + t.stats.errors.push(ErrorRecord { node, kind, name, msg }); + } + // The body must end here. + if r.byte() != Err(EOF.into()) { + return Err("snapshot: trailing data in body".into()); + } + Ok((t, m)) +} + +fn valid_name(s: &[u8]) -> bool { + !s.is_empty() && s != b"." && s != b".." && !s.iter().any(|&c| c == b'/' || c == 0) +} + +/// A Go time.Time as JSON carries it: unix seconds plus nanoseconds, and +/// the UTC offset it is written with. The default is Go's zero time +/// (0001-01-01T00:00:00Z). +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +struct GoTime { + secs: i64, + nanos: u32, + off: i32, // seconds east of UTC; presentation only +} + +const GO_ZERO_SECS: i64 = -62_135_596_800; + +impl Default for GoTime { + fn default() -> Self { + GoTime { secs: GO_ZERO_SECS, nanos: 0, off: 0 } + } +} + +impl GoTime { + fn from_system(t: SystemTime) -> GoTime { + match t.duration_since(UNIX_EPOCH) { + Ok(d) => GoTime { secs: d.as_secs() as i64, nanos: d.subsec_nanos(), off: 0 }, + Err(e) => { + let d = e.duration(); + let (s, n) = (d.as_secs() as i64, d.subsec_nanos()); + let (secs, nanos) = if n == 0 { (-s, 0) } else { (-s - 1, 1_000_000_000 - n) }; + GoTime { secs, nanos, off: 0 } + } + } + } + + /// The same instant in the local time zone (as Go's time.Now()). + fn local(self) -> GoTime { + let off = localtime(self.secs).map_or(0, |tm| tm.tm_gmtoff as i32); + GoTime { off, ..self } + } + + fn is_zero(self) -> bool { + (self.secs, self.nanos) == (GO_ZERO_SECS, 0) + } + + fn to_system(self) -> SystemTime { + if self.secs >= 0 { + UNIX_EPOCH + Duration::new(self.secs as u64, self.nanos) + } else { + UNIX_EPOCH - Duration::from_secs(self.secs.unsigned_abs()) + Duration::from_nanos(self.nanos as u64) + } + } + + /// RFC 3339 with trimmed nanoseconds, like Go's MarshalJSON. + fn format(self) -> String { + let secs = self.secs + self.off as i64; + let (y, mo, d) = civil_from_days(secs.div_euclid(86400)); + let rem = secs.rem_euclid(86400); + let mut s = format!("{y:04}-{mo:02}-{d:02}T{:02}:{:02}:{:02}", rem / 3600, rem / 60 % 60, rem % 60); + if self.nanos != 0 { + s += format!(".{:09}", self.nanos).trim_end_matches('0'); + } + if self.off == 0 { + return s + "Z"; + } + let a = self.off.unsigned_abs() / 60; + format!("{s}{}{:02}:{:02}", if self.off < 0 { '-' } else { '+' }, a / 60, a % 60) + } + + /// Parses RFC 3339 ("2026-10-08T10:40:00.123456789+01:00"). + fn parse(s: &str) -> Option { + let b = s.as_bytes(); + let num = |r: std::ops::Range| -> Option { + let p = b.get(r)?; + p.iter().all(u8::is_ascii_digit).then(|| p.iter().fold(0i64, |a, &c| a * 10 + (c - b'0') as i64)) + }; + if b.len() < 20 || b[4] != b'-' || b[7] != b'-' || b[10] != b'T' || b[13] != b':' || b[16] != b':' { + return None; + } + let (y, mo, d, h, mi, se) = (num(0..4)?, num(5..7)?, num(8..10)?, num(11..13)?, num(14..16)?, num(17..19)?); + if !(1..=12).contains(&mo) || !(1..=31).contains(&d) || h > 23 || mi > 59 || se > 59 { + return None; + } + let mut i = 19; + let mut nanos = 0u32; + if b[i] == b'.' { + let start = i + 1; + i = start; + while i < b.len() && b[i].is_ascii_digit() { + if i - start < 9 { + nanos = nanos * 10 + (b[i] - b'0') as u32; + } + i += 1; + } + if i == start { + return None; + } + for _ in (i - start)..9 { + nanos *= 10; + } + } + let off = match b.get(i)? { + b'Z' if b.len() == i + 1 => 0, + &c @ (b'+' | b'-') if b.len() == i + 6 && b[i + 3] == b':' => { + let (oh, om) = (num(i + 1..i + 3)?, num(i + 4..i + 6)?); + if oh > 23 || om > 59 { + return None; + } + (oh * 3600 + om * 60) * if c == b'-' { -1 } else { 1 } + } + _ => return None, + }; + let secs = days_from_civil(y, mo, d) * 86400 + h * 3600 + mi * 60 + se - off; + Some(GoTime { secs, nanos, off: off as i32 }) + } + + /// "2006-01-02 15:04" in local time. + fn local_minutes(self) -> String { + let Some(tm) = localtime(self.secs) else { return self.format() }; + format!("{:04}-{:02}-{:02} {:02}:{:02}", tm.tm_year as i64 + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min) + } +} + +fn localtime(secs: i64) -> Option { + let t = secs as libc::time_t; + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + (!unsafe { libc::localtime_r(&t, &mut tm) }.is_null()).then_some(tm) +} + +impl Serialize for GoTime { + fn serialize(&self, s: S) -> Result { + s.serialize_str(&self.format()) + } +} + +impl<'de> Deserialize<'de> for GoTime { + fn deserialize>(d: D) -> Result { + // null leaves the zero time, as in Go. + match Option::::deserialize(d)? { + None => Ok(GoTime::default()), + Some(s) => GoTime::parse(&s).ok_or_else(|| serde::de::Error::custom(format!("parsing time {s:?}"))), + } + } +} + +/// Civil-from-days (Howard Hinnant). +fn civil_from_days(days: i64) -> (i64, i64, i64) { + let z = days + 719468; + let era = z.div_euclid(146097); + let doe = z - era * 146097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + (yoe + era * 400 + (m <= 2) as i64, m, d) +} + +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::inventory::{SizeMode, FLAG_SPARSE}; + use std::collections::HashMap; + + fn sample() -> Tree { + let mut t = Tree::new(b"/srv", Kind::Dir); + let a = t.add(0, b"a", Kind::Dir); + let b = t.add(0, b"b", Kind::Dir); + let f1 = t.add(a, b"x.iso", Kind::File); + (t.node_mut(f1).size, t.node_mut(f1).alloc) = (1000, 1024); + let f2 = t.add(b, b"y.log", Kind::File); + (t.node_mut(f2).size, t.node_mut(f2).alloc) = (50, 4096); + t.node_mut(f2).flags = FLAG_SPARSE; + t.recompute(); + (t.stats.files, t.stats.dirs) = (2, 3); + t.stats.add_error(ErrorRecord { node: b, kind: ErrKind::Permission, msg: "denied".into(), ..Default::default() }); + t + } + + fn saved(t: &Tree) -> Vec { + let mut buf = Vec::new(); + save(&mut buf, t).unwrap(); + buf + } + + #[test] + fn round_trip() { + let src = sample(); + let got = load(&saved(&src)[..]).unwrap(); + assert_eq!(got.len(), src.len()); + for i in 0..src.len() as NodeId { + let (a, b) = (src.node(i), got.node(i)); + assert!( + a.name == b.name && a.tot_size == b.tot_size && a.tot_alloc == b.tot_alloc && a.flags == b.flags + && src.path(i) == got.path(i) && src.ext_name(a) == got.ext_name(b), + "node {i} differs: {a:?} vs {b:?}" + ); + } + assert_eq!(got.stats.err_counts[ErrKind::Permission as usize], 1); + assert_eq!(got.stats.errors.len(), 1); + assert_eq!(got.stats.start, src.stats.start); + assert_eq!(got.stats.end, None); + } + + #[test] + fn corruption_detected() { + let data = saved(&sample()); + let mut bad = data.clone(); + *bad.last_mut().unwrap() ^= 0xff; // flip a byte in the checksum trailer + assert!(load(&bad[..]).is_err(), "checksum corruption not detected"); + // Every truncation must fail cleanly. + for i in (0..data.len() - 1).step_by(7) { + assert!(load(&data[..i]).is_err(), "truncation at {i} accepted"); + } + } + + #[test] + fn compare_trees() { + let old = sample(); + let mut nw = Tree::new(b"/srv", Kind::Dir); + let a = nw.add(0, b"a", Kind::Dir); + let f1 = nw.add(a, b"x.iso", Kind::File); + (nw.node_mut(f1).size, nw.node_mut(f1).alloc) = (5000, 5120); + let c = nw.add(0, b"c", Kind::Dir); + let f3 = nw.add(c, b"z", Kind::File); + (nw.node_mut(f3).size, nw.node_mut(f3).alloc) = (10, 4096); + nw.recompute(); + + let d = compare(old, &nw); + let al = SizeMode::Allocated; + assert_eq!(d.status_of(&nw, f1, al), Status::Grew); + assert_eq!(d.delta(&nw, f1, al), 4096); + assert_eq!(d.status_of(&nw, c, al), Status::Added); + assert_eq!(d.removed.len(), 1); + assert_eq!(&d.old.node(d.removed[0]).name[..], b"b"); + let ch = d.changes(&nw, 10, al); + let paths: HashMap<&str, Status> = ch.iter().map(|c| (c.path.as_str(), c.status)).collect(); + assert_eq!(paths.get("/srv/a/x.iso"), Some(&Status::Grew), "{ch:?}"); + assert_eq!(paths.get("/srv/b"), Some(&Status::Removed), "{ch:?}"); + assert_eq!(paths.get("/srv/c"), Some(&Status::Added), "{ch:?}"); + assert!(!paths.contains_key("/srv/a"), "/srv/a should be explained by its child"); + assert!(!paths.contains_key("/srv/c/z"), "contents of new dirs should not be listed separately"); + } + + fn body(b: &[u8]) -> Body { + let (tx, rx) = crossbeam_channel::unbounded(); + tx.send(Ok(b.to_vec())).unwrap(); + Body::new(rx) + } + + fn put_uvarint(v: u64) -> Vec { + let mut w = Writer { gz: GzEncoder::new(Vec::new(), Compression::fast()), h: Sha256::new(), buf: Vec::new() }; + w.uvarint(v).unwrap(); + w.buf + } + + #[test] + fn read_uvarint_matches_go() { + for v in [0, 1, 127, 128, 300, 1 << 35, u64::MAX] { + assert_eq!(read_uvarint(&mut body(&put_uvarint(v))), Ok(v)); + } + assert_eq!(put_uvarint(300), [0xac, 0x02]); + assert!(read_uvarint(&mut body(&[0xff; 11])).is_err(), "overflow not detected"); + // Zig-zag as Go's PutVarint. + for (v, enc) in [(0i64, 0u64), (-1, 1), (1, 2), (-2, 3), (i64::MIN, u64::MAX)] { + assert_eq!(((v << 1) ^ (v >> 63)) as u64, enc); + assert_eq!(body(&put_uvarint(enc)).varint("v"), Ok(v)); + } + } + + #[test] + fn go_time_json() { + let t = GoTime::parse("2026-10-08T10:40:00.123456789+01:00").unwrap(); + assert_eq!((t.secs, t.nanos, t.off), (1_791_452_400, 123_456_789, 3600)); + assert_eq!(t.format(), "2026-10-08T10:40:00.123456789+01:00"); + assert_eq!(GoTime { off: 0, ..t }.format(), "2026-10-08T09:40:00.123456789Z"); + assert_eq!(GoTime { off: -5 * 3600 - 1800, ..t }.format(), "2026-10-08T04:10:00.123456789-05:30"); + assert_eq!(GoTime::parse("2026-10-08T09:40:00.1Z").unwrap().format(), "2026-10-08T09:40:00.1Z"); + assert_eq!(GoTime::parse("0001-01-01T00:00:00Z"), Some(GoTime::default())); + assert_eq!(GoTime::default().format(), "0001-01-01T00:00:00Z"); + assert_eq!(GoTime::from_system(GoTime::default().to_system()), GoTime::default()); + for bad in ["", "2026-10-08", "2026-13-08T00:00:00Z", "2026-10-08T00:00:00", "2026-10-08T00:00:00.Z"] { + assert_eq!(GoTime::parse(bad), None, "{bad}"); + } + } + + /// A tiny tree saved by the Go binary + /// (tree/{a/x.iso 10000 B, a/b/note.txt, c/y.log 3000 B, c/link -> ../a}). + #[test] + fn loads_go_snapshot() { + let t = load_file(Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/src/snapshot/testdata/go.msz"))).unwrap(); + let r = t.node(0); + assert_eq!((r.tot_size, r.tot_alloc), (13310, 20480)); + assert_eq!((t.stats.files, t.stats.dirs, t.stats.symlinks), (3, 4, 1)); + assert_eq!(t.len(), 8); + assert!(t.stats.root.ends_with(b"/tree")); + assert!(t.stats.end.is_some() && t.stats.complete); + assert!(t.stats.from_snapshot.contains("mapsize"), "{}", t.stats.from_snapshot); + let iso = (0..t.len() as NodeId).find(|&i| &t.node(i).name[..] == b"x.iso").unwrap(); + assert_eq!(t.node(iso).size, 10000); + assert_eq!(t.ext_name(t.node(iso)), "iso"); + assert!(t.path_string(iso).ends_with("/tree/a/x.iso")); + // And it survives a Rust round trip unchanged. + let again = load(&saved(&t)[..]).unwrap(); + assert_eq!((again.node(0).tot_size, again.len(), again.stats.start), (13310, 8, t.stats.start)); + } + + // Hostile input. + + /// Builds a well-formed (correctly checksummed) but hostile snapshot. + fn craft(n: usize, meta: &str, rec: impl Fn(usize, &mut Writer>)) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(brand::SNAPSHOT_MAGIC); + out.extend_from_slice(&VERSION.to_le_bytes()); + out.extend_from_slice(&FLAG_GZIPPED.to_le_bytes()); + let mut w = Writer { gz: GzEncoder::new(out, Compression::best()), h: Sha256::new(), buf: Vec::new() }; + w.uvarint(meta.len() as u64).unwrap(); + w.bytes(meta.as_bytes()).unwrap(); + w.uvarint(1).unwrap(); + w.str(b"").unwrap(); + w.uvarint(n as u64).unwrap(); + for i in 0..n { + rec(i, &mut w); + } + w.uvarint(0).unwrap(); + w.flush().unwrap(); + let mut out = w.gz.finish().unwrap(); + let sum = w.h.finalize(); + out.extend_from_slice(&sum); + out + } + + fn node(w: &mut Writer>, pd: u64, kind: Kind, name: &str, size: i64) { + w.uvarint(pd).unwrap(); + w.bytes(&[kind as u8, 0]).unwrap(); + w.uvarint(0).unwrap(); + w.str(name.as_bytes()).unwrap(); + w.varint(size).unwrap(); + w.varint(size).unwrap(); + w.varint(0).unwrap(); + for _ in 0..5 { + w.uvarint(0).unwrap(); + } + } + + fn chain(n: usize) -> Vec { + craft(n, r#"{"root":"/r"}"#, |i, w| if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::Dir, "a", 0) }) + } + + #[test] + fn deep_chain_rejected() { + load(&chain(MAX_DEPTH as usize + 1)[..]).expect("depth at the limit must load"); + let err = load(&chain(MAX_DEPTH as usize + 2)[..]).err().expect("over-deep chain accepted"); + assert!(err.to_string().contains("deeper"), "{err}"); + } + + #[test] + fn decompression_bomb_limited() { + let flat = craft(100_000, r#"{"root":"/r"}"#, |i, w| { + if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", 0) } + }); + let err = load_limit(&flat[..], 64 << 10).err().expect("bomb not limited"); + assert!(err.to_string().contains("limit"), "{err}"); + assert!(body_limit(flat.len() as i64) >= 64 << 20, "body_limit floor"); + } + + #[test] + fn huge_sizes_do_not_wrap() { + let data = craft(3, r#"{"root":"/r"}"#, |i, w| { + if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", MAX_SIZE) } + }); + let tr = load(&data[..]).unwrap(); + assert!(tr.node(0).tot_size >= 0, "aggregate wrapped negative"); + let over = craft(2, r#"{"root":"/r"}"#, |i, w| { + if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::File, "f", MAX_SIZE + 1) } + }); + assert!(load(&over[..]).is_err(), "size above MAX_SIZE accepted"); + } + + #[test] + fn hostile_metadata_sanitised_in_summary() { + let data = craft(1, r#"{"root":"/r\u001b[2J","excludes":["\u001b]0;PWNED\u0007"],"excluded":1}"#, |_, w| { + node(w, 0, Kind::Dir, "", 0) + }); + let mut tr = load(&data[..]).unwrap(); + tr.stats.excluded = 1; + let mut b = Vec::new(); + crate::export::summary(&mut b, &tr).unwrap(); + assert!(!b.iter().any(|&c| c == 0x1b || c == 0x07), "raw control characters in summary: {:?}", String::from_utf8_lossy(&b)); + } + + /// Blocks of one directory plus K-1 files repeat byte for byte, so a + /// small file describes millions of nodes; load_file must refuse before + /// allocating. + #[test] + fn node_amplification_limited() { + const K: usize = 1000; + const N: usize = 1_200_001; + let data = craft(N, r#"{"root":"/r"}"#, |i, w| match i { + 0 => node(w, 0, Kind::Dir, "", 0), + 1 => node(w, 1, Kind::Dir, "d", 0), + _ if (i - 1) % K == 0 => node(w, K as u64, Kind::Dir, "d", 0), + _ => node(w, ((i - 1) % K) as u64, Kind::File, "f", 0), + }); + load(&data[..]).expect("crafted tree is otherwise valid"); + let p = std::env::temp_dir().join(format!("mapsize-amp-{}.msz", std::process::id())); + fs::write(&p, &data).unwrap(); + let res = load_file(&p); + let _ = fs::remove_file(&p); + let err = res.err().unwrap_or_else(|| panic!("{N} nodes from a {}-byte file accepted", data.len())); + assert!(err.to_string().contains("node count"), "{err}"); + } + + #[test] + fn save_file_is_atomic_and_detected() { + let dir = std::env::temp_dir().join(format!("mapsize-save-{}", std::process::id())); + fs::create_dir_all(&dir).unwrap(); + let p = dir.join("s.msz"); + save_file(&p, &sample()).unwrap(); + assert!(is_snapshot(&p)); + assert_eq!(load_file(&p).unwrap().len(), 5); + assert_eq!(fs::read_dir(&dir).unwrap().count(), 1, "temp file left behind"); + fs::remove_dir_all(&dir).unwrap(); + } +} diff --git a/rust/src/snapshot/testdata/go.msz b/rust/src/snapshot/testdata/go.msz new file mode 100644 index 0000000..f7949d8 Binary files /dev/null and b/rust/src/snapshot/testdata/go.msz differ diff --git a/rust/src/textutil.rs b/rust/src/textutil.rs new file mode 100644 index 0000000..a64cea0 --- /dev/null +++ b/rust/src/textutil.rs @@ -0,0 +1,373 @@ +//! Display helpers: sanitising untrusted strings for the terminal, +//! width-aware truncation, and human-readable units. + +use std::fmt::Write; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use unicode_segmentation::UnicodeSegmentation; +use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; + +/// Makes an untrusted string (a filename, snapshot data) safe to write to a +/// terminal. Control characters (C0, DEL, C1) and bidi overrides become +/// visible escapes; invalid UTF-8 becomes U+FFFD. Keep the original bytes +/// for filesystem operations. +pub fn sanitize_bytes(s: &[u8]) -> String { + if s.iter().all(|&c| (0x20..0x7f).contains(&c)) { + // SAFETY-free: printable ASCII is valid UTF-8. + return String::from_utf8(s.to_vec()).unwrap(); + } + let mut b = String::with_capacity(s.len() + 8); + for chunk in s.utf8_chunks() { + for r in chunk.valid().chars() { + let c = r as u32; + if c < 0x20 || c == 0x7f || (0x80..=0x9f).contains(&c) { + let _ = write!(b, "\\x{c:02x}"); + } else if is_bidi_control(c) { + let _ = write!(b, "\\u{c:04x}"); + } else { + b.push(r); + } + } + for _ in chunk.invalid() { + b.push('\u{FFFD}'); + } + } + b +} + +/// sanitize_bytes for strings. +pub fn sanitize(s: &str) -> String { + sanitize_bytes(s.as_bytes()) +} + +fn is_bidi_control(r: u32) -> bool { + (0x202a..=0x202e).contains(&r) || (0x2066..=0x2069).contains(&r) || r == 0x200e || r == 0x200f || r == 0x061c +} + +/// Calls f for each grapheme cluster of s with its display width. Clusters +/// whose width terminals disagree on (per-char wcwidth versus grapheme +/// width, e.g. emoji ZWJ sequences) are replaced by U+FFFD with width 1, and +/// zero-width clusters are dropped, so measuring and drawing always agree +/// with the terminal. f returns false to stop. +pub fn clusters(s: &str, mut f: impl FnMut(&str, usize) -> bool) { + for c in s.graphemes(true) { + let mut chars = c.chars(); + let first = chars.next().unwrap(); + let w = if chars.next().is_none() { first.width().unwrap_or(0) } else { c.width() }; + if w == 0 { + continue; + } + let ok = if c.len() > 1 && c.chars().map(|r| r.width().unwrap_or(0)).sum::() != w { + f("\u{FFFD}", 1) + } else { + f(c, w) + }; + if !ok { + return; + } + } +} + +/// Display width of s in terminal cells. +pub fn width(s: &str) -> usize { + if s.bytes().all(|c| (0x20..0x7f).contains(&c)) { + return s.len(); + } + let mut n = 0; + clusters(s, |_, w| { + n += w; + true + }); + n +} + +/// Clips s to at most w cells, appending "…" if anything was cut. +pub fn truncate(s: &str, w: usize) -> String { + if w == 0 { + return String::new(); + } + if width(s) <= w { + return s.to_string(); + } + clip(s, w - 1) + "…" +} + +/// Clips from the left, prefixing "…" (for paths and breadcrumbs where the +/// tail is the informative part). +pub fn truncate_left(s: &str, w: usize) -> String { + if w == 0 { + return String::new(); + } + let total = width(s); + if total <= w { + return s.to_string(); + } + let mut skip = (total - (w - 1)) as isize; + let mut b = String::from("…"); + clusters(s, |c, cw| { + if skip > 0 { + skip -= cw as isize; + if skip < 0 { + // A wide cluster overshot by one cell; pad to keep width exact. + b.push_str(&" ".repeat((-skip) as usize)); + } + return true; + } + b.push_str(c); + true + }); + b +} + +/// The longest prefix of s that fits in w cells. +fn clip(s: &str, w: usize) -> String { + let mut b = String::new(); + let mut used = 0; + clusters(s, |c, cw| { + if used + cw > w { + return false; + } + used += cw; + b.push_str(c); + true + }); + b +} + +/// Pads s with spaces to exactly w cells (truncating if longer). +pub fn pad_right(s: &str, w: usize) -> String { + let mut s = truncate(s, w); + let sw = width(&s); + if sw < w { + s.push_str(&" ".repeat(w - sw)); + } + s +} + +/// Right-aligns s in w cells. +pub fn pad_left(s: &str, w: usize) -> String { + let s = truncate(s, w); + let sw = width(&s); + if sw < w { + return " ".repeat(w - sw) + &s; + } + s +} + +static SI: AtomicBool = AtomicBool::new(false); + +/// Selects decimal units (kB, MB, …) instead of IEC (KiB, MiB, …). Set once +/// at start-up. +pub fn set_si(v: bool) { + SI.store(v, Ordering::Relaxed); +} + +/// Whether decimal units are selected. +pub fn si() -> bool { + SI.load(Ordering::Relaxed) +} + +const IEC_UNITS: [&str; 7] = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"]; +const SI_UNITS: [&str; 7] = ["B", "kB", "MB", "GB", "TB", "PB", "EB"]; + +fn scaled(n: i64) -> (f64, &'static str) { + let (base, units) = if si() { (1000.0, &SI_UNITS) } else { (1024.0, &IEC_UNITS) }; + let neg = n < 0; + let mut v = (n as f64).abs(); + let mut i = 0; + while v >= base && i < units.len() - 1 { + v /= base; + i += 1; + } + (if neg { -v } else { v }, units[i]) +} + +fn num(v: f64, unit: &str) -> String { + let a = v.abs(); + if unit == "B" { + format!("{v:.0}") + } else if a < 10.0 { + format!("{v:.2}") + } else if a < 100.0 { + format!("{v:.1}") + } else { + format!("{v:.0}") + } +} + +/// Formats a byte count, e.g. "86.4 GiB". +pub fn size(n: i64) -> String { + let (v, u) = scaled(n); + num(v, u) + " " + u +} + +/// Formats a byte count in at most ~7 cells, e.g. "86GiB". +pub fn size_compact(n: i64) -> String { + let (v, u) = scaled(n); + if u != "B" && v.abs() < 10.0 { + format!("{v:.1}{u}") + } else { + format!("{v:.0}{u}") + } +} + +/// Formats a delta with an explicit sign. +pub fn signed_size(n: i64) -> String { + match n { + n if n > 0 => "+".to_string() + &size(n), + n if n < 0 => "-".to_string() + &size(n.saturating_neg()), + _ => "±0 B".to_string(), + } +} + +/// Formats an integer with thousands separators. +pub fn count(n: i64) -> String { + let s = n.unsigned_abs().to_string(); + let mut out = String::with_capacity(s.len() + s.len() / 3 + 1); + if n < 0 { + out.push('-'); + } + for (i, c) in s.chars().enumerate() { + if i > 0 && (s.len() - i) % 3 == 0 { + out.push(','); + } + out.push(c); + } + out +} + +/// Formats an elapsed time as mm:ss or h:mm:ss. +pub fn duration(d: Duration) -> String { + let s = d.as_secs(); + if s >= 3600 { + format!("{}:{:02}:{:02}", s / 3600, s / 60 % 60, s % 60) + } else { + format!("{:02}:{:02}", s / 60, s % 60) + } +} + +/// Formats part/whole as a percentage. +pub fn percent(part: i64, whole: i64) -> String { + if whole <= 0 { + return "–".to_string(); + } + let p = part as f64 * 100.0 / whole as f64; + if p < 10.0 { + format!("{p:.1}%") + } else { + format!("{p:.0}%") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_cases() { + let cases: [(&[u8], &str); 8] = [ + (b"plain.txt", "plain.txt"), + (b"evil\x1b[2Jname", r"evil\x1b[2Jname"), + (b"bell\x07", r"bell\x07"), + ("c1\u{009b}x".as_bytes(), r"c1\x9bx"), + (b"bad\xffutf8", "bad\u{FFFD}utf8"), + ("rtl\u{202e}gnp.exe".as_bytes(), "rtl\\u202egnp.exe"), + ("日本語".as_bytes(), "日本語"), + (b"tab\tnew\nline\rdel\x7f", r"tab\x09new\x0aline\x0ddel\x7f"), + ]; + for (input, want) in cases { + let got = sanitize_bytes(input); + assert_eq!(got, want); + assert!(!got.contains(['\x1b', '\x07', '\u{009b}'])); + } + } + + #[test] + fn truncation() { + assert_eq!(truncate("hello world", 5), "hell…"); + assert!(width(&truncate("日本語テキスト", 5)) <= 5); + assert_eq!(truncate("abc", 3), "abc"); + assert_eq!(truncate("abc", 0), ""); + let got = truncate_left("/a/very/long/path", 8); + assert!(width(&got) == 8 && got.ends_with("path"), "{got}"); + assert_eq!(width(&truncate_left("ab日本語", 4)), 4); + } + + #[test] + fn sizes() { + for (n, want) in [(0, "0 B"), (1023, "1023 B"), (1024, "1.00 KiB"), (1536, "1.50 KiB"), (181 << 30, "181 GiB"), (1 << 62, "4.00 EiB")] { + assert_eq!(size(n), want); + } + assert_eq!(count(2184392), "2,184,392"); + assert_eq!(signed_size(-2048), "-2.00 KiB"); + } + + #[test] + fn clusters_agree_with_width() { + for s in ["abc", "日本", "é", "👨‍👩‍👧", "🎉x", "\u{200b}"] { + let mut n = 0; + clusters(s, |_, w| { + n += w; + true + }); + assert_eq!(n, width(s), "{s:?}"); + } + } +} + +/// Formats unix nanoseconds as an RFC 3339 UTC timestamp, e.g. +/// "2026-10-08T09:30:00Z". +pub fn rfc3339_utc(unix_nanos: i64) -> String { + let secs = unix_nanos.div_euclid(1_000_000_000); + let (days, rem) = (secs.div_euclid(86400), secs.rem_euclid(86400)); + // Civil-from-days (Howard Hinnant). + let z = days + 719468; + let era = z.div_euclid(146097); + let doe = z - era * 146097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + (m <= 2) as i64; + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem / 60 % 60, rem % 60) +} + +/// Formats a duration like Go's time.Duration.String after rounding to +/// milliseconds, e.g. "1.234s", "250ms", "2m3.5s". +pub fn go_duration(d: Duration) -> String { + let ms = (d.as_nanos() + 500_000) / 1_000_000; + if ms == 0 { + return "0s".into(); + } + if ms < 1000 { + return format!("{ms}ms"); + } + let (h, m, s) = (ms / 3_600_000, ms / 60_000 % 60, ms % 60_000); + let mut secs = format!("{}.{:03}", s / 1000, s % 1000); + while secs.ends_with('0') { + secs.pop(); + } + if secs.ends_with('.') { + secs.pop(); + } + match (h, m) { + (0, 0) => format!("{secs}s"), + (0, m) => format!("{m}m{secs}s"), + (h, m) => format!("{h}h{m}m{secs}s"), + } +} + +#[cfg(test)] +mod time_tests { + use super::*; + + #[test] + fn formats() { + assert_eq!(rfc3339_utc(0), "1970-01-01T00:00:00Z"); + assert_eq!(rfc3339_utc(1_791_456_000 * 1_000_000_000), "2026-10-08T10:40:00Z"); + assert_eq!(go_duration(Duration::from_micros(1_234_400)), "1.234s"); + assert_eq!(go_duration(Duration::from_millis(250)), "250ms"); + assert_eq!(go_duration(Duration::from_millis(123_500)), "2m3.5s"); + } +} diff --git a/rust/src/treemap/mod.rs b/rust/src/treemap/mod.rs new file mode 100644 index 0000000..c412aea --- /dev/null +++ b/rust/src/treemap/mod.rs @@ -0,0 +1,302 @@ +//! Pure geometry: squarified layout on a cell grid, level-of-detail +//! selection and spatial neighbour search. No knowledge of filesystems or +//! terminals. + +mod nav; + +pub use nav::{neighbour, Direction}; + +/// An integer cell rectangle. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Rect { + pub x: i32, + pub y: i32, + pub w: i32, + pub h: i32, +} + +impl Rect { + /// Whether the rectangle has no cells. + pub fn empty(&self) -> bool { + self.w <= 0 || self.h <= 0 + } + + /// w*h. + pub fn area(&self) -> i32 { + self.w.max(0) * self.h.max(0) + } + + /// Whether the cell (x, y) is inside. + pub fn contains(&self, x: i32, y: i32) -> bool { + x >= self.x && x < self.x + self.w && y >= self.y && y < self.y + self.h + } + + /// Shrinks by n cells on each side. + pub fn inset(&self, n: i32) -> Rect { + Rect { x: self.x + n, y: self.y + n, w: (self.w - 2 * n).max(0), h: (self.h - 2 * n).max(0) } + } +} + +/// Something to lay out. Size must be > 0 to receive space. +#[derive(Clone, Copy, Debug)] +pub struct Item { + pub id: i64, + pub size: f64, +} + +/// A laid-out item. +#[derive(Clone, Copy, Debug, PartialEq)] +pub struct Block { + pub id: i64, + pub rect: Rect, +} + +/// Lays items out inside bounds using the squarified treemap algorithm +/// (Bruls, Huizing, van Wijk). Items with non-positive or non-finite size are +/// ignored. The output partitions bounds exactly: rectangle edges are snapped +/// to the grid by rounding shared float edges, so there are no gaps and no +/// overlaps. Items that round to zero width or height are omitted. The result +/// is deterministic for a given input set regardless of input order. +pub fn squarify(items: &[Item], bounds: Rect) -> Vec { + if bounds.empty() { + return Vec::new(); + } + let mut its: Vec = items.iter().copied().filter(|it| it.size > 0.0 && it.size.is_finite()).collect(); + let total: f64 = its.iter().map(|it| it.size).sum(); + if its.is_empty() || total <= 0.0 { + return Vec::new(); + } + its.sort_by(|a, b| b.size.total_cmp(&a.size).then(a.id.cmp(&b.id))); + let scale = bounds.w as f64 * bounds.h as f64 / total; + let areas: Vec = its.iter().map(|it| it.size * scale).collect(); + let frs = layout(&areas, bounds.x as f64, bounds.y as f64, bounds.w as f64, bounds.h as f64); + let mut out = Vec::with_capacity(its.len()); + for (it, f) in its.iter().zip(frs) { + let (x0, y0) = (f[0].round() as i32, f[1].round() as i32); + let x1 = ((f[0] + f[2]).round() as i32).min(bounds.x + bounds.w); + let y1 = ((f[1] + f[3]).round() as i32).min(bounds.y + bounds.h); + if x1 > x0 && y1 > y0 { + out.push(Block { id: it.id, rect: Rect { x: x0, y: y0, w: x1 - x0, h: y1 - y0 } }); + } + } + out +} + +/// The worst aspect ratio of a row with the given sum, largest and smallest +/// areas laid along a side of length side. +fn worst(sum: f64, largest: f64, smallest: f64, side: f64) -> f64 { + let (s2, w2) = (sum * sum, side * side); + (w2 * largest / s2).max(s2 / (w2 * smallest)) +} + +/// Float rectangles [x, y, w, h] for areas sorted descending. +fn layout(areas: &[f64], mut x: f64, mut y: f64, mut w: f64, mut h: f64) -> Vec<[f64; 4]> { + let n = areas.len(); + let mut out = vec![[0.0; 4]; n]; + let mut i = 0; + while i < n { + let short = w.min(h); + let mut j = i + 1; + let mut sum = areas[i]; + let mut cur = worst(sum, areas[i], areas[i], short); + while j < n { + let s2 = sum + areas[j]; + let nw = worst(s2, areas[i], areas[j], short); + if nw > cur { + break; + } + cur = nw; + sum = s2; + j += 1; + } + let last = j == n; + if w >= h { + let mut thick = sum / h; + if last || thick > w { + thick = w; + } + let mut yy = y; + for k in i..j { + let hh = if k == j - 1 { y + h - yy } else { areas[k] / sum * h }; + out[k] = [x, yy, thick, hh]; + yy += hh; + } + x += thick; + w -= thick; + } else { + let mut thick = sum / w; + if last || thick > h { + thick = h; + } + let mut xx = x; + for k in i..j { + let ww = if k == j - 1 { x + w - xx } else { areas[k] / sum * w }; + out[k] = [xx, y, ww, thick]; + xx += ww; + } + y += thick; + h -= thick; + } + i = j; + } + out +} + +/// How many leading items of a size-sorted (descending) list deserve their +/// own block when area cells represent total: items are kept while each +/// would get at least min_area cells and fewer than max_items have been +/// kept. The remainder should be merged into one group block. +pub fn visible(sizes: &[f64], total: f64, area: i32, min_area: i32, max_items: usize) -> usize { + if total <= 0.0 || area <= 0 { + return 0; + } + let cells = |s: f64| s / total * area as f64; + let mut k = 0; + while k < sizes.len() && k < max_items { + if cells(sizes[k]) < min_area as f64 { + break; + } + k += 1; + } + // Grouping a single leftover item gains nothing; show it directly if it + // gets at least one cell. + if k + 1 == sizes.len() && k < max_items && cells(sizes[k]) >= 1.0 { + k += 1; + } + k +} + +/// The block containing the cell (x, y). +pub fn at(blocks: &[Block], x: i32, y: i32) -> Option { + blocks.iter().find(|b| b.rect.contains(x, y)).map(|b| b.id) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + pub fn r(x: i32, y: i32, w: i32, h: i32) -> Rect { + Rect { x, y, w, h } + } + + pub fn it(id: i64, size: f64) -> Item { + Item { id, size } + } + + /// Small deterministic PRNG for property checks (splitmix64). + pub struct Rng(u64); + impl Rng { + pub fn new(seed: u64) -> Rng { + Rng(seed) + } + pub fn intn(&mut self, n: u64) -> u64 { + self.0 = self.0.wrapping_add(0x9E3779B97F4A7C15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58476D1CE4E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D049BB133111EB); + (z ^ (z >> 31)) % n + } + } + + fn check_partition(blocks: &[Block], b: Rect, exact: bool) { + let mut grid = vec![false; (b.w * b.h) as usize]; + for bl in blocks { + let br = bl.rect; + assert!(br.w > 0 && br.h > 0, "non-positive block {bl:?}"); + assert!(br.x >= b.x && br.y >= b.y && br.x + br.w <= b.x + b.w && br.y + br.h <= b.y + b.h, "block {bl:?} outside bounds {b:?}"); + for y in br.y..br.y + br.h { + for x in br.x..br.x + br.w { + let i = ((y - b.y) * b.w + (x - b.x)) as usize; + assert!(!grid[i], "overlap at {x},{y}"); + grid[i] = true; + } + } + } + if exact { + if let Some(i) = grid.iter().position(|v| !v) { + panic!("gap at {},{}", b.x + i as i32 % b.w, b.y + i as i32 / b.w); + } + } + } + + #[test] + fn squarify_basic() { + let b = r(2, 3, 60, 20); + let items = [it(1, 6.0), it(2, 6.0), it(3, 4.0), it(4, 3.0), it(5, 2.0), it(6, 2.0), it(7, 1.0)]; + let blocks = squarify(&items, b); + assert_eq!(blocks.len(), items.len()); + check_partition(&blocks, b, true); + // Areas roughly proportional. + for bl in &blocks { + let want = b.area() as f64 * items[(bl.id - 1) as usize].size / 24.0; + let got = bl.rect.area() as f64; + assert!(got >= want * 0.6 - 4.0 && got <= want * 1.4 + 4.0, "block {} area {got} want ~{want}", bl.id); + } + // Deterministic regardless of input order. + let mut rev = items.to_vec(); + rev.reverse(); + assert_eq!(blocks, squarify(&rev, b), "layout depends on input order"); + } + + #[test] + fn squarify_edge_cases() { + assert!(squarify(&[], r(0, 0, 10, 10)).is_empty()); + assert!(squarify(&[it(1, 0.0), it(2, -5.0)], r(0, 0, 10, 10)).is_empty(), "zero sizes should produce nothing"); + assert!(squarify(&[it(1, f64::NAN), it(2, f64::INFINITY)], r(0, 0, 10, 10)).is_empty()); + for b in [r(0, 0, 0, 0), r(0, 0, 1, 1), r(0, 0, 1, 50), r(0, 0, 50, 1), r(0, 0, -3, 4)] { + let bl = squarify(&[it(1, 5.0), it(2, 3.0), it(3, 1.0)], b); + if !b.empty() { + check_partition(&bl, b, true); + } + } + // One enormous item and many tiny ones. + let mut items = vec![it(0, 1e15)]; + items.extend((1..500).map(|i| it(i, 1.0))); + check_partition(&squarify(&items, r(0, 0, 80, 24)), r(0, 0, 80, 24), true); + } + + #[test] + fn resize_recomputes() { + let items = [it(1, 50.0), it(2, 30.0), it(3, 20.0)]; + for b in [r(0, 0, 80, 24), r(0, 0, 120, 40), r(0, 0, 300, 80), r(0, 0, 70, 20), r(0, 0, 180, 50)] { + let bl = squarify(&items, b); + check_partition(&bl, b, true); + assert_eq!(bl.len(), 3, "{b:?}"); + } + } + + /// The Go fuzz target's property, run over fixed seeds. + #[test] + fn squarify_random_partitions() { + let mut rng = Rng::new(1); + for _ in 0..300 { + let (w, h, n) = (rng.intn(400) as i32, rng.intn(150) as i32, rng.intn(3000) as usize); + let items: Vec = (0..n) + .map(|i| { + let bits = rng.intn(40) + 1; + it(i as i64, rng.intn(1 << bits) as f64) + }) + .collect(); + let b = r(rng.intn(10) as i32, rng.intn(10) as i32, w, h); + let bl = squarify(&items, b); + if !b.empty() && !bl.is_empty() { + check_partition(&bl, b, true); + } + } + } + + #[test] + fn visible_counts() { + let sizes = [100.0, 50.0, 10.0, 1.0, 1.0, 1.0]; + assert_eq!(visible(&sizes, 163.0, 1630, 20, 100), 3); + assert_eq!(visible(&sizes, 163.0, 1630, 20, 2), 2, "max_items"); + assert_eq!(visible(&[10.0, 1.0], 11.0, 1100, 200, 10), 2, "single leftover"); + } + + #[test] + fn at_finds_block() { + let bl = [Block { id: 1, rect: r(0, 0, 5, 5) }, Block { id: 2, rect: r(5, 0, 5, 5) }]; + assert_eq!(at(&bl, 6, 2), Some(2)); + assert_eq!(at(&bl, 10, 2), None); + } +} diff --git a/rust/src/treemap/nav.rs b/rust/src/treemap/nav.rs new file mode 100644 index 0000000..86187d7 --- /dev/null +++ b/rust/src/treemap/nav.rs @@ -0,0 +1,141 @@ +use super::Block; + +/// An arrow-key direction. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Direction { + Left, + Right, + Up, + Down, +} + +/// Finds the block a user expects to reach by pressing an arrow key from the +/// block with id sel: +/// +/// 1. Only blocks entirely beyond the selected block's edge in the +/// direction of travel are candidates. +/// 2. Blocks whose span overlaps the selected block's orthogonal span are +/// strongly preferred over all others. +/// 3. Then the smallest gap along the direction of travel wins. +/// 4. Then the largest overlap with the selected block's span. +/// 5. Then the block whose orthogonal span is closest to the selected +/// block's centre line (0 if it contains it). +/// 6. Ties break by position (top, then left) and finally id. +/// +/// None if there is no block in that direction. +pub fn neighbour(blocks: &[Block], sel: i64, d: Direction) -> Option { + let s = blocks.iter().find(|b| b.id == sel)?.rect; + // Lexicographic score, smaller is better; overlap is negated so larger wins. + let mut best: Option<(i32, i32, i32, i32, i32, i32, i64)> = None; + for b in blocks { + if b.id == sel { + continue; + } + let c = b.rect; + let (ok, gap, lo, hi, clo, chi) = match d { + Direction::Right => (c.x >= s.x + s.w, c.x - (s.x + s.w), s.y, s.y + s.h, c.y, c.y + c.h), + Direction::Left => (c.x + c.w <= s.x, s.x - (c.x + c.w), s.y, s.y + s.h, c.y, c.y + c.h), + Direction::Down => (c.y >= s.y + s.h, c.y - (s.y + s.h), s.x, s.x + s.w, c.x, c.x + c.w), + Direction::Up => (c.y + c.h <= s.y, s.y - (c.y + c.h), s.x, s.x + s.w, c.x, c.x + c.w), + }; + if !ok { + continue; + } + let ovl = hi.min(chi) - lo.max(clo); + let (tier, ovl) = if ovl > 0 { (0, ovl) } else { (1, 0) }; + // Distance (doubled, to stay integral) from the selected centre line + // to the candidate's span; 0 if the span contains it. + let centre = lo + hi; + let off = if centre < 2 * clo { + 2 * clo - centre + } else if centre > 2 * chi { + centre - 2 * chi + } else { + 0 + }; + let sc = (tier, gap, -ovl, off, c.y, c.x, b.id); + if best.is_none_or(|b| sc < b) { + best = Some(sc); + } + } + best.map(|b| b.6) +} + +#[cfg(test)] +mod tests { + use super::super::tests::{it, r, Rng}; + use super::super::*; + use super::Direction::*; + + fn blk(id: i64, x: i32, y: i32, w: i32, h: i32) -> Block { + Block { id, rect: r(x, y, w, h) } + } + + // Layout: + // + // ┌───────┬───────┐ + // │ A │ B │ + // ├───┬───┴───────┤ + // │ C │ D │ + // └───┴───────────┘ + #[test] + fn neighbour_basic() { + let a = blk(1, 0, 0, 8, 4); + let b = blk(2, 8, 0, 8, 4); + let c = blk(3, 0, 4, 4, 4); + let d = blk(4, 4, 4, 12, 4); + let bs = [d, c, b, a]; // order must not matter + let cases = [ + (1, Right, Some(2)), (2, Left, Some(1)), (1, Down, Some(3)), (2, Down, Some(4)), + (3, Right, Some(4)), (4, Left, Some(3)), (3, Up, Some(1)), (4, Up, Some(2)), + (1, Left, None), (1, Up, None), (4, Right, None), (4, Down, None), + ]; + for (from, dir, want) in cases { + assert_eq!(neighbour(&bs, from, dir), want, "from {from} dir {dir:?}"); + } + } + + #[test] + fn neighbour_prefers_adjacent_over_array_order() { + // S is on the left; R is directly right; F is far right and lower but + // appears first in the slice. + let s = blk(1, 0, 0, 10, 5); + let rr = blk(2, 10, 0, 10, 5); + let f = blk(3, 10, 20, 30, 5); + assert_eq!(neighbour(&[f, s, rr], 1, Right), Some(2)); + // Tall block on the left with three stacked blocks to the right: pick + // the one at the centre line. + let t = blk(10, 0, 0, 10, 9); + let r1 = blk(11, 10, 0, 10, 3); + let r2 = blk(12, 10, 3, 10, 3); + let r3 = blk(13, 10, 6, 10, 3); + assert_eq!(neighbour(&[r3, r1, r2, t], 10, Right), Some(12), "centre preference"); + assert_eq!(neighbour(&[r3, r1, r2, t], 13, Left), Some(10), "back left"); + } + + #[test] + fn neighbour_random_layouts_always_adjacent() { + let mut rng = Rng::new(7); + for iter in 0..200 { + let n = 2 + rng.intn(30) as i64; + let items: Vec = (0..n).map(|i| it(i, (1 + rng.intn(1000)) as f64)).collect(); + let bl = squarify(&items, r(0, 0, 120, 40)); + for s in &bl { + for d in [Left, Right, Up, Down] { + let Some(id) = neighbour(&bl, s.id, d) else { continue }; + let c = bl.iter().find(|b| b.id == id).unwrap().rect; + let s = s.rect; + // In a partition, if any block touches the edge in direction d + // with overlapping span, the chosen one must touch too. + let touch = match d { + Right => c.x == s.x + s.w, + Left => c.x + c.w == s.x, + Down => c.y == s.y + s.h, + Up => c.y + c.h == s.y, + }; + assert!(touch, "iter {iter}: from {s:?} dir {d:?} chose non-adjacent {c:?}"); + } + } + } + } +} diff --git a/rust/src/tui/actions.rs b/rust/src/tui/actions.rs new file mode 100644 index 0000000..3ca119e --- /dev/null +++ b/rust/src/tui/actions.rs @@ -0,0 +1,268 @@ +//! Mutating and long-running actions: trash/delete and duplicate search. + +use super::canvas::{Canvas, Style, BOLD}; +use super::modals::{new_info_modal, ConfirmModal}; +use super::model::{Cmd, Model, Msg}; +use super::san; +use super::table::{col, Table, TableRow}; +use crate::cancel::Cancel; +use crate::duplicate::{self, Finder}; +use crate::inventory::{Delta, NodeId, Tree, FLAG_DELETED, NO_NODE}; +use crate::platform::{self, Expect}; +use crate::textutil; +use crate::treemap::Rect; +use std::sync::{Arc, RwLock}; + +// ---- Trash --------------------------------------------------------------- + +impl Model { + /// Asks for confirmation before moving the selection to the trash. On a + /// headless system (no desktop, so nobody empties the trash and the space + /// is never freed) it deletes permanently instead, after a confirmation + /// that says so. + pub fn request_trash(&mut self, tr: &Tree) -> Cmd { + if self.opts.read_only { + return self.warn("Read-only mode: modifying actions are disabled"); + } + if self.snapshot { + return self.warn("Snapshot view: nothing to delete on the live filesystem"); + } + if self.scanning { + return self.warn("Wait for the scan to finish before deleting items"); + } + let id = self.selected_node(tr); + if id == NO_NODE || id == 0 { + return Cmd::None; + } + let n = tr.node(id); + let path = tr.path(id); + let want = Expect { uid: n.uid, mode: n.mode }; + let what = if n.is_dir() { format!("directory with {} files", textutil::count(n.files as i64)) } else { "file".into() }; + let mut body = vec![san(&tr.path_bytes(id)), String::new(), format!("{what} · {}", textutil::size(n.total(self.size_mode))), String::new()]; + if platform::headless() { + body.push("This is a headless system (no X or Wayland display), so there is no trash:".into()); + body.push("it will be permanently deleted and cannot be recovered.".into()); + self.open_modal(Box::new(ConfirmModal { + ttl: "Delete permanently? Are you sure?".into(), + body, + yes: "delete permanently".into(), + on_yes: Some(Box::new(move |m: &mut Model| { + let t = m.tree.clone(); + Cmd::run(move || { + let err = platform::delete(&path, want).err().map(|e| e.to_string()); + Msg::TrashDone { tree: t, id, err, done: "Deleted".into() } + }) + })), + })); + return Cmd::None; + } + body.push(format!("It will be moved to the {}, not permanently deleted.", platform::TRASH_NAME)); + self.open_modal(Box::new(ConfirmModal { + ttl: "Move to trash?".into(), + body, + yes: format!("move to {}", platform::TRASH_NAME), + on_yes: Some(Box::new(move |m: &mut Model| { + let t = m.tree.clone(); + Cmd::run(move || { + let err = platform::trash(&path, want).err().map(|e| e.to_string()); + Msg::TrashDone { tree: t, id, err, done: format!("Moved to {}", platform::TRASH_NAME) } + }) + })), + })); + Cmd::None + } + + pub fn trash_done(&mut self, tree: Arc>, id: NodeId, err: Option, done: String) -> Cmd { + if let Some(e) = err { + return self.warn(format!("Delete failed: {}", textutil::sanitize(&e))); + } + if !Arc::ptr_eq(&tree, &self.tree) || self.scanning { + // A rescan replaced the inventory; the new scan reflects the change. + return self.info(done); + } + // The scan has finished, so the UI thread is the tree's only writer. + let name = { + let mut t = self.tree.write().unwrap(); + let n = t.node(id).clone(); + let mut d = Delta { size: -n.tot_size, alloc: -n.tot_alloc, files: -(n.files as i64), dirs: -(n.dirs as i64), ..Default::default() }; + if n.is_dir() { + d.dirs -= 1; + if let Some(cs) = t.cat_sizes(id) { + for (c, v) in d.cat.iter_mut().zip(cs) { + *c = -v; + } + } + } else { + d.files -= 1; + d.cat[n.cat as usize] = -n.tot_alloc; + } + t.propagate(n.parent, &d); + let nm = t.node_mut(id); + nm.flags |= FLAG_DELETED; + (nm.tot_size, nm.tot_alloc) = (0, 0); + n.name + }; + self.invalidate(); + self.info(format!("{done}: {}", san(&name))) + } + + // ---- Duplicates ---------------------------------------------------------- + + pub fn start_duplicates(&mut self) -> Cmd { + if self.snapshot { + return self.warn("Duplicate detection needs the live filesystem"); + } + if self.scanning { + return self.warn("Wait for the scan to finish first"); + } + if self.dups.is_some() { + self.ensure_dup_view(); + return Cmd::None; + } + self.open_modal(Box::new(ConfirmModal { + ttl: "Find duplicate files?".into(), + body: vec![ + "Files are grouped by size, then sampled, then fully hashed with SHA-256.".into(), + "Only identical hashes are reported as duplicates.".into(), + String::new(), + "This reads file contents (it may take a while and can update access times).".into(), + ], + yes: "start".into(), + on_yes: Some(Box::new(|m: &mut Model| { + let cancel = Cancel::new(); + let f = Arc::new(Finder::default()); + m.dups = Some(DupState { running: true, finder: f.clone(), groups: Vec::new(), err: None, cancel: cancel.clone() }); + m.ensure_dup_view(); + let t = m.tree.clone(); + let tick = m.start_ticking(); + Cmd::batch(vec![ + tick, + Cmd::run(move || { + let g = f.find(&cancel, &t, duplicate::Options { min_size: 1, workers: 4 }); + Msg::DupDone { finder: f, groups: g } + }), + ]) + })), + })); + Cmd::None + } + + pub fn ensure_dup_view(&mut self) { + if let Some(i) = self.views.iter().position(|v| matches!(v, super::views::View::Dups(_))) { + self.set_view(i); + return; + } + self.views.push(super::views::View::Dups(DupView::default())); + self.set_view(self.views.len() - 1); + } + + pub fn dup_done(&mut self, finder: Arc, groups: Option>) -> Cmd { + let Some(d) = self.dups.as_mut().filter(|d| Arc::ptr_eq(&d.finder, &finder)) else { return Cmd::None }; + d.running = false; + d.err = groups.is_none().then(|| "cancelled".to_string()); + d.groups = groups.unwrap_or_default(); + let (n, wasted) = (d.groups.len(), d.groups.iter().map(|g| g.wasted()).sum::()); + self.dirty = true; + self.info(format!("Duplicates: {n} verified groups, {} reclaimable", textutil::size(wasted))) + } +} + +pub(crate) struct DupState { + pub running: bool, + pub finder: Arc, + pub groups: Vec, + pub err: Option, + pub cancel: Cancel, +} + +#[derive(Default)] +pub(crate) struct DupView { + pub tb: Table, + pub rows: Vec<(usize, NodeId)>, // (group, file); NO_NODE for a group header +} + +impl DupView { + pub fn paint(&mut self, m: &mut Model, tr: &Tree, cv: &mut Canvas, r: Rect) { + let t = m.theme.clone(); + let Some(d) = &m.dups else { + cv.text_centered(r.x, r.y + r.h / 2, r.w, "Press D to search for duplicates", t.muted()); + return; + }; + if d.running { + let p = d.finder.progress(); + cv.fill(r, " ", t.base()); + let y = r.y + r.h / 2 - 2; + cv.text_centered(r.x, y, r.w, &format!("Finding duplicates — {}…", p.stage), Style::new(t.accent, t.bg).with(BOLD)); + cv.text_centered(r.x, y + 2, r.w, + &format!("{} candidate files in {} size groups", textutil::count(p.candidates), textutil::count(p.candidate_groups)), t.base()); + if p.bytes_to_hash > 0 { + let frac = p.bytes_hashed as f64 / p.bytes_to_hash as f64; + let bw = 60.min(r.w - 10); + m.paint_bar(cv, r.x + (r.w - bw) / 2, y + 4, bw, frac, t.accent, false); + cv.text_centered(r.x, y + 5, r.w, + &format!("{} of {} hashed · {} skipped", textutil::size(p.bytes_hashed), textutil::size(p.bytes_to_hash), textutil::count(p.skipped)), + t.muted()); + } + return; + } + self.rows.clear(); + let mut rows = Vec::new(); + for (gi, g) in d.groups.iter().enumerate() { + self.rows.push((gi, NO_NODE)); + let hash: String = g.hash[..8].iter().map(|b| format!("{b:02x}")).collect(); + rows.push(TableRow { + cells: vec![textutil::size(g.wasted()), format!("{} × {}", g.files.len(), textutil::size(g.size)), + format!("verified identical · sha256 {hash}…")], + styles: vec![Some(Style::new(t.warn, t.bg).with(BOLD)), Some(Style::new(t.fg, t.bg).with(BOLD)), Some(Style::new(t.ok, t.bg))], + bar: -1.0, + ..Default::default() + }); + for &id in &g.files { + self.rows.push((gi, id)); + rows.push(TableRow { cells: vec![String::new(), String::new(), format!(" {}", san(&tr.path_bytes(id)))], bar: -1.0, ..Default::default() }); + } + } + let mut title = format!("Duplicates — {} groups (Enter details · Space go to)", d.groups.len()); + if let Some(e) = &d.err { + title += &format!(" — stopped: {}", textutil::sanitize(e)); + } + let cols = [col("WASTED", 10, true, false), col("COPIES", 16, false, false), col("", 0, false, false)]; + self.tb.paint(m, cv, r, &cols, &rows, &title); + } + + pub fn key(&mut self, m: &mut Model, tr: &Tree, k: &str) -> (bool, Cmd) { + if self.tb.key(k, self.rows.len()) { + return (true, Cmd::None); + } + if self.tb.at() < self.rows.len() { + let (g, mut id) = self.rows[self.tb.at()]; + if id == NO_NODE { + if let Some(d) = m.dups.as_ref().filter(|d| g < d.groups.len()) { + id = d.groups[g].files[0]; + } + } + if id != NO_NODE { + match k { + "enter" => { + let md = new_info_modal(m, tr, id); + m.open_modal(md); + return (true, Cmd::None); + } + "space" => { + m.jump_to(tr, id); + return (true, Cmd::None); + } + "c" | "o" | "d" => m.sel = id as i64, + _ => {} + } + } + } + if k == "esc" { + if let Some(d) = m.dups.as_ref().filter(|d| d.running) { + d.cancel.cancel(); + return (true, Cmd::None); + } + } + (false, Cmd::None) + } +} diff --git a/rust/src/tui/bench.rs b/rust/src/tui/bench.rs new file mode 100644 index 0000000..35a0d3f --- /dev/null +++ b/rust/src/tui/bench.rs @@ -0,0 +1,126 @@ +//! Benchmark hooks: the model driven with no terminal, rendering in +//! true colour, as the Go benchmarks do. + +use super::model::{Key, Model, Msg, Options, Profile}; +use crate::cancel::Cancel; +use crate::config::Settings; +use crate::inventory::{Category, Kind, NodeId, SizeMode, Tree}; +use std::sync::{Arc, RwLock}; + +/// A model with no terminal. Every method that renders returns the frame +/// length so the work cannot be optimised away. +pub struct BenchModel { + m: Model, +} + +pub(crate) fn opts_for(tree: Arc>) -> Options { + Options { + read_only: false, + theme: String::new(), + size_mode: SizeMode::Allocated, + mouse: false, + color: String::new(), + save: String::new(), + settings: Settings::default(), + start: Arc::new(move |_| Ok((None, tree.clone()))), + diff: None, + } +} + +impl BenchModel { + pub fn new(tree: Arc>, w: u16, h: u16) -> BenchModel { + let mut m = Model::new(opts_for(tree.clone()), None, tree, Cancel::new()); + m.profile = Profile::TrueColor; + m.update(Msg::Size(w as i32, h as i32)); + BenchModel { m } + } + + /// Invalidates the data caches and renders: sorting children, LOD + /// selection, layout and paint. + pub fn cold_frame(&mut self) -> usize { + self.m.invalidate(); + self.m.view().len() + } + + /// Resizes and renders (child lists stay cached). + pub fn resize(&mut self, w: u16, h: u16) -> usize { + self.m.update(Msg::Size(w as i32, h as i32)); + self.m.view().len() + } + + /// Sends a key ("right", "down", "left", "up", …) and renders. + pub fn key(&mut self, k: &str) -> usize { + self.m.update(Msg::Key(Key::from_name(k))); + self.m.view().len() + } + + /// Makes the root's child with this name the zoom root. + pub fn zoom_to_child_named(&mut self, name: &[u8]) { + let t = self.m.tree.clone(); + let g = t.read().unwrap(); + if let Some((id, _)) = g.children(0).find(|(_, n)| &*n.name == name) { + self.m.zoom = id; + } + } + + /// Sets the query and runs the filter synchronously. + pub fn apply_filter(&mut self, q: &str) { + self.m.search.ed.set(q); + let _ = self.m.query_changed(); + for msg in self.m.apply_filter().run_sync() { + self.m.update(msg); + } + } +} + +/// A small deterministic PRNG (splitmix64). +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 = self.0.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.0; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + /// Uniform in (0, 1]. + fn unit(&mut self) -> f64 { + ((self.next() >> 11) + 1) as f64 / (1u64 << 53) as f64 + } + fn exp(&mut self) -> f64 { + -self.unit().ln() + } + fn intn(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// ~1M nodes in memory: 200 top-level dirs with 40 subdirectories of 100 +/// files each, plus one flat directory with 200k files. +pub fn big_tree() -> Tree { + let mut r = Rng(1); + let mut t = Tree::new(b"/big", Kind::Dir); + let mut file = |t: &mut Tree, p: NodeId, i: usize| { + let id = t.add(p, format!("f{i}.dat").as_bytes(), Kind::File); + let n = t.node_mut(id); + n.size = (r.exp() * 1e6) as i64; + n.alloc = n.size; + n.cat = Category::from_u8(r.intn(crate::inventory::NUM_CATEGORIES as u64) as u8); + }; + let flat = t.add(0, b"flat", Kind::Dir); + for i in 0..200_000 { + file(&mut t, flat, i); + } + for d in 0..200 { + let top = t.add(0, format!("dir{d}").as_bytes(), Kind::Dir); + for s in 0..40 { + let sub = t.add(top, format!("sub{s}").as_bytes(), Kind::Dir); + for f in 0..100 { + file(&mut t, sub, f); + } + } + } + t.recompute(); + t +} diff --git a/rust/src/tui/canvas.rs b/rust/src/tui/canvas.rs new file mode 100644 index 0000000..a8a04b0 --- /dev/null +++ b/rust/src/tui/canvas.rs @@ -0,0 +1,360 @@ +//! An in-memory cell grid that every view paints into; serialised once per +//! frame with minimal SGR changes. + +use super::color256::{quant16, quant256}; +use crate::textutil; +use crate::treemap::Rect; +use std::fmt::Write; + +/// 0 for the terminal default, otherwise 0x01RRGGBB. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Default, Debug)] +pub(crate) struct Color(pub u32); + +/// Builds a colour. +pub(crate) const fn rgb(r: u8, g: u8, b: u8) -> Color { + Color(1 << 24 | (r as u32) << 16 | (g as u32) << 8 | b as u32) +} + +impl Color { + /// Parses "#rrggbb". + pub fn hex(s: &str) -> Option { + let s = s.strip_prefix('#').unwrap_or(s); + if s.len() != 6 || !s.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + u32::from_str_radix(s, 16).ok().map(|v| Color(1 << 24 | v)) + } + + pub fn is_default(self) -> bool { + self.0 == 0 + } + + pub fn r(self) -> u8 { + (self.0 >> 16) as u8 + } + pub fn g(self) -> u8 { + (self.0 >> 8) as u8 + } + pub fn b(self) -> u8 { + self.0 as u8 + } + + pub fn comps(self) -> (f64, f64, f64) { + (self.r() as f64 / 255.0, self.g() as f64 / 255.0, self.b() as f64 / 255.0) + } + + /// Blends c towards o by t (0..1). Default colours are returned as-is. + pub fn mix(self, o: Color, t: f64) -> Color { + if self.0 == 0 || o.0 == 0 { + return self; + } + let (r1, g1, b1) = self.comps(); + let (r2, g2, b2) = o.comps(); + let f = |a: f64, b: f64| ((a + (b - a) * t) * 255.0) as u8; + rgb(f(r1, r2), f(g1, g2), f(b1, b2)) + } + + /// Perceived brightness 0..1. + pub fn luma(self) -> f64 { + let (r, g, b) = self.comps(); + 0.299 * r + 0.587 * g + 0.114 * b + } +} + +/// Text attribute bit set. +pub(crate) type Attr = u8; +pub(crate) const BOLD: Attr = 1; +pub(crate) const FAINT: Attr = 2; +pub(crate) const ITALIC: Attr = 4; +pub(crate) const UNDERLINE: Attr = 8; +pub(crate) const REVERSE: Attr = 16; + +/// A cell style. +#[derive(Clone, Copy, PartialEq, Eq, Default, Debug)] +pub(crate) struct Style { + pub fg: Color, + pub bg: Color, + pub attr: Attr, +} + +impl Style { + pub const fn new(fg: Color, bg: Color) -> Style { + Style { fg, bg, attr: 0 } + } + pub const fn with(self, a: Attr) -> Style { + Style { attr: self.attr | a, ..self } + } +} + +const INLINE: usize = 15; +const HEAP: u8 = 255; + +/// One grid cell. The cluster is stored inline (n bytes) or, if longer, +/// in the canvas side table (n == HEAP, index in the first four bytes). +/// n == 0 marks the continuation of a wide cluster. +#[derive(Clone, Copy)] +struct Cell { + g: [u8; INLINE], + n: u8, + w: u8, + st: Style, +} + +impl Cell { + fn space(st: Style) -> Cell { + let mut g = [0; INLINE]; + g[0] = b' '; + Cell { g, n: 1, w: 1, st } + } + fn is_cont(&self) -> bool { + self.n == 0 + } + fn set_space(&mut self) { + self.g[0] = b' '; + self.n = 1; + self.w = 1; + } +} + +/// Box glyph sets. +#[derive(Clone, Copy)] +pub(crate) struct BoxChars { + pub h: &'static str, + pub v: &'static str, + pub tl: &'static str, + pub tr: &'static str, + pub bl: &'static str, + pub br: &'static str, +} + +pub(crate) const BOX_LIGHT: BoxChars = BoxChars { h: "─", v: "│", tl: "┌", tr: "┐", bl: "└", br: "┘" }; +pub(crate) const BOX_ROUND: BoxChars = BoxChars { h: "─", v: "│", tl: "╭", tr: "╮", bl: "╰", br: "╯" }; +pub(crate) const BOX_DOUBLE: BoxChars = BoxChars { h: "═", v: "║", tl: "╔", tr: "╗", bl: "╚", br: "╝" }; +pub(crate) const BOX_ASCII: BoxChars = BoxChars { h: "-", v: "|", tl: "+", tr: "+", bl: "+", br: "+" }; +pub(crate) const BOX_ASCII_SEL: BoxChars = BoxChars { h: "=", v: "#", tl: "#", tr: "#", bl: "#", br: "#" }; + +/// Selects how colours are serialised. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(crate) enum Depth { + True, + C256, + C16, + None, // attributes only +} + +/// An in-memory cell grid. +pub(crate) struct Canvas { + pub w: i32, + pub h: i32, + c: Vec, + big: Vec, +} + +impl Canvas { + /// Creates a canvas filled with spaces in style st. + pub fn new(w: i32, h: i32, st: Style) -> Canvas { + let (w, h) = (w.max(0), h.max(0)); + Canvas { w, h, c: vec![Cell::space(st); (w * h) as usize], big: Vec::new() } + } + + fn inside(&self, x: i32, y: i32) -> bool { + x >= 0 && y >= 0 && x < self.w && y < self.h + } + + fn cell_str<'a>(&'a self, c: &'a Cell) -> &'a str { + if c.n == HEAP { + let i = u32::from_le_bytes([c.g[0], c.g[1], c.g[2], c.g[3]]) as usize; + return &self.big[i]; + } + // Only whole UTF-8 clusters are ever stored. + std::str::from_utf8(&c.g[..c.n as usize]).unwrap_or("?") + } + + fn make(&mut self, s: &str, w: i32, st: Style) -> Cell { + let mut g = [0; INLINE]; + let n = if s.len() <= INLINE { + g[..s.len()].copy_from_slice(s.as_bytes()); + s.len() as u8 + } else { + g[..4].copy_from_slice(&(self.big.len() as u32).to_le_bytes()); + self.big.push(s.to_string()); + HEAP + }; + Cell { g, n, w: w as u8, st } + } + + /// Writes one cluster of width w at (x, y), repairing any wide cluster + /// it overlaps so the grid never holds half a character. + fn put(&mut self, x: i32, y: i32, s: &str, w: i32, st: Style) { + if !self.inside(x, y) || x + w > self.w { + return; + } + let row = (y * self.w) as usize; + let (xu, wu, cw) = (x as usize, w as usize, self.w as usize); + for i in xu..xu + wu { + if self.c[row + i].is_cont() && i > 0 && i == xu { + // overwriting the tail of a wide cluster + self.c[row + i - 1].set_space(); + } + let c = self.c[row + i]; + if !c.is_cont() && c.w > 1 && i + 1 < cw && self.c[row + i + 1].is_cont() { + // overwriting its head + self.c[row + i + 1].set_space(); + } + } + self.c[row + xu] = self.make(s, w, st); + for i in 1..wu { + self.c[row + xu + i] = Cell { g: [0; INLINE], n: 0, w: 0, st }; + } + } + + /// Draws s at (x, y), clipped to max_w cells and the canvas. Control + /// characters are replaced as a second line of defence (callers sanitise + /// untrusted strings first). It returns the number of cells used. + pub fn text(&mut self, x: i32, y: i32, s: &str, max_w: i32, st: Style) -> i32 { + if y < 0 || y >= self.h { + return 0; + } + let limit = (x + max_w).min(self.w); + let mut cx = x; + textutil::clusters(s, |c, w| { + let (mut c, mut w) = (c, w as i32); + let b = c.as_bytes(); + if b[0] < 0x20 || b[0] == 0x7f || (b.len() > 1 && b[0] == 0xc2 && b[1] < 0xa0) { + (c, w) = ("?", 1); + } + if cx + w > limit { + // A wide cluster that does not fit leaves a blank cell. + while cx < limit { + if cx >= 0 { + self.put(cx, y, " ", 1, st); + } + cx += 1; + } + return false; + } + if cx >= 0 { + self.put(cx, y, c, w, st); + } + cx += w; + true + }); + cx - x + } + + /// Draws s centred within [x, x+w). + pub fn text_centered(&mut self, x: i32, y: i32, w: i32, s: &str, st: Style) { + let s = textutil::truncate(s, w.max(0) as usize); + self.text(x + (w - textutil::width(&s) as i32) / 2, y, &s, w, st); + } + + /// Draws s right-aligned ending at x+w. + pub fn text_right(&mut self, x: i32, y: i32, w: i32, s: &str, st: Style) { + let s = textutil::truncate(s, w.max(0) as usize); + self.text(x + w - textutil::width(&s) as i32, y, &s, w, st); + } + + /// Fills r with the single-cell cluster ch. + pub fn fill(&mut self, r: Rect, ch: &str, st: Style) { + for y in r.y.max(0)..(r.y + r.h).min(self.h) { + for x in r.x.max(0)..(r.x + r.w).min(self.w) { + self.put(x, y, ch, 1, st); + } + } + } + + /// Applies f to the style of every cell in r. + pub fn restyle(&mut self, r: Rect, f: impl Fn(Style) -> Style) { + for y in r.y.max(0)..(r.y + r.h).min(self.h) { + for x in r.x.max(0)..(r.x + r.w).min(self.w) { + let c = &mut self.c[(y * self.w + x) as usize]; + c.st = f(c.st); + } + } + } + + /// Draws a border around r (which must be at least 2×2). + pub fn draw_box(&mut self, r: Rect, b: BoxChars, st: Style) { + if r.w < 2 || r.h < 2 { + return; + } + let (x1, y1) = (r.x + r.w - 1, r.y + r.h - 1); + for x in r.x + 1..x1 { + self.put(x, r.y, b.h, 1, st); + self.put(x, y1, b.h, 1, st); + } + for y in r.y + 1..y1 { + self.put(r.x, y, b.v, 1, st); + self.put(x1, y, b.v, 1, st); + } + self.put(r.x, r.y, b.tl, 1, st); + self.put(x1, r.y, b.tr, 1, st); + self.put(r.x, y1, b.bl, 1, st); + self.put(x1, y1, b.br, 1, st); + } + + /// Draws a horizontal rule. + pub fn hline(&mut self, x: i32, y: i32, w: i32, ch: &str, st: Style) { + for i in x..x + w { + self.put(i, y, ch, 1, st); + } + } + + /// Serialises the canvas as lines of text with SGR styling. For 256 and + /// 16 colours it quantizes with its own perceptual mapping (see + /// color256.rs); Depth::None keeps attributes only. + pub fn render(&self, depth: Depth) -> String { + let mut b = String::with_capacity((self.w * self.h * 4) as usize); + for y in 0..self.h { + if y > 0 { + b.push('\n'); + } + let mut cur = Style::default(); + let mut first = true; + for x in 0..self.w { + let c = &self.c[(y * self.w + x) as usize]; + if c.is_cont() { + continue; + } + if first || c.st != cur { + sgr(&mut b, c.st, depth); + cur = c.st; + first = false; + } + b.push_str(self.cell_str(c)); + } + b.push_str("\x1b[0m"); + } + b + } +} + +fn sgr(b: &mut String, st: Style, depth: Depth) { + b.push_str("\x1b[0"); + for (bit, code) in [(BOLD, ";1"), (FAINT, ";2"), (ITALIC, ";3"), (UNDERLINE, ";4"), (REVERSE, ";7")] { + if st.attr & bit != 0 { + b.push_str(code); + } + } + let col = |b: &mut String, c: Color, bg: bool| { + if c.is_default() { + return; + } + let _ = match depth { + Depth::None => Ok(()), + Depth::C256 => write!(b, ";{};5;{}", if bg { 48 } else { 38 }, quant256(c)), + Depth::C16 => { + let n = quant16(c) as u32; + let mut code = if n >= 8 { 90 + n - 8 } else { 30 + n }; + if bg { + code += 10; + } + write!(b, ";{code}") + } + Depth::True => write!(b, ";{};2;{};{};{}", if bg { 48 } else { 38 }, c.r(), c.g(), c.b()), + }; + }; + col(b, st.fg, false); + col(b, st.bg, true); + b.push('m'); +} diff --git a/rust/src/tui/color256.rs b/rust/src/tui/color256.rs new file mode 100644 index 0000000..8e5b056 --- /dev/null +++ b/rust/src/tui/color256.rs @@ -0,0 +1,92 @@ +//! The stock 256-colour conversion maps dark, slightly tinted colours to +//! saturated cube entries (olive, navy), which wrecks the subtle shading the +//! treemap relies on. quant256 picks the perceptually nearest entry (redmean +//! distance) among the cube and the grey ramp, never the palette-dependent +//! first 16 entries. + +use super::canvas::Color; +use std::cell::RefCell; +use std::collections::HashMap; + +const CUBE_LEVELS: [i32; 6] = [0, 95, 135, 175, 215, 255]; + +thread_local! { + static Q256: RefCell> = RefCell::new(HashMap::new()); + static Q16: RefCell> = RefCell::new(HashMap::new()); +} + +fn redmean(r1: i32, g1: i32, b1: i32, r2: i32, g2: i32, b2: i32) -> i32 { + let rm = (r1 + r2) / 2; + let (dr, dg, db) = (r1 - r2, g1 - g2, b1 - b2); + (((512 + rm) * dr * dr) >> 8) + 4 * dg * dg + (((767 - rm) * db * db) >> 8) +} + +pub(crate) fn quant256(c: Color) -> u8 { + if let Some(v) = Q256.with(|q| q.borrow().get(&c.0).copied()) { + return v; + } + let (r, g, b) = (c.r() as i32, c.g() as i32, c.b() as i32); + let (mut best, mut best_d) = (16, i32::MAX); + for i in 0..216 { + let (cr, cg, cb) = (CUBE_LEVELS[i / 36], CUBE_LEVELS[i / 6 % 6], CUBE_LEVELS[i % 6]); + let d = redmean(r, g, b, cr, cg, cb); + if d < best_d { + (best, best_d) = (16 + i, d); + } + } + for i in 0..24 { + let v = 8 + 10 * i as i32; + let d = redmean(r, g, b, v, v, v); + if d < best_d { + (best, best_d) = (232 + i, d); + } + } + Q256.with(|q| q.borrow_mut().insert(c.0, best as u8)); + best as u8 +} + +/// Maps a colour to a basic colour. Dark shades become black and +/// unsaturated colours grey/white, so tints and shadows stay quiet; +/// saturated colours keep their hue (bright variant when light), so category +/// colours survive. The stock conversion instead turns dark tints into +/// bright yellow/cyan. +pub(crate) fn quant16(c: Color) -> u8 { + if let Some(v) = Q16.with(|q| q.borrow().get(&c.0).copied()) { + return v; + } + let (rf, gf, bf) = c.comps(); + let (mx, mn) = (rf.max(gf).max(bf), rf.min(gf).min(bf)); + let luma = c.luma(); + let n = if luma < 0.2 { + 0 + } else if mx - mn < 0.12 { + // grey + if luma < 0.45 { + 8 + } else if luma < 0.8 { + 7 + } else { + 15 + } + } else { + // Hue in sixths: red, yellow, green, cyan, blue, magenta. + let mut h = if mx == rf { + ((gf - bf) / (mx - mn)) % 6.0 + } else if mx == gf { + (bf - rf) / (mx - mn) + 2.0 + } else { + (rf - gf) / (mx - mn) + 4.0 + }; + if h < 0.0 { + h += 6.0 + } + let ansi = [1u8, 3, 2, 6, 4, 5][h.round() as usize % 6]; + if luma > 0.5 { + ansi + 8 + } else { + ansi + } + }; + Q16.with(|q| q.borrow_mut().insert(c.0, n)); + n +} diff --git a/rust/src/tui/data.rs b/rust/src/tui/data.rs new file mode 100644 index 0000000..3b0f29b --- /dev/null +++ b/rust/src/tui/data.rs @@ -0,0 +1,181 @@ +//! Size lookups, the cached sorted child lists, selection upkeep and +//! colouring. + +use super::canvas::Color; +use super::model::{group_id, group_parent, is_group, Model, NO_SEL}; +use crate::inventory::{NodeId, Tree, FLAG_DELETED, NO_NODE}; +use crate::snapshot::Status; +use std::collections::HashSet; +use std::rc::Rc; + +/// A cached, size-sorted child list (zero-size entries removed). +#[derive(Default)] +pub(crate) struct Kids { + pub ids: Vec, + pub sizes: Vec, + pub total: i64, +} + +impl Model { + /// The size that drives layout: the filtered size when a filter is + /// active, otherwise the aggregate in the current size mode. + pub fn size_of(&self, tr: &Tree, id: NodeId) -> i64 { + if let Some(r) = &self.search.result { + return r.size(id); + } + let n = tr.node(id); + if n.has(FLAG_DELETED) { + return 0; + } + n.total(self.size_mode) + } + + /// The sorted children of id, cached until data changes. + pub fn kids_of(&mut self, tr: &Tree, id: NodeId) -> Rc { + if self.kids_ver != self.data_ver { + self.kids_cache.clear(); + self.kids_ver = self.data_ver; + } + if let Some(k) = self.kids_cache.get(&id) { + return k.clone(); + } + let mut v: Vec<(i64, NodeId)> = + tr.children(id).map(|(c, _)| (self.size_of(tr, c), c)).filter(|&(s, _)| s > 0).collect(); + v.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + let k = Rc::new(Kids { + total: v.iter().map(|e| e.0).sum(), + sizes: v.iter().map(|e| e.0).collect(), + ids: v.iter().map(|e| e.1).collect(), + }); + self.kids_cache.insert(id, k.clone()); + k + } + + /// The selected inventory node (NO_NODE for groups). + pub fn selected_node(&self, tr: &Tree) -> NodeId { + if self.sel == NO_SEL || is_group(self.sel) || self.sel >= tr.len() as i64 { + return NO_NODE; + } + self.sel as NodeId + } + + /// Keeps the selection meaningful after any change: if the selected node + /// is no longer a direct, visible child of the zoom root it falls back to + /// the visible ancestor, then the group containing it, then the largest + /// child. + pub fn ensure_selection(&mut self, tr: &Tree) { + let k = self.kids_of(tr, self.zoom); + if k.ids.is_empty() { + self.sel = NO_SEL; + return; + } + if !self.user_sel { + // Until the user picks something, follow the largest block (it + // changes while a scan is running). + self.sel = k.ids[0] as i64; + return; + } + if is_group(self.sel) { + if group_parent(self.sel) == self.zoom { + return; + } + self.sel = NO_SEL; + } + let id = self.selected_node(tr); + if id != NO_NODE { + // Walk up to the direct child of the zoom root. + let mut cur = id; + while cur != NO_NODE { + if tr.node(cur).parent == self.zoom { + if self.size_of(tr, cur) > 0 { + self.sel = cur as i64; + return; + } + break; + } + cur = tr.node(cur).parent; + } + } + self.sel = k.ids[0] as i64; + } + + /// Maps the selection onto the treemap blocks: a node collapsed into the + /// LOD group selects the group. + pub fn visible_sel(&self, blocks: &HashSet) -> i64 { + if blocks.contains(&self.sel) { + return self.sel; + } + if !is_group(self.sel) && blocks.contains(&group_id(self.zoom)) { + return group_id(self.zoom); + } + self.sel + } + + /// Makes dir the treemap root. + pub fn zoom_to(&mut self, tr: &Tree, dir: NodeId) { + if dir == self.zoom { + return; + } + let prev = self.zoom; + self.zoom = dir; + // Coming back out, keep the directory we left selected. + self.sel = if tr.is_ancestor(dir, prev) { prev as i64 } else { NO_SEL }; + self.user_sel = self.sel != NO_SEL; + self.zoomed = true; + self.dirty = true; + } + + pub fn set_view(&mut self, i: usize) { + if i < self.views.len() { + self.view = i; + self.dirty = true; + } + } + + /// The base colour for a block. + pub fn color_of(&self, tr: &Tree, id: i64) -> Color { + let t = &self.theme; + if is_group(id) { + return t.group; + } + let nid = id as NodeId; + let n = tr.node(nid); + if let Some(d) = &self.opts.diff { + let total = n.total(self.size_mode); + return match d.status_of(tr, nid, self.size_mode) { + Status::Added => t.diff_new, + Status::Grew => { + let old = d.old_size(nid, self.size_mode).unwrap_or(0); + t.diff_same.mix(t.diff_grow, ratio(total - old, old)) + } + Status::Shrank => { + let old = d.old_size(nid, self.size_mode).unwrap_or(0); + t.diff_same.mix(t.diff_shrink, ratio(old - total, old)) + } + _ => t.diff_same, + }; + } + if !n.is_dir() { + if let Some(c) = t.ext.as_ref().and_then(|e| e.get(tr.ext_name(n))) { + return *c; + } + return t.cat[n.cat as usize]; + } + let (mut best, mut best_v) = (None, 0); + for (i, &v) in tr.cat_sizes(nid).into_iter().flatten().enumerate() { + if v > best_v { + (best, best_v) = (Some(i), v); + } + } + best.map_or(t.empty_dir, |b| t.cat[b]) + } +} + +/// Maps a relative change to 0.35..1 for colour intensity. +fn ratio(delta: i64, base: i64) -> f64 { + if base <= 0 { + return 1.0; + } + let r = delta as f64 / base as f64; + (0.35 + r * 0.65).min(1.0) +} diff --git a/rust/src/tui/infoview.rs b/rust/src/tui/infoview.rs new file mode 100644 index 0000000..6a5a858 --- /dev/null +++ b/rust/src/tui/infoview.rs @@ -0,0 +1,160 @@ +//! The scan information view: statistics and the error list. + +use super::canvas::{Canvas, Style, BOLD}; +use super::model::{Cmd, Model}; +use super::table::{col, Table, TableRow}; +use super::san; +use crate::inventory::{ErrKind, Tree, FLAG_ALLOC_UNKNOWN}; +use crate::textutil; +use crate::treemap::Rect; + +#[derive(Default)] +pub(crate) struct InfoView { + pub tb: Table, +} + +impl InfoView { + pub fn focus_errors(&mut self) { + self.tb.cursor = 0; + } + + pub fn paint(&mut self, m: &mut Model, tr: &Tree, cv: &mut Canvas, r: Rect) { + let t = m.theme.clone(); + cv.fill(r, " ", t.base()); + let st = &tr.stats; + let root = tr.node(0); + let lw = if r.w < 100 { r.w - 2 } else { 52.min(r.w / 2) }; + let mut y = r.y + 1; + let x = r.x + 2; + let label = Style::new(t.muted, t.bg); + let value = Style::new(t.fg, t.bg).with(BOLD); + let line = |cv: &mut Canvas, y: &mut i32, k: &str, val: &str, vs: Style| { + if *y >= r.y + r.h { + return; + } + cv.text(x, *y, k, 20, label); + cv.text(x + 20, *y, val, lw - 20, vs); + *y += 1; + }; + let head = |cv: &mut Canvas, y: &mut i32, s: &str| { + if *y >= r.y + r.h - 1 { + return; + } + *y += 1; + cv.text(x, *y, s, lw, Style::new(t.accent, t.bg).with(BOLD)); + *y += 1; + }; + let (state, mut state_st) = ("Complete", Style::new(t.ok, t.bg).with(BOLD)); + let state = if m.scanning { + state_st.fg = t.accent; + "Scanning…" + } else if st.cancelled { + state_st.fg = t.warn; + "Cancelled — totals incomplete" + } else if st.incomplete() { + state_st.fg = t.warn; + "Complete, with unreadable areas" + } else { + state + }; + cv.text(x, y, "SCAN INFORMATION", lw, Style::new(t.accent, t.bg).with(BOLD)); + y += 1; + line(cv, &mut y, "Root", &san(&st.root), value); + line(cv, &mut y, "State", state, state_st); + if !st.from_snapshot.is_empty() { + line(cv, &mut y, "Snapshot", &textutil::sanitize(&st.from_snapshot), value); + } + let files = st.files + st.symlinks + st.others; + line(cv, &mut y, "Items scanned", &textutil::count(files + st.dirs), value); + line(cv, &mut y, " Files", &textutil::count(st.files), value); + line(cv, &mut y, " Directories", &textutil::count(st.dirs), value); + line(cv, &mut y, " Symlinks", &textutil::count(st.symlinks), value); + line(cv, &mut y, " Special", &textutil::count(st.others), value); + line(cv, &mut y, "Logical size", &format!("{} ({} bytes)", textutil::size(root.tot_size), textutil::count(root.tot_size)), value); + line(cv, &mut y, "Allocated size", &format!("{} ({} bytes)", textutil::size(root.tot_alloc), textutil::count(root.tot_alloc)), value); + let el = st.elapsed(); + if st.from_snapshot.is_empty() { + line(cv, &mut y, "Elapsed", &textutil::duration(el), value); + let rate = (files + st.dirs) as f64 / el.as_secs_f64().max(0.001); + line(cv, &mut y, "Rate", &(textutil::count(rate as i64) + " items/s"), value); + let cpus = std::thread::available_parallelism().map_or(1, |n| n.get()); + line(cv, &mut y, "Workers", &format!("{} ({} CPUs)", st.workers, cpus), value); + } + + head(cv, &mut y, "ERRORS AND OMISSIONS"); + let mut any_issue = false; + for (k, &c) in st.err_counts.iter().enumerate() { + if c > 0 { + line(cv, &mut y, ErrKind::from_u8(k as u8).name(), &textutil::count(c), Style::new(t.err, t.bg).with(BOLD)); + any_issue = true; + } + } + for (k, v) in [("Unscanned dirs", st.unscanned), ("Excluded", st.excluded), ("Other FS skipped", st.skipped_mounts), + ("Virtual FS skipped", st.virtual_skipped), ("Loops skipped", st.loops_skipped), ("Broken symlinks", st.broken_links), + ("Hard-link dups", st.hardlink_dups)] + { + if v > 0 { + line(cv, &mut y, k, &textutil::count(v), Style::new(t.warn, t.bg).with(BOLD)); + any_issue = true; + } + } + if !any_issue { + line(cv, &mut y, "None", "every entry was read", Style::new(t.ok, t.bg)); + } + head(cv, &mut y, "OPTIONS"); + let excl = if st.excludes.is_empty() { "none".to_string() } else { textutil::sanitize(&st.excludes.join(", ")) }; + line(cv, &mut y, "Exclusions", &excl, value); + line(cv, &mut y, "One filesystem", &st.one_file_system.to_string(), value); + line(cv, &mut y, "Follow symlinks", &textutil::sanitize(&st.follow), value); + line(cv, &mut y, "Size mode", m.size_mode.name(), value); + line(cv, &mut y, "Units", if crate::textutil::si() { "SI (1000)" } else { "IEC (1024)" }, value); + if root.has(FLAG_ALLOC_UNKNOWN) { + line(cv, &mut y, "Note", "allocated size unavailable on this platform", Style::new(t.warn, t.bg)); + } + + // Error list on the right (or below in narrow terminals). + let er = if r.w < 100 { + Rect { x: r.x, y: y + 1, w: r.w, h: r.y + r.h - y - 1 } + } else { + Rect { x: r.x + lw + 4, y: r.y, w: r.w - lw - 4, h: r.h } + }; + if er.w < 30 || er.h < 4 { + return; + } + let rows: Vec = st + .errors + .iter() + .map(|e| { + let mut p = tr.path_bytes(e.node); + if !e.name.is_empty() { + p.push(b'/'); + p.extend_from_slice(&e.name); + } + TableRow { + cells: vec![e.kind.name().into(), san(&p), textutil::sanitize(&e.msg)], + styles: vec![Some(Style::new(t.err, t.bg)), None, Some(Style::new(t.muted, t.bg))], + bar: -1.0, + ..Default::default() + } + }) + .collect(); + let mut title = format!("ERROR LIST ({})", textutil::count(st.total_errors())); + if (st.errors.len() as i64) < st.total_errors() { + title += &format!(" — first {} shown", textutil::count(st.errors.len() as i64)); + } + let cols = [col("KIND", 18, false, false), col("PATH", 0, false, true), col("DETAIL", 22, false, false)]; + self.tb.paint(m, cv, er, &cols, &rows, &title); + } + + pub fn key(&mut self, m: &mut Model, tr: &Tree, k: &str) -> (bool, Cmd) { + let errs = &tr.stats.errors; + if self.tb.key(k, errs.len()) { + return (true, Cmd::None); + } + if self.tb.at() < errs.len() && (k == "enter" || k == "space") { + m.jump_to(tr, errs[self.tb.at()].node); + return (true, Cmd::None); + } + (false, Cmd::None) + } +} diff --git a/rust/src/tui/keys.rs b/rust/src/tui/keys.rs new file mode 100644 index 0000000..3dcfbb0 --- /dev/null +++ b/rust/src/tui/keys.rs @@ -0,0 +1,228 @@ +//! Key dispatch and the actions keys trigger. + +use super::model::{is_group, Cmd, Key, Model, Msg}; +use super::modals::{new_group_modal, new_help_modal, new_info_modal, new_save_modal}; +use super::theme::{load_theme, theme_names}; +use super::views::View; +use super::{fmt_stamp_now, san}; +use crate::brand; +use crate::inventory::{NodeId, SizeMode, Tree, NO_NODE}; +use crate::platform; +use crate::snapshot; +use crate::textutil; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::Path; +use std::rc::Rc; + +impl Model { + /// Dispatches a key press. It holds the tree read lock for the whole + /// handler; helpers it calls must not lock again (a recursive read lock + /// can deadlock against a waiting writer). + pub fn handle_key(&mut self, key: &Key) -> Cmd { + let k = key.name.as_str(); + if k == "ctrl+c" { + return self.ctrl_c_pressed(); + } + let arc = self.tree.clone(); + let tr = arc.read().unwrap(); + self.ensure_selection(&tr); + if !self.modals.is_empty() { + return self.modal_key(&tr, key); + } + if self.search.editing { + return self.search_key(key); + } + let (handled, cmd) = self.view_key(&tr, k); + if handled { + return cmd; + } + match k { + "q" => self.quit = true, + "?" | "f1" => { + let md = new_help_modal(); + self.open_modal(md); + } + "/" => self.search.begin(), + "esc" => { + if self.search.query.is_some() { + self.clear_filter(); + } else if self.zoom != 0 { + self.zoom_out(&tr); + } + } + "tab" => self.set_view((self.view + 1) % self.views.len()), + "shift+tab" => self.set_view((self.view + self.views.len() - 1) % self.views.len()), + "1" | "2" | "3" | "4" | "5" | "6" | "7" => self.set_view((k.as_bytes()[0] - b'1') as usize), + "enter" => return self.inspect_selection(&tr), + "space" | "right" => return self.zoom_into_selection(&tr), + "backspace" | "left" => self.zoom_out(&tr), + "home" => self.zoom_to(&tr, 0), + "a" => { + self.size_mode = if self.size_mode == SizeMode::Allocated { SizeMode::Logical } else { SizeMode::Allocated }; + self.invalidate(); + let mut cmds = Vec::new(); + if self.search.query.is_some() { + cmds.push(self.apply_filter()); + } + let msg = format!("Sizing by {} size", self.size_mode.name()); + cmds.push(self.info(msg)); + return Cmd::batch(cmds); + } + "r" => return self.rescan(&tr), + "s" => { + let md = new_save_modal(&tr); + self.open_modal(md); + } + "e" => { + self.set_view_by_name("Scan"); + if let View::Info(v) = &mut self.views[self.view] { + v.focus_errors(); + } + } + "x" => self.set_view_by_name("Types"), + "g" => self.set_view_by_name("Top"), + "D" => return self.start_duplicates(), + "c" => return self.copy_selected(&tr), + "o" => return self.reveal(&tr), + "d" | "delete" => return self.request_trash(&tr), + "T" => { + let names = theme_names(); + let cur = names.iter().position(|n| *n == self.theme.name).unwrap_or(0); + let ascii = self.theme.ascii; + let mut th = load_theme(names[(cur + 1) % names.len()], &self.opts.settings); + th.ascii = ascii; + self.theme = Rc::new(th); + let msg = format!("Theme: {}", self.theme.name); + return self.info(msg); + } + _ => {} + } + Cmd::None + } + + pub fn inspect_selection(&mut self, tr: &Tree) -> Cmd { + if is_group(self.sel) { + if let Some(g) = self.group_info(self.sel) { + self.open_modal(new_group_modal(g)); + } + return Cmd::None; + } + let id = self.selected_node(tr); + if id != NO_NODE { + let md = new_info_modal(self, tr, id); + self.open_modal(md); + } + Cmd::None + } + + pub fn zoom_into_selection(&mut self, tr: &Tree) -> Cmd { + if is_group(self.sel) { + // Groups cannot be zoomed: show their members in the list view. + if let Some(g) = self.group_info(self.sel) { + if let Some(&first) = g.ids().first() { + self.sel = first as i64; + self.set_view_by_name("List"); + } + } + return Cmd::None; + } + let id = self.selected_node(tr); + if id == NO_NODE { + return Cmd::None; + } + if !tr.node(id).is_dir() { + return self.info("Not a directory — Enter shows details"); + } + if self.kids_of(tr, id).ids.is_empty() { + return self.info("Directory is empty"); + } + self.zoom_to(tr, id); + Cmd::None + } + + pub fn zoom_out(&mut self, tr: &Tree) { + let p = tr.node(self.zoom).parent; + if p != NO_NODE { + self.zoom_to(tr, p); + } + } + + /// Zooms to a node's parent directory and selects it. + pub fn jump_to(&mut self, tr: &Tree, id: NodeId) { + let p = tr.node(id).parent; + if p == NO_NODE { + self.zoom_to(tr, id); + return; + } + self.zoom_to(tr, p); + self.sel = id as i64; + self.set_view_by_name("Map"); + } + + fn copy_selected(&mut self, tr: &Tree) -> Cmd { + let id = self.selected_node(tr); + if id == NO_NODE { + return Cmd::None; + } + self.copy_path(&tr.path_bytes(id)) + } + + /// Puts p on the clipboard (OSC 52). Paths that would not survive + /// display unchanged are refused: a newline in a hostile filename runs + /// commands when pasted into a shell, and bidi controls disguise them. + pub fn copy_path(&mut self, p: &[u8]) -> Cmd { + let s = san(p); + if s.as_bytes() != p { + return self.warn("Not copied: path contains control or bidi characters"); + } + let i = self.info("Copied path to clipboard (OSC 52)"); + Cmd::batch(vec![Cmd::Clipboard(s), i]) + } + + pub fn reveal(&mut self, tr: &Tree) -> Cmd { + if self.snapshot { + return self.warn("Snapshot — the files may not exist on this machine"); + } + let id = self.selected_node(tr); + if id == NO_NODE { + return Cmd::None; + } + if platform::headless() { + // No file manager: show the folder in the map instead. + if tr.node(id).is_dir() && !self.kids_of(tr, id).ids.is_empty() { + self.zoom_to(tr, id); + self.set_view_by_name("Map"); + } else { + self.jump_to(tr, id); + } + let msg = format!("Showing {}", san(&tr.path_bytes(self.zoom))); + return self.info(msg); + } + if let Err(e) = platform::reveal(&tr.path(id)) { + return self.warn(format!("Open failed: {}", textutil::sanitize(&e.to_string()))); + } + self.info("Opened in file manager") + } + + pub fn save_snapshot(&mut self, path: String) -> Cmd { + let t = self.tree.clone(); + Cmd::run(move || { + let err = { + let g = t.read().unwrap(); + snapshot::save_file(Path::new(&path), &g).err().map(|e| e.to_string()) + }; + Msg::SaveDone { path, err } + }) + } +} + +pub(crate) fn default_snapshot_name(root: &[u8]) -> String { + let p = Path::new(OsStr::from_bytes(root)); + let base = match p.file_name() { + Some(b) => String::from_utf8_lossy(b.as_bytes()).into_owned(), + None => "root".into(), + }; + let wd = std::env::current_dir().unwrap_or_default(); + wd.join(format!("{base}-{}{}", fmt_stamp_now(), brand::SNAPSHOT_EXT)).to_string_lossy().into_owned() +} diff --git a/rust/src/tui/mapview.rs b/rust/src/tui/mapview.rs new file mode 100644 index 0000000..d78aa01 --- /dev/null +++ b/rust/src/tui/mapview.rs @@ -0,0 +1,383 @@ +//! The treemap view: layout with level-of-detail grouping, nested previews +//! inside directories, painting and spatial navigation. + +use super::canvas::{rgb, Canvas, Style, BOLD, BOX_ROUND, FAINT}; +use super::data::Kids; +use super::model::{group_id, is_group, Cmd, Hit, HitAct, Model}; +use super::render::kind_label; +use super::{san, trunc, tw}; +use crate::inventory::{NodeId, Tree, NO_NODE}; +use crate::textutil; +use crate::treemap::{self, Block, Direction, Item, Rect}; +use std::collections::{HashMap, HashSet}; +use std::rc::Rc; + +/// The treemap geometry for one (zoom, rect, data version). It is recomputed +/// from scratch whenever any of those change — after a resize the layout +/// algorithm runs again for the new rectangle rather than scaling the old +/// one. +pub(crate) struct TmLayout { + pub zoom: NodeId, + pub rect: Rect, + pub ver: u64, + pub blocks: Vec, // top level: navigable + pub nested: Vec, // previews inside directories: drawn only + pub groups: HashMap, + pub visible: HashSet, + pub inner: HashSet, // top-level blocks with nested previews +} + +#[derive(Clone, Copy)] +pub(crate) struct Nested { + pub id: i64, + pub r: Rect, + pub depth: i32, + pub alt: bool, + pub inner: bool, // has nested children drawn inside it +} + +/// An LOD group: the children of parent from index `from` on. +#[derive(Clone)] +pub(crate) struct Group { + pub parent: NodeId, + kids: Rc, + from: usize, + pub size: i64, +} + +impl Group { + pub fn ids(&self) -> &[NodeId] { + &self.kids.ids[self.from..] + } +} + +// LOD parameters. +pub(crate) const TOP_MIN_AREA: i32 = 12; // cells: room for a 4×3 framed block +const TOP_MAX_ITEMS: usize = 400; +const NEST_MIN_AREA: i32 = 3; +const NEST_MAX_ITEMS: usize = 120; +const MAX_NEST_DEPTH: i32 = 8; // block-size thresholds stop recursion long before this +const NEST_MIN_W: i32 = 4; +const NEST_MIN_H: i32 = 2; +const NEST_RECURSE_W: i32 = 14; +const NEST_RECURSE_H: i32 = 5; + +impl Model { + pub fn layout_treemap(&mut self, tr: &Tree, r: Rect) -> Rc { + if let Some(l) = &self.tm { + if l.zoom == self.zoom && l.rect == r && l.ver == self.data_ver { + return l.clone(); + } + } + let mut l = TmLayout { + zoom: self.zoom, rect: r, ver: self.data_ver, blocks: Vec::new(), nested: Vec::new(), + groups: HashMap::new(), visible: HashSet::new(), inner: HashSet::new(), + }; + l.blocks = self.layout_level(tr, &mut l, self.zoom, r, TOP_MIN_AREA, TOP_MAX_ITEMS); + for i in 0..l.blocks.len() { + let b = l.blocks[i]; + l.visible.insert(b.id); + if b.id < 0 || b.rect.w < 3 || b.rect.h < 2 { + continue; + } + if tr.node(b.id as NodeId).is_dir() { + let inn = b.rect.inset(1); + if inn.w >= NEST_MIN_W && inn.h >= NEST_MIN_H { + let before = l.nested.len(); + let end = self.chain_end(tr, b.id as NodeId); + self.layout_nested(tr, &mut l, end, inn, 2); + if l.nested.len() > before { + l.inner.insert(b.id); + } + } + } + } + let l = Rc::new(l); + self.tm = Some(l.clone()); + l + } + + /// Lays out the children of parent in r with level-of-detail grouping: + /// children too small to be useful collapse into one group block. + fn layout_level(&mut self, tr: &Tree, l: &mut TmLayout, parent: NodeId, r: Rect, min_area: i32, max_items: usize) -> Vec { + let k = self.kids_of(tr, parent); + if k.ids.is_empty() || r.empty() { + return Vec::new(); + } + let sizes: Vec = k.sizes.iter().map(|&s| s as f64).collect(); + let vis = treemap::visible(&sizes, k.total as f64, r.area(), min_area, max_items); + let mut items: Vec = (0..vis).map(|i| Item { id: k.ids[i] as i64, size: sizes[i] }).collect(); + if vis < k.ids.len() { + let size = k.sizes[vis..].iter().sum(); + let gid = group_id(parent); + l.groups.insert(gid, Group { parent, kids: k.clone(), from: vis, size }); + items.push(Item { id: gid, size: size as f64 }); + } + treemap::squarify(&items, r) + } + + fn layout_nested(&mut self, tr: &Tree, l: &mut TmLayout, parent: NodeId, r: Rect, depth: i32) { + // Deeper levels need bigger blocks to stay legible rather than noisy. + let blocks = self.layout_level(tr, l, parent, r, NEST_MIN_AREA * (depth - 1), NEST_MAX_ITEMS); + for (i, b) in blocks.iter().enumerate() { + let idx = l.nested.len(); + l.nested.push(Nested { id: b.id, r: b.rect, depth, alt: i % 2 == 1, inner: false }); + let br = b.rect; + if depth < MAX_NEST_DEPTH && b.id >= 0 && br.w >= NEST_RECURSE_W && br.h >= NEST_RECURSE_H && tr.node(b.id as NodeId).is_dir() { + l.nested[idx].inner = true; + let end = self.chain_end(tr, b.id as NodeId); + self.layout_nested(tr, l, end, Rect { x: br.x, y: br.y + 1, w: br.w - 1, h: br.h - 2 }, depth + 1); + } + } + } + + /// Follows directories whose only non-empty child is a directory + /// (mnt → raid → media), so nesting spends its space on the level that + /// actually branches instead of on full-size wrappers. + pub fn chain_end(&mut self, tr: &Tree, mut id: NodeId) -> NodeId { + loop { + let k = self.kids_of(tr, id); + if k.ids.len() != 1 || !tr.node(k.ids[0]).is_dir() { + return id; + } + id = k.ids[0]; + } + } + + /// The LOD group for a group ID in the current layout. + pub fn group_info(&self, id: i64) -> Option { + self.tm.as_ref()?.groups.get(&id).cloned() + } + + /// A sanitised display name for a block. + pub fn block_name(&mut self, tr: &Tree, id: i64) -> String { + if is_group(id) { + return match self.group_info(id) { + Some(g) => textutil::count(g.ids().len() as i64) + " smaller items", + None => "smaller items".into(), + }; + } + let mut nid = id as NodeId; + let n = tr.node(nid); + let mut name = san(&n.name); + if n.is_dir() && nid != 0 { + name.push('/'); + let end = self.chain_end(tr, nid); + while nid != end { + nid = self.kids_of(tr, nid).ids[0]; + name += &san(&tr.node(nid).name); + name.push('/'); + } + } + name + } + + fn block_size(&self, tr: &Tree, id: i64) -> i64 { + if is_group(id) { + return self.group_info(id).map_or(0, |g| g.size); + } + self.size_of(tr, id as NodeId) + } + + /// The size of the folder a block sits in. + fn parent_size(&self, tr: &Tree, id: i64) -> i64 { + if is_group(id) { + return self.group_info(id).map_or(0, |g| self.size_of(tr, g.parent)); + } + let p = tr.node(id as NodeId).parent; + if p != NO_NODE { + return self.size_of(tr, p); + } + 0 + } + + fn paint_top(&mut self, tr: &Tree, cv: &mut Canvas, b: Block, sel: bool, hov: bool, total: i64) { + let t = self.theme.clone(); + let base = self.color_of(tr, b.id); + let fill = base.mix(t.bg, 0.62); + let fill_ch = if !is_group(b.id) { + " " + } else if t.ascii { + "." + } else { + "░" + }; + let name = self.block_name(tr, b.id); + let size = self.block_size(tr, b.id); + let r = b.rect; + if r.w < 3 || r.h < 2 { + // Too small for a frame: a solid tile. + let mut st = Style::new(base, base.mix(t.bg, 0.3)); + let mut ch = fill_ch; + if sel { + st = Style::new(t.text_on(t.sel), t.sel).with(BOLD | self.mono_rev()); + ch = if t.ascii { "#" } else { "◆" }; + } else if t.mono { + ch = "▒"; + } + cv.fill(r, " ", st); + cv.text(r.x + (r.w - 1) / 2, r.y + (r.h - 1) / 2, ch, 1, st); + return; + } + cv.fill(r, fill_ch, Style::new(base.mix(t.bg, 0.35), fill)); + let mut frame = Style::new(base.mix(t.bg, 0.15), fill); + let mut bx = t.box_chars(); + let mut title = Style::new(base.mix(rgb(255, 255, 255), 0.35), fill).with(BOLD); + if sel { + bx = t.sel_box(); + frame = Style::new(t.sel, fill).with(BOLD); + title = Style::new(t.text_on(t.sel), t.sel).with(BOLD | self.mono_rev()); + } else if hov { + frame.fg = base.mix(rgb(255, 255, 255), 0.5); + } + if t.mono && !sel { + title.attr = BOLD; + } + cv.draw_box(r, bx, frame); + cv.text(r.x + 1, r.y, &trunc(&format!(" {name} "), r.w - 2), r.w - 2, title); + + let mut label = textutil::size(size); + let pct = textutil::percent(size, total); + if r.w - 2 >= tw(&label) + tw(&pct) + 5 { + label += &format!(" · {pct}"); + } else if r.w - 2 < tw(&label) + 2 { + label = textutil::size_compact(size); + } + let label = format!(" {label} "); + if r.h >= 3 && tw(&label) <= r.w - 2 { + cv.text_right(r.x + 1, r.y + r.h - 1, r.w - 2, &label, Style { fg: frame.fg, bg: fill, attr: frame.attr }); + } + // Interior labels for blocks without nested previews. + let inn = r.inset(1); + if inn.empty() || self.has_nested(b.id) { + return; + } + let mut lines = Vec::new(); + if !is_group(b.id) { + let nd = tr.node(b.id as NodeId); + if !nd.is_dir() { + lines.push(kind_label(nd)); + } else if nd.files > 0 { + lines.push(textutil::count(nd.files as i64) + " files"); + } + } + if r.h < 4 || inn.w < 6 { + return; + } + let y0 = inn.y + (inn.h - lines.len() as i32) / 2; + for (i, s) in lines.iter().enumerate() { + cv.text_centered(inn.x, y0 + i as i32, inn.w, s, Style::new(base.mix(t.fg, 0.4), fill)); + } + } + + fn has_nested(&self, id: i64) -> bool { + self.tm.as_ref().is_some_and(|l| l.inner.contains(&id)) + } + + fn paint_nested(&mut self, tr: &Tree, cv: &mut Canvas, nb: &Nested) { + let t = self.theme.clone(); + let base = self.color_of(tr, nb.id); + let mut f = (0.38 - 0.1 * (nb.depth - 2) as f64).max(0.1); + if nb.alt { + f += 0.07; + } + let bg = base.mix(t.bg, f); + let shadow = bg.mix(rgb(0, 0, 0), 0.45); + let mut st = Style::new(t.text_on(bg), bg); + let mut ch = " "; + if is_group(nb.id) { + ch = if t.ascii { "." } else { "░" }; + st.fg = bg.mix(t.fg, 0.25); + } + let r = nb.r; + cv.fill(r, ch, st); + if t.mono { + // No colour: draw a light outline so neighbours stay distinguishable. + if r.w >= 2 && r.h >= 2 { + cv.draw_box(r, BOX_ROUND, Style::default().with(FAINT)); + } + } else { + if r.w >= 4 { + cv.fill(Rect { x: r.x + r.w - 1, y: r.y, w: 1, h: r.h }, " ", Style::new(Default::default(), shadow)); + } + if r.h >= 3 { + cv.fill(Rect { x: r.x, y: r.y + r.h - 1, w: r.w, h: 1 }, " ", Style::new(Default::default(), shadow)); + } + } + let lw = if t.mono { r.w - 2 } else { r.w - 1 }; + if lw < 5 { + return; + } + let lx = if t.mono { r.x + 1 } else { r.x }; + let ly = if t.mono && r.h >= 3 { r.y + 1 } else { r.y }; + let name = self.block_name(tr, nb.id); + if tw(&name) > lw && lw < 8 { + return; // a stub like "te…" is noise, not information + } + cv.text(lx, ly, &trunc(&name, lw), lw, Style::new(st.fg, bg).with(BOLD)); + if !nb.inner && r.h >= 3 && lw >= 5 { + let size = self.block_size(tr, nb.id); + let mut label = textutil::size_compact(size); + let pct = textutil::percent(size, self.parent_size(tr, nb.id)); + if tw(&label) + tw(&pct) + 3 <= lw { + label += &format!(" · {pct}"); + } + cv.text(lx, ly + 1, &trunc(&label, lw), lw, Style::new(bg.mix(st.fg, 0.7), bg)); + } + } +} + +pub(crate) fn map_paint(m: &mut Model, tr: &Tree, cv: &mut Canvas, r: Rect) { + let t = m.theme.clone(); + let l = m.layout_treemap(tr, r); + if l.blocks.is_empty() { + let z = tr.node(m.zoom); + let msg = if m.search.query.is_some() { + "No matches here — Esc clears the filter".to_string() + } else if m.scanning { + "Scanning…".to_string() + } else if !z.is_dir() { + format!("{} — {}", san(&z.name), textutil::size(m.size_of(tr, m.zoom))) + } else { + "Empty directory".to_string() + }; + cv.text_centered(r.x, r.y + r.h / 2, r.w, &msg, t.muted()); + return; + } + let sel = m.visible_sel(&l.visible); + let total = m.size_of(tr, m.zoom); + for b in &l.blocks { + let hov = b.id == m.hover; + m.paint_top(tr, cv, *b, b.id == sel, hov, total); + } + for nb in &l.nested { + m.paint_nested(tr, cv, nb); + } + for b in &l.blocks { + m.hits.push(Hit { r: b.rect, act: HitAct::Block(b.id) }); + } +} + +pub(crate) fn map_key(m: &mut Model, k: &str) -> (bool, Cmd) { + let d = match k { + "left" | "h" => Direction::Left, + "right" | "l" => Direction::Right, + "up" | "k" => Direction::Up, + "down" | "j" => Direction::Down, + _ => return (false, Cmd::None), + }; + if let Some(l) = m.tm.clone() { + if let Some(id) = treemap::neighbour(&l.blocks, m.visible_sel(&l.visible), d) { + m.sel = id; + } + } + (true, Cmd::None) +} + +pub(crate) fn map_wheel(m: &mut Model, tr: &Tree, up: bool) -> Cmd { + if up { + return m.zoom_into_selection(tr); + } + m.zoom_out(tr); + Cmd::None +} diff --git a/rust/src/tui/mod.rs b/rust/src/tui/mod.rs new file mode 100644 index 0000000..f923945 --- /dev/null +++ b/rust/src/tui/mod.rs @@ -0,0 +1,83 @@ +//! The interactive terminal interface. All drawing goes to a Canvas; the +//! event loop (run.rs) only transports the finished frame to the terminal. + +mod actions; +pub mod bench; +mod canvas; +mod color256; +mod data; +mod infoview; +mod keys; +mod mapview; +mod modal; +mod modals; +mod model; +mod mouse; +mod render; +mod run; +mod search; +mod side; +mod table; +mod theme; +mod views; + +#[cfg(test)] +mod tests; + +pub use model::{Options, StartFn}; +pub use run::run; + +use crate::textutil; + +// Width-aware text helpers taking the canvas's signed cell counts. + +pub(crate) fn tw(s: &str) -> i32 { + textutil::width(s) as i32 +} + +pub(crate) fn trunc(s: &str, w: i32) -> String { + textutil::truncate(s, w.max(0) as usize) +} + +pub(crate) fn trunc_left(s: &str, w: i32) -> String { + textutil::truncate_left(s, w.max(0) as usize) +} + +pub(crate) fn pad_left(s: &str, w: i32) -> String { + textutil::pad_left(s, w.max(0) as usize) +} + +pub(crate) fn pad_right(s: &str, w: i32) -> String { + textutil::pad_right(s, w.max(0) as usize) +} + +pub(crate) fn san(b: &[u8]) -> String { + textutil::sanitize_bytes(b) +} + + +fn local_tm(unix_nanos: i64) -> libc::tm { + let secs = unix_nanos.div_euclid(1_000_000_000) as libc::time_t; + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + unsafe { libc::localtime_r(&secs, &mut tm) }; + tm +} + +/// Formats a timestamp as a local date ("2006-01-02"). +pub(crate) fn fmt_date(unix_nanos: i64) -> String { + let t = local_tm(unix_nanos); + format!("{:04}-{:02}-{:02}", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday) +} + +/// Formats a timestamp as local date and time ("2006-01-02 15:04:05"). +pub(crate) fn fmt_time(unix_nanos: i64) -> String { + let t = local_tm(unix_nanos); + format!("{:04}-{:02}-{:02} {:02}:{:02}:{:02}", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min, t.tm_sec) +} + +/// The current local time as "20060102-1504". +pub(crate) fn fmt_stamp_now() -> String { + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH).unwrap_or_default(); + let t = local_tm(now.as_nanos() as i64); + format!("{:04}{:02}{:02}-{:02}{:02}", t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, t.tm_min) +} diff --git a/rust/src/tui/modal.rs b/rust/src/tui/modal.rs new file mode 100644 index 0000000..e10ec70 --- /dev/null +++ b/rust/src/tui/modal.rs @@ -0,0 +1,229 @@ +//! The generic modal frame: a centred, clamped, scrollable overlay. + +use super::canvas::{rgb, Canvas, Style, BOLD, BOX_ASCII, BOX_ROUND, FAINT}; +use super::model::{Cmd, Hit, HitAct, Key, Model}; +use super::pad_right; +use super::search::LineEditor; +use crate::inventory::Tree; +use crate::textutil; +use crate::treemap::Rect; + +/// A run of styled text; a line is a list of spans. +pub(crate) struct Span { + pub s: String, + pub st: Option