From a530bc7a7549a19bd067ba0d26d2686703f91904 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Thu, 8 Oct 2026 17:08:49 +0100 Subject: [PATCH] rust: foundation (inventory, platform, scan, export, textutil, config) --- rust/.gitignore | 1 + rust/Cargo.lock | 550 +++++++++++++++++++++++ rust/Cargo.toml | 22 + rust/src/brand.rs | 11 + rust/src/cancel.rs | 41 ++ rust/src/config.rs | 75 ++++ rust/src/export.rs | 222 ++++++++++ rust/src/inventory/category.rs | 134 ++++++ rust/src/inventory/mod.rs | 481 ++++++++++++++++++++ rust/src/inventory/query.rs | 141 ++++++ rust/src/inventory/stats.rs | 124 ++++++ rust/src/lib.rs | 13 + rust/src/platform.rs | 707 ++++++++++++++++++++++++++++++ rust/src/scan.rs | 775 +++++++++++++++++++++++++++++++++ rust/src/textutil.rs | 368 ++++++++++++++++ 15 files changed, 3665 insertions(+) create mode 100644 rust/.gitignore create mode 100644 rust/Cargo.lock create mode 100644 rust/Cargo.toml create mode 100644 rust/src/brand.rs create mode 100644 rust/src/cancel.rs create mode 100644 rust/src/config.rs create mode 100644 rust/src/export.rs create mode 100644 rust/src/inventory/category.rs create mode 100644 rust/src/inventory/mod.rs create mode 100644 rust/src/inventory/query.rs create mode 100644 rust/src/inventory/stats.rs create mode 100644 rust/src/lib.rs create mode 100644 rust/src/platform.rs create mode 100644 rust/src/scan.rs create mode 100644 rust/src/textutil.rs diff --git a/rust/.gitignore b/rust/.gitignore new file mode 100644 index 0000000..ea8c4bf --- /dev/null +++ b/rust/.gitignore @@ -0,0 +1 @@ +/target diff --git a/rust/Cargo.lock b/rust/Cargo.lock new file mode 100644 index 0000000..e96506f --- /dev/null +++ b/rust/Cargo.lock @@ -0,0 +1,550 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crossterm" +version = "0.28.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" +dependencies = [ + "bitflags", + "crossterm_winapi", + "mio", + "parking_lot", + "rustix", + "signal-hook", + "signal-hook-mio", + "winapi", +] + +[[package]] +name = "crossterm_winapi" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acdd7c62a3665c7f6830a51635d9ac9b23ed385797f70a83bb8bafe9c572ab2b" +dependencies = [ + "winapi", +] + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "flate2" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb" +dependencies = [ + "crc32fast", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.190" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce5d3ddc6d3fa000eb1536d85e147bfe31aacaba692ed6a876f95cb7c855be78" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "lock_api" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" + +[[package]] +name = "mapsize" +version = "1.2.0" +dependencies = [ + "crossbeam-channel", + "crossterm", + "flate2", + "libc", + "serde", + "serde_json", + "sha2", + "unicode-segmentation", + "unicode-width", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "miniz_oxide" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "mio" +version = "1.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" +dependencies = [ + "libc", + "log", + "wasi", + "windows-sys 0.61.2", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-link", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags", +] + +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.151" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "signal-hook" +version = "0.3.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d881a16cf4426aa584979d30bd82cb33429027e42122b169753d6ef1085ed6e2" +dependencies = [ + "libc", + "signal-hook-registry", +] + +[[package]] +name = "signal-hook-mio" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b75a19a7a740b25bc7944bdee6172368f988763b744e3d4dfe753f6b4ece40cc" +dependencies = [ + "libc", + "mio", + "signal-hook", +] + +[[package]] +name = "signal-hook-registry" +version = "1.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4db69cba1110affc0e9f7bcd48bbf87b3f4fc7c61fc9155afd4c469eb3d6c1b" +dependencies = [ + "errno", + "libc", +] + +[[package]] +name = "simd-adler32" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a219298ac11a56ea9a6d2120044824d6f01aeb034955e7af7bc16858527deea" + +[[package]] +name = "smallvec" +version = "1.16.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9395f0f0eee849a9b707b2f06bb92a6a422090e2123bb2ef8e87a0e61892a8e" + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "typenum" +version = "1.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "zlib-rs" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/rust/Cargo.toml b/rust/Cargo.toml new file mode 100644 index 0000000..efd2908 --- /dev/null +++ b/rust/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "mapsize" +version = "1.2.0" +edition = "2021" +license = "GPL-3.0-or-later" +description = "Interactive terminal disk usage analyser (Rust port)" + +[dependencies] +libc = "0.2" +crossbeam-channel = "0.5" +crossterm = "0.28" +flate2 = "1" +sha2 = "0.10" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +unicode-segmentation = "1" +unicode-width = "0.2" + +[profile.release] +lto = true +codegen-units = 1 +strip = true diff --git a/rust/src/brand.rs b/rust/src/brand.rs new file mode 100644 index 0000000..5747cde --- /dev/null +++ b/rust/src/brand.rs @@ -0,0 +1,11 @@ +//! Product name isolated so it can be changed in one place. + +pub const NAME: &str = "mapsize"; +pub const SNAPSHOT_EXT: &str = ".msz"; +pub const SNAPSHOT_MAGIC: &[u8; 8] = b"MAPSIZE\x00"; + +/// Overridden at release time via the MAPSIZE_VERSION environment variable. +pub const VERSION: &str = match option_env!("MAPSIZE_VERSION") { + Some(v) => v, + None => env!("CARGO_PKG_VERSION"), +}; diff --git a/rust/src/cancel.rs b/rust/src/cancel.rs new file mode 100644 index 0000000..fdf8542 --- /dev/null +++ b/rust/src/cancel.rs @@ -0,0 +1,41 @@ +//! A clonable cancellation token, the equivalent of a Go context's Done. + +use crossbeam_channel::{Receiver, Sender}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +#[derive(Clone)] +pub struct Cancel(Arc); + +struct Inner { + flag: AtomicBool, + tx: Mutex>>, + rx: Receiver<()>, +} + +impl Default for Cancel { + fn default() -> Self { + Self::new() + } +} + +impl Cancel { + pub fn new() -> Cancel { + let (tx, rx) = crossbeam_channel::bounded(0); + Cancel(Arc::new(Inner { flag: AtomicBool::new(false), tx: Mutex::new(Some(tx)), rx })) + } + + pub fn cancel(&self) { + self.0.flag.store(true, Ordering::Relaxed); + self.0.tx.lock().unwrap().take(); + } + + pub fn is_cancelled(&self) -> bool { + self.0.flag.load(Ordering::Relaxed) + } + + /// Becomes ready (disconnected) once cancelled; for use in select. + pub fn done(&self) -> &Receiver<()> { + &self.0.rx + } +} diff --git a/rust/src/config.rs b/rust/src/config.rs new file mode 100644 index 0000000..422838d --- /dev/null +++ b/rust/src/config.rs @@ -0,0 +1,75 @@ +//! User settings from a JSON file in the OS-specific user configuration +//! directory. Knows nothing about rendering. + +use crate::brand; +use serde::Deserialize; +use std::collections::HashMap; +use std::path::PathBuf; + +/// User preferences. Load starts from the defaults and overlays the file. +#[derive(Clone, Debug, Deserialize)] +#[serde(default)] +pub struct Settings { + pub workers: String, // conservative|balanced|aggressive|N + pub follow_symlinks: String, // none|same-filesystem|all + pub one_file_system: bool, + pub units: String, // iec|si + pub size_mode: String, // allocated|logical + pub theme: String, + pub mouse: bool, + pub excludes: Vec, + /// Maps an extension (without dot) to "#rrggbb". + pub extension_colors: HashMap, + /// Maps a category name to "#rrggbb". + pub category_colors: HashMap, + /// UI refresh rate while scanning. + pub refresh_hz: i64, +} + +impl Default for Settings { + fn default() -> Self { + Settings { + workers: "balanced".into(), + follow_symlinks: "none".into(), + one_file_system: false, + units: "iec".into(), + size_mode: "allocated".into(), + theme: "default".into(), + mouse: true, + excludes: Vec::new(), + extension_colors: HashMap::new(), + category_colors: HashMap::new(), + refresh_hz: 10, + } + } +} + +/// The config file location. +pub fn path() -> Option { + let env = |k: &str| std::env::var_os(k).filter(|v| !v.is_empty()).map(PathBuf::from); + let dir = if cfg!(target_os = "macos") { + env("HOME")?.join("Library/Application Support") + } else { + env("XDG_CONFIG_HOME").or_else(|| env("HOME").map(|h| h.join(".config")))? + }; + Some(dir.join(brand::NAME).join("config.json")) +} + +/// Reads the config file if present. A missing file is not an error. +pub fn load() -> (Settings, Option) { + let Some(p) = path() else { return (Settings::default(), None) }; + let b = match std::fs::read(&p) { + Ok(b) => b, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return (Settings::default(), None), + Err(e) => return (Settings::default(), Some(e.to_string())), + }; + match serde_json::from_slice::(&b) { + Ok(mut s) => { + if !(1..=60).contains(&s.refresh_hz) { + s.refresh_hz = 10; + } + (s, None) + } + Err(e) => (Settings::default(), Some(e.to_string())), + } +} diff --git a/rust/src/export.rs b/rust/src/export.rs new file mode 100644 index 0000000..60490bd --- /dev/null +++ b/rust/src/export.rs @@ -0,0 +1,222 @@ +//! Writes inventories as JSON, CSV or plain-text reports. Callers hold the +//! tree's read lock (or own the tree exclusively). + +use crate::brand; +use crate::inventory::{self as inv, Flags, Kind, NodeId, SizeMode, Tree, NO_NODE}; +use crate::textutil; +use std::io::{self, Write}; +use std::time::UNIX_EPOCH; + +const FLAG_NAMES: [(Flags, &str); 12] = [ + (inv::FLAG_HARDLINK_DUP, "hardlink-duplicate"), (inv::FLAG_MOUNT_POINT, "mount-point"), + (inv::FLAG_VIRTUAL_FS, "virtual-fs-skipped"), (inv::FLAG_SKIPPED_FS, "other-fs-skipped"), + (inv::FLAG_SPARSE, "sparse"), (inv::FLAG_ERROR, "error"), (inv::FLAG_INCOMPLETE, "incomplete"), + (inv::FLAG_LOOP, "loop-skipped"), (inv::FLAG_FOLLOWED, "followed-symlink"), + (inv::FLAG_ALLOC_UNKNOWN, "allocated-unknown"), (inv::FLAG_BROKEN_LINK, "broken-symlink"), + (inv::FLAG_HARDLINKED, "hardlinked"), +]; + +/// Human-readable names for a node's flags. +pub fn flag_list(f: Flags) -> Vec<&'static str> { + FLAG_NAMES.iter().filter(|(b, _)| f & b != 0).map(|(_, n)| *n).collect() +} + +fn kind_name(k: Kind) -> &'static str { + ["file", "dir", "symlink", "other"][k as usize] +} + +fn json_str(s: &str) -> String { + serde_json::to_string(s).unwrap() +} + +/// Writes the tree beneath root as nested JSON, down to max_depth levels +/// (negative means unlimited). +pub fn json(w: &mut dyn Write, t: &Tree, root: NodeId, max_depth: i64) -> io::Result<()> { + let mut bw = io::BufWriter::new(w); + let st = &t.stats; + let errors: serde_json::Map = st + .err_counts + .iter() + .enumerate() + .filter(|(_, &c)| c > 0) + .map(|(k, &c)| (inv::ErrKind::from_u8(k as u8).name().to_string(), c.into())) + .collect(); + let started = st.start.duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos() as i64); + let head = serde_json::json!({ + "generator": format!("{} {}", brand::NAME, brand::VERSION), + "root": t.path_string(root), + "complete": !st.incomplete(), + "cancelled": st.cancelled, + "started": textutil::rfc3339_utc(started), + "elapsed_s": st.elapsed().as_secs_f64(), + "errors": errors, + "excluded": st.excluded, + "excludes": st.excludes, + "hardlink_duplicates": st.hardlink_dups, + }); + let hb = serde_json::to_string(&head)?; + bw.write_all(&hb.as_bytes()[..hb.len() - 1])?; + bw.write_all(b",\"tree\":")?; + write_node(&mut bw, t, root, max_depth)?; + bw.write_all(b"}\n")?; + bw.flush() +} + +/// Streams one node; children are written recursively so memory stays +/// proportional to depth, not tree size. +fn write_node(bw: &mut impl Write, t: &Tree, id: NodeId, depth: i64) -> io::Result<()> { + let n = t.node(id); + write!(bw, "{{\"name\":{},\"type\":\"{}\",\"size\":{},\"allocated\":{}", json_str(&n.name_str()), kind_name(n.kind), n.tot_size, n.tot_alloc)?; + for (k, v) in [("files", n.files), ("dirs", n.dirs), ("errors", n.errors)] { + if v != 0 { + write!(bw, ",\"{k}\":{v}")?; + } + } + if n.mtime != 0 { + write!(bw, ",\"modified\":\"{}\"", textutil::rfc3339_utc(n.mtime))?; + } + let flags = flag_list(n.flags); + if !flags.is_empty() { + write!(bw, ",\"flags\":{}", serde_json::to_string(&flags)?)?; + } + if n.first_child == NO_NODE || depth == 0 { + return bw.write_all(b"}"); + } + bw.write_all(b",\"children\":[")?; + for (i, c) in t.sorted_children(id, SizeMode::Allocated).into_iter().enumerate() { + if i > 0 { + bw.write_all(b",")?; + } + write_node(bw, t, c, depth - 1)?; + } + bw.write_all(b"]}") +} + +fn csv_field(s: &str) -> String { + if s.contains([',', '"', '\n', '\r']) || s.starts_with(' ') { + format!("\"{}\"", s.replace('"', "\"\"")) + } else { + s.to_string() + } +} + +/// Writes one row per node beneath root (depth-limited). Paths are +/// sanitised so the output is safe to print to a terminal. +pub fn csv(w: &mut dyn Write, t: &Tree, root: NodeId, max_depth: i64) -> io::Result<()> { + let mut bw = io::BufWriter::new(w); + bw.write_all(b"path,type,size,allocated,files,dirs,errors,modified,flags\n")?; + fn rec(bw: &mut impl Write, t: &Tree, id: NodeId, path: &[u8], depth: i64) -> io::Result<()> { + let n = t.node(id); + let modified = if n.mtime != 0 { textutil::rfc3339_utc(n.mtime) } else { String::new() }; + writeln!( + bw, + "{},{},{},{},{},{},{},{},{}", + csv_field(&textutil::sanitize_bytes(path)), + kind_name(n.kind), + n.tot_size, n.tot_alloc, n.files, n.dirs, n.errors, modified, + flag_list(n.flags).join("|") + )?; + if depth == 0 { + return Ok(()); + } + for c in t.sorted_children(id, SizeMode::Allocated) { + rec(bw, t, c, &crate::platform::join_path(path, &t.node(c).name), depth - 1)?; + } + Ok(()) + } + rec(&mut bw, t, root, &t.path_bytes(root), max_depth)?; + bw.flush() +} + +/// Writes "size % files path" rows for the given nodes. +pub fn table(w: &mut dyn Write, t: &Tree, ids: &[NodeId], m: SizeMode) -> io::Result<()> { + let total = t.node(t.root()).total(m); + writeln!(w, "{:>12} {:>6} {:>12} PATH", "SIZE", "%", "FILES")?; + for &id in ids { + let n = t.node(id); + let files = if n.is_dir() { textutil::count(n.files as i64) } else { String::new() }; + writeln!( + w, + "{:>12} {:>6} {:>12} {}", + textutil::size(n.total(m)), + textutil::percent(n.total(m), total), + files, + textutil::sanitize_bytes(&t.path_bytes(id)) + )?; + } + Ok(()) +} + +/// Writes the scan summary, making incompleteness explicit. +pub fn summary(w: &mut dyn Write, t: &Tree) -> io::Result<()> { + let st = &t.stats; + let r = t.node(t.root()); + writeln!(w, "Root {}", textutil::sanitize_bytes(&st.root))?; + writeln!(w, "Files {}", textutil::count(st.files + st.symlinks + st.others))?; + writeln!(w, "Directories {}", textutil::count(st.dirs))?; + writeln!(w, "Logical size {} ({} bytes)", textutil::size(r.tot_size), r.tot_size)?; + writeln!(w, "Allocated size {} ({} bytes)", textutil::size(r.tot_alloc), r.tot_alloc)?; + writeln!(w, "Elapsed {}", textutil::go_duration(st.elapsed()))?; + for (k, &c) in st.err_counts.iter().enumerate() { + if c > 0 { + writeln!(w, "{:<15} {}", inv::ErrKind::from_u8(k as u8).name(), textutil::count(c))?; + } + } + if st.excluded > 0 { + writeln!(w, "Excluded {} entries ({})", textutil::count(st.excluded), textutil::sanitize(&st.excludes.join(", ")))?; + } + if st.skipped_mounts > 0 { + writeln!(w, "Other FS skipped {}", textutil::count(st.skipped_mounts))?; + } + if st.virtual_skipped > 0 { + writeln!(w, "Virtual FS skipped {}", textutil::count(st.virtual_skipped))?; + } + if st.hardlink_dups > 0 { + writeln!(w, "Hard-link dups {} (counted once)", textutil::count(st.hardlink_dups))?; + } + if st.incomplete() { + writeln!(w, "WARNING: totals are incomplete — some data could not be inspected.")?; + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn hostile_tree() -> Tree { + let mut t = Tree::new(b"/r", Kind::Dir); + let d = t.add(0, b"esc\x1b[2Jdir", Kind::Dir); + let f = t.add(d, b"nl\nname\x07.txt", Kind::File); + t.node_mut(f).size = 10; + t.node_mut(f).alloc = 4096; + t.stats.complete = true; + t.recompute(); + t + } + + #[test] + fn outputs_are_terminal_safe() { + let t = hostile_tree(); + for out in [ + { let mut b = Vec::new(); json(&mut b, &t, 0, -1).unwrap(); b }, + { let mut b = Vec::new(); csv(&mut b, &t, 0, -1).unwrap(); b }, + { let mut b = Vec::new(); table(&mut b, &t, &[1, 2], SizeMode::Allocated).unwrap(); b }, + ] { + assert!(!out.iter().any(|&c| c == 0x1b || c == 0x07), "{}", String::from_utf8_lossy(&out)); + } + let mut b = Vec::new(); + json(&mut b, &t, 0, -1).unwrap(); + let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); + assert_eq!(v["tree"]["children"][0]["children"][0]["size"], 10); + } + + #[test] + fn json_depth_limit() { + let t = hostile_tree(); + let mut b = Vec::new(); + json(&mut b, &t, 0, 1).unwrap(); + let v: serde_json::Value = serde_json::from_slice(&b).unwrap(); + assert!(v["tree"]["children"][0].get("children").is_none()); + } +} diff --git a/rust/src/inventory/category.rs b/rust/src/inventory/category.rs new file mode 100644 index 0000000..9b1a2a8 --- /dev/null +++ b/rust/src/inventory/category.rs @@ -0,0 +1,134 @@ +use std::collections::HashMap; +use std::sync::OnceLock; + +/// Coarse file classification used for colouring and stats. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug, Default)] +#[repr(u8)] +pub enum Category { + #[default] + Other, + Image, + Video, + Audio, + Archive, + DiskImage, + Database, + Code, + Document, + Executable, + Log, + Cache, +} + +pub const NUM_CATEGORIES: usize = 12; + +const CATEGORY_NAMES: [&str; NUM_CATEGORIES] = [ + "Other", "Image", "Video", "Audio", "Archive", "Disk image", "Database", + "Source code", "Document", "Executable", "Log", "Cache", +]; + +const ALL: [Category; NUM_CATEGORIES] = [ + Category::Other, Category::Image, Category::Video, Category::Audio, Category::Archive, + Category::DiskImage, Category::Database, Category::Code, Category::Document, + Category::Executable, Category::Log, Category::Cache, +]; + +impl Category { + pub fn from_u8(v: u8) -> Category { + ALL.get(v as usize).copied().unwrap_or(Category::Other) + } + pub fn all() -> &'static [Category; NUM_CATEGORIES] { + &ALL + } + pub fn name(self) -> &'static str { + CATEGORY_NAMES[self as usize] + } +} + +impl std::fmt::Display for Category { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.name()) + } +} + +/// Resolves a category by (case-insensitive) name or prefix. +pub fn parse_category(s: &str) -> Option { + let s = s.trim().to_lowercase(); + for (i, n) in CATEGORY_NAMES.iter().enumerate() { + let ln = n.to_lowercase(); + if ln == s || ln.replace(' ', "") == s || ln.starts_with(&s) && s.len() >= 3 { + return Some(ALL[i]); + } + } + match s.as_str() { + "vm" | "virtual machine" | "disk" => Some(Category::DiskImage), + "db" => Some(Category::Database), + "src" | "source" => Some(Category::Code), + "exe" | "binary" => Some(Category::Executable), + _ => None, + } +} + +fn ext_table() -> &'static HashMap<&'static str, Category> { + static T: OnceLock> = OnceLock::new(); + T.get_or_init(|| { + let mut m = HashMap::new(); + let mut add = |c: Category, exts: &'static str| { + for e in exts.split_whitespace() { + m.insert(e, c); + } + }; + add(Category::Image, "jpg jpeg png gif bmp tif tiff webp heic heif raw cr2 cr3 nef arw dng svg ico psd xcf avif jxl orf rw2"); + add(Category::Video, "mp4 mkv mov avi wmv flv webm m4v mpg mpeg ts m2ts vob 3gp ogv mts"); + add(Category::Audio, "mp3 flac wav ogg oga opus m4a aac wma aiff aif alac mid midi"); + add(Category::Archive, "zip tar gz tgz bz2 tbz xz txz zst zstd 7z rar lz lz4 lzma cab deb rpm apk jar war cpio pkg snap whl gem"); + add(Category::DiskImage, "iso img qcow qcow2 vmdk vdi vhd vhdx hdd ova ovf dmg raw vmem vmsn nvram sav"); + add(Category::Database, "db sqlite sqlite3 db3 mdb accdb frm ibd myd myi ldf mdf ndf dbf wal kdbx realm lmdb rdb aof"); + add(Category::Code, "c h cc cpp cxx hpp hh go rs py pyc js mjs cjs ts tsx jsx java kt kts scala rb php pl pm sh bash zsh fish lua swift m mm cs fs vb r jl hs ml ex exs erl clj dart sql css scss sass less html htm vue svelte json yaml yml toml xml proto gradle cmake mk ipynb o a lib obj class"); + add(Category::Document, "pdf doc docx odt rtf txt md rst tex xls xlsx ods csv tsv ppt pptx odp epub mobi djvu pages numbers key org adoc"); + add(Category::Executable, "exe dll so dylib bin msi app elf ko sys efi appimage com"); + add(Category::Log, "log journal"); + add(Category::Cache, "cache tmp temp swp bak old part crdownload"); + m + }) +} + +/// Classifies a file by its (lower-case) extension and by whether it lives +/// beneath a cache or log directory. +pub fn category_for(ext: &str, in_cache: bool, in_log: bool) -> Category { + let c = ext_table().get(ext).copied(); + match c { + _ if in_cache && matches!(c, None | Some(Category::Other) | Some(Category::Code)) => Category::Cache, + _ if in_log && matches!(c, None | Some(Category::Archive) | Some(Category::Other)) => Category::Log, + Some(c) => c, + None => Category::Other, + } +} + +/// Reports whether a directory path contains a cache or log component. It is +/// evaluated once per directory, not per file. +pub fn path_hints(path: &[u8]) -> (bool, bool) { + let (mut in_cache, mut in_log) = (false, false); + for part in path.split(|&b| b == b'/' || b == b'\\') { + if part.is_empty() || part.len() > 12 { + continue; + } + match part.to_ascii_lowercase().as_slice() { + b"cache" | b".cache" | b"caches" | b"__pycache__" | b".npm" | b".gradle" | b"inetcache" | b"temp" | b"tmp" => in_cache = true, + b"log" | b"logs" | b".logs" | b"journal" => in_log = true, + _ => {} + } + } + (in_cache, in_log) +} + +/// Returns the lower-case extension of name without the dot ("" if none, or +/// if the name is a dotfile like ".bashrc"). +pub fn ext(name: &[u8]) -> String { + match name.iter().rposition(|&b| b == b'.') { + Some(i) if i > 0 && i != name.len() - 1 && name.len() - i <= 16 => { + String::from_utf8_lossy(&name[i + 1..]).to_lowercase() + } + _ => String::new(), + } +} diff --git a/rust/src/inventory/mod.rs b/rust/src/inventory/mod.rs new file mode 100644 index 0000000..4905241 --- /dev/null +++ b/rust/src/inventory/mod.rs @@ -0,0 +1,481 @@ +//! The in-memory filesystem model: an append-only arena of nodes addressed +//! by stable NodeIds, with incrementally maintained directory aggregates. +//! +//! Concurrency: a Tree has exactly one writer (the scan controller or a +//! snapshot loader), shared as `Arc>`. Nodes are never removed, +//! so a NodeId stays valid for the Tree's lifetime. + +mod category; +mod query; +mod stats; + +pub use category::*; +pub use query::*; +pub use stats::*; + +use std::collections::HashMap; +use std::ffi::OsStr; +use std::os::unix::ffi::OsStrExt; +use std::path::PathBuf; + +/// Addresses a node within one Tree. The root is always 0. +pub type NodeId = u32; + +/// The nil NodeId. +pub const NO_NODE: NodeId = u32::MAX; + +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum Kind { + #[default] + File, + Dir, + Symlink, + Other, // devices, sockets, fifos, reparse points +} + +impl Kind { + pub fn from_u8(v: u8) -> Kind { + [Kind::File, Kind::Dir, Kind::Symlink, Kind::Other][(v & 3) as usize] + } + pub fn name(self) -> &'static str { + ["File", "Directory", "Symlink", "Special"][self as usize] + } +} + +/// Per-node facts (bit set). +pub type Flags = u16; +pub const FLAG_HARDLINK_DUP: Flags = 1 << 0; // additional link to an inode counted elsewhere +pub const FLAG_MOUNT_POINT: Flags = 1 << 1; // directory on a different device than its parent +pub const FLAG_VIRTUAL_FS: Flags = 1 << 2; // virtual filesystem, not descended +pub const FLAG_SKIPPED_FS: Flags = 1 << 3; // other filesystem, not descended (--one-file-system) +pub const FLAG_SPARSE: Flags = 1 << 4; // allocated < logical (sparse or compressed) +pub const FLAG_ERROR: Flags = 1 << 5; // stat or read failed +pub const FLAG_INCOMPLETE: Flags = 1 << 6; // directory contents not (fully) read +pub const FLAG_SCANNED: Flags = 1 << 7; // directory fully read +pub const FLAG_LOOP: Flags = 1 << 8; // directory already visited via another path +pub const FLAG_FOLLOWED: Flags = 1 << 9; // reached through a followed symlink +pub const FLAG_ALLOC_UNKNOWN: Flags = 1 << 10; // platform cannot report allocated size +pub const FLAG_BROKEN_LINK: Flags = 1 << 11; // symlink target does not exist +pub const FLAG_HARDLINKED: Flags = 1 << 12; // regular file with link count > 1 +pub const FLAG_DELETED: Flags = 1 << 13; // moved to trash from the UI + +/// One filesystem object. Sizes are own sizes; tot_* fields are aggregates +/// including the node itself and (for directories) everything beneath it. +#[derive(Clone, Debug)] +pub struct Node { + pub parent: NodeId, + pub first_child: NodeId, + pub next_sibling: NodeId, + dir: u32, // index into dir side table (directories only) + + /// Single path component (raw bytes); the root holds the full root path. + pub name: Box<[u8]>, + + pub size: i64, // own logical bytes + pub alloc: i64, // own allocated bytes + pub tot_size: i64, // aggregates (hard-link duplicates contribute 0) + pub tot_alloc: i64, + pub mtime: i64, // unix nanoseconds + + pub files: u32, // recursive counts beneath (dirs excludes self) + pub dirs: u32, + pub errors: u32, // recursive error count beneath and including self + pub mode: u32, // Go fs.FileMode bit layout (see platform::mode) + pub uid: u32, + pub gid: u32, + pub nlink: u32, + + pub ext: u16, + pub flags: Flags, + pub kind: Kind, + pub cat: Category, +} + +impl Node { + pub fn is_dir(&self) -> bool { + self.kind == Kind::Dir + } + pub fn has(&self, f: Flags) -> bool { + self.flags & f != 0 + } + /// The name as (lossy) UTF-8, unsanitised. + pub fn name_str(&self) -> std::borrow::Cow<'_, str> { + String::from_utf8_lossy(&self.name) + } +} + +/// An aggregate change applied to a node and all its ancestors. +#[derive(Clone, Copy, Default, PartialEq, Eq, Debug)] +pub struct Delta { + pub size: i64, + pub alloc: i64, + pub files: i64, + pub dirs: i64, + pub errors: i64, + pub cat: [i64; NUM_CATEGORIES], +} + +const CHUNK_BITS: u32 = 16; +const CHUNK_SIZE: usize = 1 << CHUNK_BITS; + +/// A growable array that never moves existing elements, so growth never +/// doubles peak memory. +#[derive(Clone)] +struct Chunked { + c: Vec>, + n: usize, +} + +impl Chunked { + fn new() -> Self { + Chunked { c: Vec::new(), n: 0 } + } + fn add(&mut self, v: T) -> u32 { + if self.n & (CHUNK_SIZE - 1) == 0 && self.n >> CHUNK_BITS == self.c.len() { + self.c.push(Vec::with_capacity(CHUNK_SIZE)); + } + self.c[self.n >> CHUNK_BITS].push(v); + self.n += 1; + (self.n - 1) as u32 + } + #[inline] + fn at(&self, i: u32) -> &T { + &self.c[(i >> CHUNK_BITS) as usize][i as usize & (CHUNK_SIZE - 1)] + } + #[inline] + fn at_mut(&mut self, i: u32) -> &mut T { + &mut self.c[(i >> CHUNK_BITS) as usize][i as usize & (CHUNK_SIZE - 1)] + } +} + +/// The inventory for one scan root. +#[derive(Clone)] +pub struct Tree { + nodes: Chunked, + dirs: Chunked<[i64; NUM_CATEGORIES]>, + exts: Vec, + ext_idx: HashMap, + pub stats: Stats, +} + +impl Tree { + /// Creates a tree whose root has the given full path and kind. + pub fn new(root_path: &[u8], kind: Kind) -> Tree { + let mut t = Tree { + nodes: Chunked::new(), + dirs: Chunked::new(), + exts: vec![String::new()], + ext_idx: HashMap::from([(String::new(), 0)]), + stats: Stats::default(), + }; + t.stats.root = root_path.to_vec(); + t.stats.start = std::time::SystemTime::now(); + t.add(NO_NODE, root_path, kind); + t + } + + pub fn len(&self) -> usize { + self.nodes.n + } + pub fn is_empty(&self) -> bool { + self.nodes.n == 0 + } + #[inline] + pub fn node(&self, id: NodeId) -> &Node { + self.nodes.at(id) + } + #[inline] + pub fn node_mut(&mut self, id: NodeId) -> &mut Node { + self.nodes.at_mut(id) + } + pub fn root(&self) -> NodeId { + 0 + } + + /// Appends a child node and links it under parent. It does not touch + /// aggregates; use propagate. + pub fn add(&mut self, parent: NodeId, name: &[u8], kind: Kind) -> NodeId { + let mut n = Node { + parent, + first_child: NO_NODE, + next_sibling: NO_NODE, + dir: 0, + name: name.into(), + size: 0, alloc: 0, tot_size: 0, tot_alloc: 0, mtime: 0, + files: 0, dirs: 0, errors: 0, mode: 0, uid: 0, gid: 0, nlink: 0, + ext: 0, flags: 0, kind, cat: Category::Other, + }; + if kind == Kind::Dir { + n.dir = self.dirs.add([0; NUM_CATEGORIES]); + } else { + n.ext = self.intern_ext(&ext(name)); + } + let id = self.nodes.add(n); + if parent != NO_NODE { + let first = self.node(parent).first_child; + self.node_mut(id).next_sibling = first; + self.node_mut(parent).first_child = id; + } + id + } + + /// Interns an extension string; used by loaders rebuilding a tree. + pub fn intern_ext(&mut self, e: &str) -> u16 { + if let Some(&i) = self.ext_idx.get(e) { + return i; + } + if self.exts.len() >= (1 << 16) - 1 { + return 0; + } + self.exts.push(e.to_string()); + let i = (self.exts.len() - 1) as u16; + self.ext_idx.insert(e.to_string(), i); + i + } + + /// The extension string for a node ("" if none). + pub fn ext_name(&self, n: &Node) -> &str { + &self.exts[n.ext as usize] + } + pub fn ext_count(&self) -> usize { + self.exts.len() + } + pub fn ext_by_index(&self, i: usize) -> &str { + &self.exts[i] + } + + /// Per-category allocated bytes beneath a directory. + pub fn cat_sizes(&self, id: NodeId) -> Option<&[i64; NUM_CATEGORIES]> { + let n = self.node(id); + (n.kind == Kind::Dir).then(|| self.dirs.at(n.dir)) + } + + /// Adds d to id and every ancestor. Cost is O(depth); callers batch many + /// entries into one Delta. + pub fn propagate(&mut self, mut id: NodeId, d: &Delta) { + while id != NO_NODE { + let n = self.nodes.at_mut(id); + n.tot_size += d.size; + n.tot_alloc += d.alloc; + n.files = add_u32(n.files, d.files); + n.dirs = add_u32(n.dirs, d.dirs); + n.errors = add_u32(n.errors, d.errors); + let (is_dir, dir, parent) = (n.kind == Kind::Dir, n.dir, n.parent); + if is_dir { + let cs = self.dirs.at_mut(dir); + for (c, v) in cs.iter_mut().zip(d.cat) { + *c += v; + } + } + id = parent; + } + } + + /// Iterates the children of id (in reverse insertion order). + pub fn children(&self, id: NodeId) -> Children<'_> { + Children { t: self, cur: self.node(id).first_child } + } + + pub fn child_count(&self, id: NodeId) -> usize { + self.children(id).count() + } + + /// Reconstructs the full path of id as raw bytes. + pub fn path_bytes(&self, mut id: NodeId) -> Vec { + let mut parts: Vec<&[u8]> = Vec::new(); + while id != NO_NODE { + let n = self.node(id); + parts.push(&n.name); + id = n.parent; + } + let mut out = Vec::with_capacity(parts.iter().map(|p| p.len() + 1).sum()); + for p in parts.iter().rev() { + if !out.is_empty() && out.last() != Some(&b'/') { + out.push(b'/'); + } + out.extend_from_slice(p); + } + out + } + + /// Reconstructs the full path of id. + pub fn path(&self, id: NodeId) -> PathBuf { + PathBuf::from(OsStr::from_bytes(&self.path_bytes(id))) + } + + /// The full path as (lossy) UTF-8, unsanitised. + pub fn path_string(&self, id: NodeId) -> String { + String::from_utf8_lossy(&self.path_bytes(id)).into_owned() + } + + /// The chain root..id inclusive. + pub fn ancestors(&self, mut id: NodeId) -> Vec { + let mut out = Vec::new(); + while id != NO_NODE { + out.push(id); + id = self.node(id).parent; + } + out.reverse(); + out + } + + /// Reports whether a is id or an ancestor of id. + pub fn is_ancestor(&self, a: NodeId, mut id: NodeId) -> bool { + while id != NO_NODE { + if id == a { + return true; + } + id = self.node(id).parent; + } + false + } + + /// Number of ancestors of id. + pub fn depth(&self, id: NodeId) -> usize { + let mut d = 0; + let mut id = self.node(id).parent; + while id != NO_NODE { + d += 1; + id = self.node(id).parent; + } + d + } + + /// Rebuilds all aggregates bottom-up. Relies on a parent's ID being + /// smaller than its children's, which both the scanner and the snapshot + /// loader guarantee. Used after loading a snapshot. + pub fn recompute(&mut self) { + for i in 0..self.len() as NodeId { + let n = self.nodes.at_mut(i); + n.tot_size = 0; + n.tot_alloc = 0; + n.files = 0; + n.dirs = 0; + n.errors = (n.flags & FLAG_ERROR != 0) as u32; + if n.flags & FLAG_HARDLINK_DUP == 0 { + n.tot_size = n.size; + n.tot_alloc = n.alloc; + } + if n.kind == Kind::Dir { + let d = n.dir; + *self.dirs.at_mut(d) = [0; NUM_CATEGORIES]; + } + } + // Sums saturate: snapshot data is untrusted and must not wrap to + // negative totals. + for i in (1..self.len() as NodeId).rev() { + let n = self.nodes.at(i).clone_shallow(); + let p = self.nodes.at_mut(n.parent); + p.tot_size = sat_add(p.tot_size, n.tot_size); + p.tot_alloc = sat_add(p.tot_alloc, n.tot_alloc); + p.errors = add_u32(p.errors, n.errors as i64); + let pdir = p.dir; + if n.kind == Kind::Dir { + p.files = add_u32(p.files, n.files as i64); + p.dirs = add_u32(p.dirs, n.dirs as i64 + 1); + let child = *self.dirs.at(n.dir); + let cs = self.dirs.at_mut(pdir); + for (c, v) in cs.iter_mut().zip(child) { + *c = sat_add(*c, v); + } + } else { + p.files = add_u32(p.files, 1); + if n.flags & FLAG_HARDLINK_DUP == 0 { + let cs = self.dirs.at_mut(pdir); + cs[n.cat as usize] = sat_add(cs[n.cat as usize], n.alloc); + } + } + } + } +} + +/// The numeric fields recompute needs, without cloning the name. +struct Shallow { + parent: NodeId, + dir: u32, + tot_size: i64, + tot_alloc: i64, + alloc: i64, + files: u32, + dirs: u32, + errors: u32, + flags: Flags, + kind: Kind, + cat: Category, +} + +impl Node { + fn clone_shallow(&self) -> Shallow { + Shallow { + parent: self.parent, dir: self.dir, tot_size: self.tot_size, tot_alloc: self.tot_alloc, + alloc: self.alloc, files: self.files, dirs: self.dirs, errors: self.errors, + flags: self.flags, kind: self.kind, cat: self.cat, + } + } +} + +pub struct Children<'a> { + t: &'a Tree, + cur: NodeId, +} + +impl<'a> Iterator for Children<'a> { + type Item = (NodeId, &'a Node); + fn next(&mut self) -> Option { + if self.cur == NO_NODE { + return None; + } + let id = self.cur; + let n = self.t.node(id); + self.cur = n.next_sibling; + Some((id, n)) + } +} + +/// Adds non-negative i64s, saturating at the maximum. +pub fn sat_add(a: i64, b: i64) -> i64 { + if b > 0 && a > i64::MAX - b { + return i64::MAX; + } + a + b +} + +fn add_u32(a: u32, d: i64) -> u32 { + (a as i64 + d).clamp(0, u32::MAX as i64) as u32 +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn topk_and_recompute() { + let mut tr = Tree::new(b"/r", Kind::Dir); + let d = tr.add(0, b"d", Kind::Dir); + for i in 0..100 { + let id = tr.add(d, b"f", Kind::File); + let n = tr.node_mut(id); + n.size = i; + n.alloc = i; + } + tr.recompute(); + assert_eq!(tr.node(0).tot_size, 4950); + assert_eq!((tr.node(0).files, tr.node(0).dirs), (100, 1)); + let top = tr.top(0, 3, |n| n.kind == Kind::File, |n| n.size); + assert_eq!(top.len(), 3); + assert_eq!(tr.node(top[0]).size, 99); + assert_eq!(tr.node(top[2]).size, 97); + assert_eq!(tr.path_string(top[0]), "/r/d/f"); + } + + #[test] + fn ext_and_category() { + for (input, want) in [("a.QCOW2", "qcow2"), (".bashrc", ""), ("noext", ""), ("x.tar.gz", "gz"), ("trail.", "")] { + assert_eq!(ext(input.as_bytes()), want, "{input}"); + } + assert_eq!(category_for("qcow2", false, false), Category::DiskImage); + assert_eq!(category_for("bin", true, false), Category::Executable); + assert_eq!(category_for("", true, false), Category::Cache); + } +} diff --git a/rust/src/inventory/query.rs b/rust/src/inventory/query.rs new file mode 100644 index 0000000..cb02600 --- /dev/null +++ b/rust/src/inventory/query.rs @@ -0,0 +1,141 @@ +use super::*; +use std::cmp::Reverse; +use std::collections::BinaryHeap; + +/// Which size drives layout and sorting. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum SizeMode { + #[default] + Allocated, // disk usage + Logical, // apparent size +} + +impl SizeMode { + pub fn name(self) -> &'static str { + match self { + SizeMode::Logical => "logical", + SizeMode::Allocated => "allocated", + } + } +} + +impl Node { + /// The aggregate size in the given mode. + #[inline] + pub fn total(&self, m: SizeMode) -> i64 { + match m { + SizeMode::Logical => self.tot_size, + SizeMode::Allocated => self.tot_alloc, + } + } + /// The node's own size in the given mode. + #[inline] + pub fn own(&self, m: SizeMode) -> i64 { + match m { + SizeMode::Logical => self.size, + SizeMode::Allocated => self.alloc, + } + } +} + +/// Keeps the k entries with the largest keys (ties: smaller id wins); O(n log k). +pub struct TopK { + k: usize, + // Min-heap on (key, Reverse(id)): the root is the weakest entry. + h: BinaryHeap)>>, +} + +impl TopK { + pub fn new(k: usize) -> TopK { + TopK { k, h: BinaryHeap::with_capacity(k.min(1 << 16)) } + } + + pub fn offer(&mut self, id: NodeId, key: i64) { + if self.k == 0 { + return; + } + let e = Reverse((key, Reverse(id))); + if self.h.len() < self.k { + self.h.push(e); + } else if let Some(mut top) = self.h.peek_mut() { + if e < *top { + *top = e; + } + } + } + + /// The collected IDs, largest key first (ties by ID). + pub fn sorted(self) -> Vec { + // into_sorted_vec is ascending in Reverse order = descending key. + self.h.into_sorted_vec().into_iter().map(|Reverse((_, Reverse(id)))| id).collect() + } +} + +impl Tree { + /// Visits id and all its descendants depth-first (iteratively). fn + /// returns false to skip a node's children. + pub fn walk(&self, id: NodeId, mut f: impl FnMut(NodeId, &Node) -> bool) { + let mut stack = vec![id]; + while let Some(cur) = stack.pop() { + let n = self.node(cur); + if !f(cur, n) { + continue; + } + stack.extend(self.children(cur).map(|(c, _)| c)); + } + } + + /// The k nodes beneath (and including) root with the largest key, + /// considering only nodes accepted by keep. + pub fn top(&self, root: NodeId, k: usize, keep: impl Fn(&Node) -> bool, key: impl Fn(&Node) -> i64) -> Vec { + let mut tk = TopK::new(k); + self.walk(root, |id, n| { + if keep(n) { + tk.offer(id, key(n)); + } + true + }); + tk.sorted() + } + + /// The children of id ordered by size descending (ties by ID). + pub fn sorted_children(&self, id: NodeId, m: SizeMode) -> Vec { + let mut out: Vec<(i64, NodeId)> = self.children(id).map(|(c, n)| (n.total(m), c)).collect(); + out.sort_unstable_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1))); + out.into_iter().map(|(_, c)| c).collect() + } + + /// Aggregates file statistics by extension beneath root. + pub fn ext_stats(&self, root: NodeId) -> Vec { + let mut stats = vec![ExtStat::default(); self.ext_count()]; + self.walk(root, |_, n| { + if n.kind == Kind::File && n.flags & FLAG_HARDLINK_DUP == 0 { + let s = &mut stats[n.ext as usize]; + s.count += 1; + s.size += n.size; + s.alloc += n.alloc; + } + true + }); + stats + .into_iter() + .enumerate() + .filter(|(_, s)| s.count > 0) + .map(|(i, mut s)| { + s.ext = self.exts[i].clone(); + s.cat = category_for(&s.ext, false, false); + s + }) + .collect() + } +} + +/// Summarises one extension. +#[derive(Clone, Default, Debug)] +pub struct ExtStat { + pub ext: String, + pub cat: Category, + pub count: i64, + pub size: i64, // logical + pub alloc: i64, +} diff --git a/rust/src/inventory/stats.rs b/rust/src/inventory/stats.rs new file mode 100644 index 0000000..a5d0d2d --- /dev/null +++ b/rust/src/inventory/stats.rs @@ -0,0 +1,124 @@ +use super::NodeId; +use std::io; +use std::time::{Duration, SystemTime}; + +/// Classifies scan errors for reporting. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +#[repr(u8)] +pub enum ErrKind { + Permission, + Vanished, + Io, + Loop, + #[default] + Other, +} + +pub const NUM_ERR_KINDS: usize = 5; + +impl ErrKind { + pub fn from_u8(v: u8) -> ErrKind { + [ErrKind::Permission, ErrKind::Vanished, ErrKind::Io, ErrKind::Loop, ErrKind::Other] + .get(v as usize) + .copied() + .unwrap_or(ErrKind::Other) + } + pub fn name(self) -> &'static str { + ["Permission denied", "Vanished during scan", "I/O error", "Symlink loop", "Other error"][self as usize] + } +} + +/// Maps an OS error to an ErrKind. +pub fn classify(err: &io::Error) -> ErrKind { + match err.raw_os_error() { + Some(libc::EACCES) | Some(libc::EPERM) => ErrKind::Permission, + Some(libc::ENOENT) => ErrKind::Vanished, + Some(libc::ELOOP) => ErrKind::Loop, + Some(libc::EIO) => ErrKind::Io, + _ => match err.kind() { + io::ErrorKind::PermissionDenied => ErrKind::Permission, + io::ErrorKind::NotFound => ErrKind::Vanished, + _ => ErrKind::Other, + }, + } +} + +/// One recorded scan error. +#[derive(Clone, Debug, Default)] +pub struct ErrorRecord { + pub node: NodeId, // node the error is attached to (the directory for listing failures) + pub name: Vec, // entry name when no node was created (e.g. vanished) + pub kind: ErrKind, + pub msg: String, +} + +/// Caps the detailed error list; counts are always exact. +pub const MAX_ERROR_RECORDS: usize = 100_000; + +/// Describes a scan as a whole. +#[derive(Clone, Debug)] +pub struct Stats { + pub root: Vec, + pub start: SystemTime, + pub end: Option, + pub complete: bool, // scan finished (possibly with errors) + pub cancelled: bool, + + pub files: i64, + pub dirs: i64, + pub symlinks: i64, + pub others: i64, + + pub err_counts: [i64; NUM_ERR_KINDS], + pub errors: Vec, + pub excluded: i64, + pub skipped_mounts: i64, + pub virtual_skipped: i64, + pub loops_skipped: i64, + pub broken_links: i64, + pub hardlink_dups: i64, + pub unscanned: i64, // directories never read because the scan was cancelled + + pub excludes: Vec, + pub one_file_system: bool, + pub follow: String, + pub workers: usize, + pub from_snapshot: String, +} + +impl Default for Stats { + fn default() -> Self { + Stats { + root: Vec::new(), start: SystemTime::now(), end: None, complete: false, cancelled: false, + files: 0, dirs: 0, symlinks: 0, others: 0, err_counts: [0; NUM_ERR_KINDS], errors: Vec::new(), + excluded: 0, skipped_mounts: 0, virtual_skipped: 0, loops_skipped: 0, broken_links: 0, + hardlink_dups: 0, unscanned: 0, excludes: Vec::new(), one_file_system: false, + follow: String::new(), workers: 0, from_snapshot: String::new(), + } + } +} + +impl Stats { + pub fn total_errors(&self) -> i64 { + self.err_counts.iter().sum() + } + + /// Records an error (the detailed list is capped). + pub fn add_error(&mut self, r: ErrorRecord) { + self.err_counts[r.kind as usize] += 1; + if self.errors.len() < MAX_ERROR_RECORDS { + self.errors.push(r); + } + } + + /// The scan duration so far (or total once finished). + pub fn elapsed(&self) -> Duration { + let end = self.end.unwrap_or_else(SystemTime::now); + end.duration_since(self.start).unwrap_or_default() + } + + /// Whether totals may under-count real usage. + pub fn incomplete(&self) -> bool { + !self.complete || self.cancelled || self.total_errors() > 0 || self.unscanned > 0 + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs new file mode 100644 index 0000000..ae752ec --- /dev/null +++ b/rust/src/lib.rs @@ -0,0 +1,13 @@ +//! mapsize: interactive terminal disk usage analyser (Rust port). + +pub mod brand; +pub mod cancel; +pub mod config; +pub mod export; +pub mod inventory; +pub mod platform; +pub mod scan; +pub mod textutil; + +/// Boxed error used across mode boundaries. +pub type Error = Box; diff --git a/rust/src/platform.rs b/rust/src/platform.rs new file mode 100644 index 0000000..9009672 --- /dev/null +++ b/rust/src/platform.rs @@ -0,0 +1,707 @@ +//! OS-specific filesystem metadata and file operations (Unix; Linux first). +//! Nothing else in the program calls libc for filesystem work. + +use std::collections::HashMap; +use std::ffi::{CStr, CString, OsStr}; +use std::fs::File; +use std::io::{self, Write}; +use std::os::fd::{AsRawFd, FromRawFd, OwnedFd, RawFd}; +use std::os::unix::ffi::OsStrExt; +use std::os::unix::fs::{MetadataExt, OpenOptionsExt}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Mutex; + +/// File mode bits in Go's io/fs.FileMode layout. Kept so snapshots stay +/// byte-compatible with the Go implementation. +pub mod mode { + pub const DIR: u32 = 1 << 31; + pub const APPEND: u32 = 1 << 30; + pub const EXCLUSIVE: u32 = 1 << 29; + pub const TEMPORARY: u32 = 1 << 28; + pub const SYMLINK: u32 = 1 << 27; + pub const DEVICE: u32 = 1 << 26; + pub const NAMED_PIPE: u32 = 1 << 25; + pub const SOCKET: u32 = 1 << 24; + pub const SETUID: u32 = 1 << 23; + pub const SETGID: u32 = 1 << 22; + pub const CHAR_DEVICE: u32 = 1 << 21; + pub const STICKY: u32 = 1 << 20; + pub const IRREGULAR: u32 = 1 << 19; + pub const TYPE: u32 = DIR | SYMLINK | NAMED_PIPE | SOCKET | DEVICE | CHAR_DEVICE | IRREGULAR; + pub const PERM: u32 = 0o777; + + pub fn is_dir(m: u32) -> bool { + m & DIR != 0 + } + pub fn is_regular(m: u32) -> bool { + m & TYPE == 0 + } + pub fn is_symlink(m: u32) -> bool { + m & SYMLINK != 0 + } + + /// Formats like Go's FileMode.String, e.g. "drwxr-xr-x". + pub fn string(m: u32) -> String { + const STR: &[u8] = b"dalTLDpSugct?"; + let mut out = String::with_capacity(12); + for (i, &c) in STR.iter().enumerate() { + if m & (1 << (31 - i)) != 0 { + out.push(c as char); + } + } + if out.is_empty() { + out.push('-'); + } + for (i, c) in "rwxrwxrwx".chars().enumerate() { + out.push(if m & (1 << (8 - i)) != 0 { c } else { '-' }); + } + out + } +} + +/// Converts a Unix st_mode to the Go FileMode layout. +pub fn mode_from_unix(m: u32) -> u32 { + let mut out = m & 0o777; + out |= match m & libc::S_IFMT { + libc::S_IFDIR => mode::DIR, + libc::S_IFLNK => mode::SYMLINK, + libc::S_IFIFO => mode::NAMED_PIPE, + libc::S_IFSOCK => mode::SOCKET, + libc::S_IFCHR => mode::DEVICE | mode::CHAR_DEVICE, + libc::S_IFBLK => mode::DEVICE, + _ => 0, + }; + if m & libc::S_ISUID != 0 { + out |= mode::SETUID; + } + if m & libc::S_ISGID != 0 { + out |= mode::SETGID; + } + if m & libc::S_ISVTX != 0 { + out |= mode::STICKY; + } + out +} + +/// Metadata of one directory entry. +#[derive(Debug, Default)] +pub struct Meta { + pub name: Vec, + pub mode: u32, // Go FileMode layout + pub size: i64, + pub alloc: i64, + pub alloc_known: bool, + pub mtime: i64, // unix nanoseconds + pub btime: i64, // birth time, 0 if unknown + pub uid: u32, + pub gid: u32, + pub nlink: u32, + pub dev: u64, + pub ino: u64, + pub has_ino: bool, + pub err: Option, // stat failed; only name is valid +} + +fn cstr(b: &[u8]) -> io::Result { + CString::new(b).map_err(|_| io::Error::from_raw_os_error(libc::EINVAL)) +} + +fn cvt(r: libc::c_int) -> io::Result { + if r < 0 { + Err(io::Error::last_os_error()) + } else { + Ok(r) + } +} + +fn open_dir(path: &[u8]) -> io::Result { + let c = cstr(path)?; + let fd = cvt(unsafe { libc::open(c.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY | libc::O_CLOEXEC) })?; + Ok(unsafe { OwnedFd::from_raw_fd(fd) }) +} + +#[cfg(target_os = "linux")] +static NO_STATX: AtomicBool = AtomicBool::new(false); + +#[cfg(target_os = "linux")] +fn stat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + if !NO_STATX.load(Ordering::Relaxed) { + let mut st: libc::statx = unsafe { std::mem::zeroed() }; + let mask = libc::STATX_BASIC_STATS | libc::STATX_BTIME; + let r = unsafe { libc::statx(dirfd, name.as_ptr(), flags | libc::AT_STATX_DONT_SYNC, mask, &mut st) }; + if r == 0 { + let mut m = Meta { + mode: mode_from_unix(st.stx_mode as u32), + size: st.stx_size as i64, + alloc: st.stx_blocks as i64 * 512, + alloc_known: true, + mtime: st.stx_mtime.tv_sec * 1_000_000_000 + st.stx_mtime.tv_nsec as i64, + uid: st.stx_uid, + gid: st.stx_gid, + nlink: st.stx_nlink, + dev: libc::makedev(st.stx_dev_major, st.stx_dev_minor), + ino: st.stx_ino, + has_ino: true, + ..Default::default() + }; + if st.stx_mask & libc::STATX_BTIME != 0 { + m.btime = st.stx_btime.tv_sec * 1_000_000_000 + st.stx_btime.tv_nsec as i64; + } + return Ok(m); + } + let err = io::Error::last_os_error(); + if err.raw_os_error() != Some(libc::ENOSYS) { + return Err(err); + } + NO_STATX.store(true, Ordering::Relaxed); + } + fstat_at(dirfd, name, flags) +} + +#[cfg(not(target_os = "linux"))] +fn stat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + fstat_at(dirfd, name, flags) +} + +fn fstat_at(dirfd: RawFd, name: &CStr, flags: libc::c_int) -> io::Result { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + cvt(unsafe { libc::fstatat(dirfd, name.as_ptr(), &mut st, flags) })?; + Ok(from_stat(&st)) +} + +#[allow(clippy::unnecessary_cast)] +fn from_stat(st: &libc::stat) -> Meta { + Meta { + mode: mode_from_unix(st.st_mode as u32), + size: st.st_size as i64, + alloc: st.st_blocks as i64 * 512, + alloc_known: true, + mtime: st.st_mtime as i64 * 1_000_000_000 + st.st_mtime_nsec as i64, + uid: st.st_uid, + gid: st.st_gid, + nlink: st.st_nlink as u32, + dev: st.st_dev as u64, + ino: st.st_ino as u64, + has_ino: true, + ..Default::default() + } +} + +/// An open directory stream (readdir over the fd we also stat relative to). +struct DirStream { + dir: *mut libc::DIR, +} + +impl Drop for DirStream { + fn drop(&mut self) { + unsafe { libc::closedir(self.dir) }; + } +} + +/// Streams the entries of a directory in chunks of up to `chunk`, stat-ing +/// each entry relative to the directory fd (no repeated path resolution). +/// Returning an error from f stops the read and returns it. +pub fn read_dir>(path: &[u8], chunk: usize, mut f: impl FnMut(Vec) -> Result<(), E>) -> Result<(), E> { + let fd = open_dir(path)?; + let raw = fd.as_raw_fd(); + let dir = unsafe { libc::fdopendir(raw) }; + if dir.is_null() { + return Err(io::Error::last_os_error().into()); + } + std::mem::forget(fd); // owned by the DIR stream now + let ds = DirStream { dir }; + let mut out = Vec::with_capacity(chunk); + loop { + unsafe { *libc::__errno_location() = 0 }; + let ent = unsafe { libc::readdir(ds.dir) }; + if ent.is_null() { + let e = io::Error::last_os_error(); + if e.raw_os_error().unwrap_or(0) != 0 { + return Err(e.into()); + } + break; + } + let name = unsafe { CStr::from_ptr((*ent).d_name.as_ptr()) }; + let nb = name.to_bytes(); + if nb == b"." || nb == b".." { + continue; + } + let mut m = stat_at(raw, name, libc::AT_SYMLINK_NOFOLLOW).unwrap_or_else(|e| Meta { err: Some(e), ..Default::default() }); + m.name = nb.to_vec(); + out.push(m); + if out.len() >= chunk { + f(std::mem::replace(&mut out, Vec::with_capacity(chunk)))?; + } + } + if !out.is_empty() { + f(out)?; + } + Ok(()) +} + +fn stat_path(path: &[u8], follow: bool) -> io::Result { + let c = cstr(path)?; + let flags = if follow { 0 } else { libc::AT_SYMLINK_NOFOLLOW }; + let mut m = stat_at(libc::AT_FDCWD, &c, flags)?; + m.name = base_name(path).to_vec(); + Ok(m) +} + +/// Metadata for path without following a final symlink. +pub fn lstat(path: &[u8]) -> io::Result { + stat_path(path, false) +} + +/// Metadata for path, following symlinks. +pub fn stat(path: &[u8]) -> io::Result { + stat_path(path, true) +} + +/// Joins a directory and an entry name without cleaning. +pub fn join_path(dir: &[u8], name: &[u8]) -> Vec { + let mut p = Vec::with_capacity(dir.len() + name.len() + 1); + p.extend_from_slice(dir); + if dir.last() != Some(&b'/') { + p.push(b'/'); + } + p.extend_from_slice(name); + p +} + +fn base_name(path: &[u8]) -> &[u8] { + let mut end = path.len(); + while end > 1 && path[end - 1] == b'/' { + end -= 1; + } + let p = &path[..end]; + match p.iter().rposition(|&b| b == b'/') { + Some(i) if p.len() > 1 => &p[i + 1..], + _ => p, + } +} + +static NAMES: Mutex, HashMap)>> = Mutex::new(None); + +fn lookup_name(group: bool, id: u32) -> String { + let mut g = NAMES.lock().unwrap(); + let (users, groups) = g.get_or_insert_with(Default::default); + let cache = if group { groups } else { users }; + if let Some(n) = cache.get(&id) { + return n.clone(); + } + let mut buf = vec![0u8; 16384]; + let name = unsafe { + if group { + let mut gr: libc::group = std::mem::zeroed(); + let mut res = std::ptr::null_mut(); + libc::getgrgid_r(id, &mut gr, buf.as_mut_ptr().cast(), buf.len(), &mut res); + (!res.is_null()).then(|| CStr::from_ptr(gr.gr_name).to_string_lossy().into_owned()) + } else { + let mut pw: libc::passwd = std::mem::zeroed(); + let mut res = std::ptr::null_mut(); + libc::getpwuid_r(id, &mut pw, buf.as_mut_ptr().cast(), buf.len(), &mut res); + (!res.is_null()).then(|| CStr::from_ptr(pw.pw_name).to_string_lossy().into_owned()) + } + } + .unwrap_or_else(|| id.to_string()); + cache.insert(id, name.clone()); + name +} + +/// Resolves a uid to a name (cached; falls back to the number). +pub fn user_name(uid: u32) -> String { + lookup_name(false, uid) +} + +/// Resolves a gid to a name (cached; falls back to the number). +pub fn group_name(gid: u32) -> String { + lookup_name(true, gid) +} + +/// Linux pseudo-filesystems whose contents are not disk usage. +#[cfg(target_os = "linux")] +const VIRTUAL_FS: [i64; 19] = [ + 0x9fa0, // proc + 0x62656572, // sysfs + 0x1cd1, // devpts + 0x27e0eb, // cgroup + 0x63677270, // cgroup2 + 0x64626720, // debugfs + 0x74726163, // tracefs + 0x73636673, // securityfs + 0x6165676c, // pstore + 0xcafe4a11, // bpf + 0x62656570, // configfs + 0x65735543, // fusectl + 0x19800202, // mqueue + 0xde5e81e4, // efivarfs + 0x42494e4d, // binfmt_misc + 0xf97cff8c, // selinuxfs + 0x958458f6, // hugetlbfs + 0x6e736673, // nsfs + 0x50495045, // pipefs +]; + +/// Whether path is the root of a pseudo filesystem (proc, sysfs, cgroup, …) +/// that should not be scanned as disk usage. +#[cfg(target_os = "linux")] +pub fn is_virtual_fs(path: &[u8]) -> bool { + let Ok(c) = cstr(path) else { return false }; + let mut st: libc::statfs = unsafe { std::mem::zeroed() }; + if unsafe { libc::statfs(c.as_ptr(), &mut st) } != 0 { + return false; + } + VIRTUAL_FS.contains(&(st.f_type as i64)) +} + +#[cfg(not(target_os = "linux"))] +pub fn is_virtual_fs(_: &[u8]) -> bool { + false +} + +/// What the scan recorded for an item about to be trashed or deleted. The +/// operation refuses an item whose owner or file type no longer matches. +#[derive(Clone, Copy, Debug)] +pub struct Expect { + pub uid: u32, + pub mode: u32, // only the type bits are compared +} + +const CHANGED: &str = "item changed since the scan (owner or type differs); rescan first"; + +fn other(msg: impl Into) -> io::Error { + io::Error::other(msg.into()) +} + +/// Guards helper invocations: an absolute path can never be parsed as an +/// option by the helper program. +fn check_abs(path: &Path) -> io::Result<()> { + if !path.is_absolute() { + return Err(other("refusing relative path")); + } + std::fs::symlink_metadata(path).map(|_| ()) +} + +/// Opens a file whose content is about to be read. It does not follow a +/// final symlink, and O_NONBLOCK keeps a FIFO swapped in since the scan from +/// blocking open(2) forever; callers still check the file type. +pub fn open_content(path: &Path) -> io::Result { + std::fs::OpenOptions::new().read(true).custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK).open(path) +} + +/// Runs a helper program with an argument vector (never a shell), detached +/// from the terminal so it cannot disturb the TUI. +fn start_detached(name: &str, arg: &Path) -> io::Result<()> { + use std::process::{Command, Stdio}; + let mut child = Command::new(name).arg(arg).stdin(Stdio::null()).stdout(Stdio::null()).stderr(Stdio::null()).spawn()?; + std::thread::spawn(move || child.wait()); + Ok(()) +} + +/// The user-facing name of the trash. +pub const TRASH_NAME: &str = "trash"; + +/// Opens the item's directory in the desktop file manager. +pub fn reveal(path: &Path) -> io::Result<()> { + check_abs(path)?; + let md = std::fs::symlink_metadata(path)?; + let target = if md.is_dir() { path } else { path.parent().unwrap_or(path) }; + let opener = if cfg!(target_os = "macos") { "open" } else { "xdg-open" }; + start_detached(opener, target) +} + +/// Whether there is no graphical session (no X or Wayland display), so there +/// is no file manager to open and no desktop trash anyone will empty. +pub fn headless() -> bool { + if cfg!(target_os = "macos") { + return false; + } + std::env::var_os("DISPLAY").is_none_or(|v| v.is_empty()) && std::env::var_os("WAYLAND_DISPLAY").is_none_or(|v| v.is_empty()) +} + +fn split_parent(path: &Path) -> io::Result<(&Path, &OsStr)> { + match (path.parent(), path.file_name()) { + (Some(d), Some(b)) => Ok((d, b)), + _ => Err(other("refusing to operate on a filesystem root")), + } +} + +fn check_expect(dirfd: RawFd, base: &CStr, want: Expect) -> io::Result { + let mut st: libc::stat = unsafe { std::mem::zeroed() }; + cvt(unsafe { libc::fstatat(dirfd, base.as_ptr(), &mut st, libc::AT_SYMLINK_NOFOLLOW) })?; + if st.st_uid != want.uid || mode_from_unix(st.st_mode as u32) & mode::TYPE != want.mode & mode::TYPE { + return Err(other(CHANGED)); + } + Ok(st) +} + +/// Permanently removes path (recursively for a directory) if it still +/// matches want. The check and the removal both go through descriptors +/// opened from the parent directory without following symlinks, so a +/// symlink swapped in since the scan cannot redirect the removal outside it. +pub fn delete(path: &Path, want: Expect) -> io::Result<()> { + check_abs(path)?; + let (dir, base) = split_parent(path)?; + let dfd = open_dir(dir.as_os_str().as_bytes())?; + let cbase = cstr(base.as_bytes())?; + let st = check_expect(dfd.as_raw_fd(), &cbase, want)?; + remove_at(dfd.as_raw_fd(), &cbase, st.st_mode & libc::S_IFMT == libc::S_IFDIR) +} + +fn remove_at(dirfd: RawFd, name: &CStr, is_dir: bool) -> io::Result<()> { + if !is_dir { + return cvt(unsafe { libc::unlinkat(dirfd, name.as_ptr(), 0) }).map(|_| ()); + } + let fd = cvt(unsafe { + libc::openat(dirfd, name.as_ptr(), libc::O_RDONLY | libc::O_DIRECTORY | libc::O_NOFOLLOW | libc::O_CLOEXEC) + })?; + let sub = unsafe { OwnedFd::from_raw_fd(fd) }; + // Collect names first: removing while iterating a DIR stream is unspecified. + let mut names = Vec::new(); + let dup = cvt(unsafe { libc::dup(sub.as_raw_fd()) })?; + let dir = unsafe { libc::fdopendir(dup) }; + if dir.is_null() { + unsafe { libc::close(dup) }; + return Err(io::Error::last_os_error()); + } + let ds = DirStream { dir }; + loop { + let ent = unsafe { libc::readdir(ds.dir) }; + if ent.is_null() { + break; + } + let n = unsafe { CStr::from_ptr((*ent).d_name.as_ptr()) }; + if n.to_bytes() != b"." && n.to_bytes() != b".." { + let t = unsafe { (*ent).d_type }; + let is_dir = match t { + libc::DT_UNKNOWN => fstat_at(sub.as_raw_fd(), n, libc::AT_SYMLINK_NOFOLLOW).is_ok_and(|m| mode::is_dir(m.mode)), + t => t == libc::DT_DIR, + }; + names.push((n.to_owned(), is_dir)); + } + } + drop(ds); + for (n, d) in names { + remove_at(sub.as_raw_fd(), &n, d)?; + } + cvt(unsafe { libc::unlinkat(dirfd, name.as_ptr(), libc::AT_REMOVEDIR) }).map(|_| ()) +} + +/// Moves path to dst if it still matches want. The parent directory is +/// opened once and both the check and the rename are relative to that +/// descriptor, so a parent swapped for a symlink since the scan (or between +/// check and rename) cannot redirect the move to another user's file. +fn rename_checked(path: &Path, dst: &Path, want: Expect) -> io::Result<()> { + let (dir, base) = split_parent(path)?; + let dfd = open_dir(dir.as_os_str().as_bytes())?; + let cbase = cstr(base.as_bytes())?; + check_expect(dfd.as_raw_fd(), &cbase, want)?; + let cdst = cstr(dst.as_os_str().as_bytes())?; + cvt(unsafe { libc::renameat(dfd.as_raw_fd(), cbase.as_ptr(), libc::AT_FDCWD, cdst.as_ptr()) }).map(|_| ()) +} + +/// Creates dir (mode 0700) if missing; an existing entry must be a real +/// directory (not a symlink) owned by us. With strict (trash on a shared +/// filesystem) it must also be inaccessible to others, as the freedesktop.org +/// spec requires. +fn ensure_private_dir(dir: &Path, strict: bool) -> io::Result<()> { + use std::os::unix::fs::DirBuilderExt; + if let Err(e) = std::fs::DirBuilder::new().mode(0o700).create(dir) { + if e.kind() != io::ErrorKind::AlreadyExists { + return Err(e); + } + } + let md = std::fs::symlink_metadata(dir)?; + if !md.is_dir() { + return Err(other("not a directory (symlink?)")); + } + if md.uid() != unsafe { libc::getuid() } { + return Err(other("owned by another user")); + } + if strict && md.mode() & 0o077 != 0 { + return Err(other("accessible to other users")); + } + Ok(()) +} + +/// The top directory of the filesystem containing path. +fn mount_top(path: &Path, dev: u64) -> PathBuf { + let mut cur = path.parent().unwrap_or(path).to_path_buf(); + loop { + let Some(parent) = cur.parent() else { return cur }; + match std::fs::metadata(parent) { + Ok(m) if m.dev() == dev => cur = parent.to_path_buf(), + _ => return cur, + } + } +} + +fn percent_encode_path(p: &[u8]) -> String { + let mut s = String::with_capacity(p.len()); + for &b in p { + if b.is_ascii_alphanumeric() || b"-._~/!$&'()*+,;=:@".contains(&b) { + s.push(b as char); + } else { + s.push_str(&format!("%{b:02X}")); + } + } + s +} + +fn local_timestamp() -> String { + let mut t: libc::time_t = unsafe { libc::time(std::ptr::null_mut()) }; + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + unsafe { libc::localtime_r(&mut t, &mut tm) }; + format!("{:04}-{:02}-{:02}T{:02}:{:02}:{:02}", tm.tm_year + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min, tm.tm_sec) +} + +/// Moves path to the freedesktop.org trash: the home trash when on the same +/// filesystem, otherwise $topdir/.Trash-$uid on the item's filesystem. It +/// never copies across filesystems and never deletes. +pub fn trash(path: &Path, want: Expect) -> io::Result<()> { + check_abs(path)?; + let dev = std::fs::symlink_metadata(path)?.dev(); + let data_home = match std::env::var_os("XDG_DATA_HOME").filter(|v| !v.is_empty()) { + Some(d) => PathBuf::from(d), + None => { + let home = std::env::var_os("HOME").filter(|v| !v.is_empty()).ok_or_else(|| other("$HOME is not defined"))?; + PathBuf::from(home).join(".local/share") + } + }; + let home = data_home.join("Trash"); + let (trash, info_path): (PathBuf, Vec) = match std::fs::metadata(&data_home) { + Ok(m) if m.dev() == dev => (home.clone(), path.as_os_str().as_bytes().to_vec()), + _ => { + let top = mount_top(path, dev); + let rel = path.strip_prefix(&top).map_err(|e| other(e.to_string()))?; + (top.join(format!(".Trash-{}", unsafe { libc::getuid() })), rel.as_os_str().as_bytes().to_vec()) + } + }; + let shared = trash != home; + if !shared { + use std::os::unix::fs::DirBuilderExt; + std::fs::DirBuilder::new().recursive(true).mode(0o700).create(&trash)?; + } else if let Err(e) = ensure_private_dir(&trash, true) { + // On a shared filesystem another user could plant a symlink or a + // world-writable directory here; refuse rather than follow it. + return Err(other(format!("trash directory {} is unsafe: {e}", trash.display()))); + } + let (files, info) = (trash.join("files"), trash.join("info")); + for d in [&files, &info] { + ensure_private_dir(d, shared).map_err(|e| other(format!("trash directory {} is unsafe: {e}", d.display())))?; + } + let base = path.file_name().ok_or_else(|| other("no file name"))?; + for i in 1..10000 { + let mut name = base.to_os_string(); + if i > 1 { + name.push(format!(".{i}")); + } + let dst = files.join(&name); + if std::fs::symlink_metadata(&dst).is_ok() { + continue; // never overwrite an earlier trashed item (even without .trashinfo) + } + let mut info_name = name.clone(); + info_name.push(".trashinfo"); + let ip = info.join(info_name); + let mut f = match std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(&ip) { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e), + }; + let w = write!(f, "[Trash Info]\nPath={}\nDeletionDate={}\n", percent_encode_path(&info_path), local_timestamp()).and_then(|_| f.sync_all()); + drop(f); + if let Err(e) = w.and_then(|_| rename_checked(path, &dst, want)) { + let _ = std::fs::remove_file(&ip); + return Err(e); + } + return Ok(()); + } + Err(other("trash: too many items with the same name")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn mode_string_matches_go() { + assert_eq!(mode::string(mode::DIR | 0o755), "drwxr-xr-x"); + assert_eq!(mode::string(0o644), "-rw-r--r--"); + assert_eq!(mode::string(mode::SYMLINK | 0o777), "Lrwxrwxrwx"); + } + + #[test] + fn delete_refuses_changed_type() { + let d = std::env::temp_dir().join(format!("mapsize-del-{}", std::process::id())); + std::fs::create_dir_all(d.join("sub/inner")).unwrap(); + std::fs::write(d.join("sub/inner/f"), b"x").unwrap(); + let uid = unsafe { libc::getuid() }; + assert!(delete(&d.join("sub"), Expect { uid, mode: 0 }).is_err()); + delete(&d.join("sub"), Expect { uid, mode: mode::DIR }).unwrap(); + assert!(!d.join("sub").exists()); + std::fs::remove_dir_all(&d).unwrap(); + } +} + +/// Shell-style glob match with Go filepath.Match semantics: `*` and `?` do +/// not match '/'. Unlike Go, glibc treats a malformed bracket literally. +pub fn glob_match(pattern: &[u8], name: &[u8]) -> bool { + let (Ok(p), Ok(n)) = (CString::new(pattern), CString::new(name)) else { return false }; + unsafe { libc::fnmatch(p.as_ptr(), n.as_ptr(), libc::FNM_PATHNAME) == 0 } +} + +/// Lexically cleans a path like Go's filepath.Clean. +pub fn clean_path(p: &[u8]) -> Vec { + if p.is_empty() { + return b".".to_vec(); + } + let rooted = p[0] == b'/'; + let mut parts: Vec<&[u8]> = Vec::new(); + for c in p.split(|&b| b == b'/') { + match c { + b"" | b"." => {} + b".." => { + if parts.last().is_some_and(|l| *l != b"..") { + parts.pop(); + } else if !rooted { + parts.push(c); + } + } + _ => parts.push(c), + } + } + let mut out = Vec::with_capacity(p.len()); + if rooted { + out.push(b'/'); + } + out.extend_from_slice(&parts.join(&b'/')); + if out.is_empty() { + out.push(b'.'); + } + out +} + +/// Makes a path absolute (relative to the working directory) and cleans it. +pub fn abs_path(p: &[u8]) -> io::Result> { + if p.first() == Some(&b'/') { + return Ok(clean_path(p)); + } + let cwd = std::env::current_dir()?; + Ok(clean_path(&join_path(cwd.as_os_str().as_bytes(), p))) +} + +#[cfg(test)] +mod path_tests { + use super::*; + + #[test] + fn clean_and_glob() { + assert_eq!(clean_path(b"/a/b/../c/./"), b"/a/c"); + assert_eq!(clean_path(b"/.."), b"/"); + assert_eq!(clean_path(b"a/../../b"), b"../b"); + assert!(glob_match(b"*.log", b"x.log")); + assert!(!glob_match(b"*.log", b"d/x.log")); + assert!(glob_match(b"/tmp/*/x", b"/tmp/a/x")); + } +} diff --git a/rust/src/scan.rs b/rust/src/scan.rs new file mode 100644 index 0000000..f117a7d --- /dev/null +++ b/rust/src/scan.rs @@ -0,0 +1,775 @@ +//! The bounded concurrent filesystem scanner. +//! +//! Architecture (see ARCHITECTURE.md): a single controller thread owns the +//! write side of the Tree and a FIFO of pending directories. A fixed pool of +//! workers performs directory I/O and sends result chunks back. The +//! controller's select loop offers jobs and accepts results at the same +//! time, so bounded channels can never deadlock: workers never enqueue work +//! themselves. + +use crate::cancel::Cancel; +use crate::inventory::{self as inv, Delta, ErrorRecord, Kind, Node, NodeId, Tree}; +use crate::platform::{self, Meta}; +use crossbeam_channel::{Receiver, Select, Sender}; +use std::collections::HashSet; +use std::io; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::{Arc, RwLock}; +use std::time::SystemTime; + +/// Symlink traversal. +#[derive(Clone, Copy, PartialEq, Eq, Debug, Default)] +pub enum FollowMode { + #[default] + None, + SameFs, + All, +} + +impl FollowMode { + pub fn name(self) -> &'static str { + ["none", "same-filesystem", "all"][self as usize] + } +} + +/// Parses a --follow-symlinks value. +pub fn parse_follow(s: &str) -> Result { + match s { + "none" | "" => Ok(FollowMode::None), + "same-filesystem" | "same-fs" => Ok(FollowMode::SameFs), + "all" => Ok(FollowMode::All), + _ => Err(format!("invalid --follow-symlinks {s:?} (want none, same-filesystem or all)")), + } +} + +/// Resolves a concurrency mode name or number to a worker count. +pub fn workers(mode: &str) -> Result { + let cpu = std::thread::available_parallelism().map_or(1, |n| n.get()); + match mode { + "conservative" => Ok(2), + "balanced" | "" => Ok((2 * cpu).clamp(2, 8)), + "aggressive" => Ok((8 * cpu).clamp(4, 64)), + _ => match mode.trim().parse::() { + Ok(n) if (1..=1024).contains(&n) => Ok(n), + _ => Err(format!("invalid workers {mode:?} (conservative, balanced, aggressive or 1-1024)")), + }, + } +} + +/// Configures a scan. Zero tuning knobs mean default. +#[derive(Clone, Debug, Default)] +pub struct Options { + pub workers: usize, + pub follow: FollowMode, + pub one_file_system: bool, + pub excludes: Vec, + + pub chunk_size: usize, // entries per result chunk + pub job_buffer: usize, // capacity of the job channel + pub result_buf: usize, // capacity of the result channel + pub apply_batch: usize, // max results applied per write-lock hold +} + +impl Options { + fn defaults(&mut self) { + if self.workers == 0 { + self.workers = workers("balanced").unwrap(); + } + if self.chunk_size == 0 { + self.chunk_size = 1024; + } + if self.job_buffer == 0 { + self.job_buffer = self.workers * 2; + } + if self.result_buf == 0 { + self.result_buf = self.workers * 4; + } + if self.apply_batch == 0 { + self.apply_batch = 64; + } + } +} + +/// A cheap, lock-free view of scan activity. +#[derive(Clone, Copy, Debug, Default)] +pub struct Progress { + pub entries: i64, + pub pending: i64, // directories discovered but not yet dispatched + pub inflight: i64, // directories being read + pub workers: usize, +} + +#[derive(Default)] +struct Counters { + entries: AtomicI64, + pending: AtomicI64, + inflight: AtomicI64, +} + +/// One running scan. `tree` is safe to read under its read lock while the +/// scan runs. +pub struct Scanner { + pub tree: Arc>, + counters: Arc, + workers: usize, + done: Receiver<()>, +} + +impl Scanner { + /// Ready (disconnected) once the scan has finished or been cancelled and + /// drained; for use in select. + pub fn done(&self) -> &Receiver<()> { + &self.done + } + + pub fn is_done(&self) -> bool { + matches!(self.done.try_recv(), Err(crossbeam_channel::TryRecvError::Disconnected)) + } + + /// Blocks until the scan is done. + pub fn wait(&self) { + let _ = self.done.recv(); + } + + /// Live counters without locking the tree. + pub fn progress(&self) -> Progress { + Progress { + entries: self.counters.entries.load(Ordering::Relaxed), + pending: self.counters.pending.load(Ordering::Relaxed), + inflight: self.counters.inflight.load(Ordering::Relaxed), + workers: self.workers, + } + } +} + +struct Job { + id: NodeId, + path: Vec, + dev: u64, + // hints derived once per directory + in_cache: bool, + in_log: bool, +} + +enum ReadErr { + Io(io::Error), + Cancelled, +} + +impl From for ReadErr { + fn from(e: io::Error) -> Self { + ReadErr::Io(e) + } +} + +struct ScanResult { + job: Arc, + entries: Vec, + is_final: bool, + err: Option, +} + +type InodeKey = (u64, u64); + +fn kind_of(m: &Meta) -> Kind { + if platform::mode::is_dir(m.mode) { + Kind::Dir + } else if platform::mode::is_regular(m.mode) { + Kind::File + } else if platform::mode::is_symlink(m.mode) { + Kind::Symlink + } else { + Kind::Other + } +} + +fn set_meta(n: &mut Node, m: &Meta) { + n.size = m.size; + n.alloc = m.alloc; + n.mtime = m.mtime; + n.mode = m.mode; + n.uid = m.uid; + n.gid = m.gid; + n.nlink = m.nlink; + if !m.alloc_known { + n.flags |= inv::FLAG_ALLOC_UNKNOWN; + } +} + +/// Runs f, aborting the process if it panics: a scanner bug must never +/// leave the controller waiting forever. The panic hook (which the TUI uses +/// to restore the terminal) runs first. +fn guard(f: impl FnOnce()) { + if std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).is_err() { + std::process::abort(); + } +} + +/// Stats root, creates the tree and begins scanning in the background. +pub fn start(root: &[u8], mut opts: Options, cancel: Cancel) -> io::Result { + opts.defaults(); + // Path-style exclusions are matched against absolute paths. + for p in &mut opts.excludes { + if p.contains(['/', '\\']) { + if let Ok(a) = platform::abs_path(p.as_bytes()) { + *p = String::from_utf8_lossy(&a).into_owned(); + } + } + } + let abs = platform::abs_path(root)?; + let m = platform::stat(&abs)?; + let kind = kind_of(&m); + let mut t = Tree::new(&abs, kind); + t.stats.workers = opts.workers; + t.stats.excludes = opts.excludes.clone(); + t.stats.one_file_system = opts.one_file_system; + t.stats.follow = opts.follow.name().to_string(); + let r = t.node_mut(0); + set_meta(r, &m); + r.tot_size = m.size; + r.tot_alloc = m.alloc; + let (done_tx, done) = crossbeam_channel::bounded::<()>(0); + let counters = Arc::new(Counters::default()); + if kind != Kind::Dir { + t.stats.files = 1; + t.stats.complete = true; + t.stats.end = Some(SystemTime::now()); + drop(done_tx); + return Ok(Scanner { tree: Arc::new(RwLock::new(t)), counters, workers: opts.workers, done }); + } + t.stats.dirs = 1; + // Loop detection needs inode identity; without it symlinks are not + // followed at all. + if opts.follow != FollowMode::None && !m.has_ino { + opts.follow = FollowMode::None; + t.stats.follow = "none (no inode identity on this platform)".into(); + } + let visited = (opts.follow != FollowMode::None).then(|| HashSet::from([(m.dev, m.ino)])); + let path_excludes = opts.excludes.iter().map(|p| (p.contains(['/', '\\']), p.as_bytes().to_vec())).collect(); + let tree = Arc::new(RwLock::new(t)); + let workers = opts.workers; + counters.pending.store(1, Ordering::Relaxed); + let c = Controller { + tree: tree.clone(), + counters: counters.clone(), + excludes: path_excludes, + opts, + pending: vec![(0, m.dev)], + head: 0, + links: HashSet::new(), + visited, + }; + std::thread::Builder::new().name("scan-controller".into()).spawn(move || { + guard(|| c.run(cancel)); + drop(done_tx); + })?; + Ok(Scanner { tree, counters, workers, done }) +} + +/// Bounds how long the controller holds the write lock, so the UI's read +/// lock is never starved for more than a few milliseconds. +const MAX_ENTRIES_PER_LOCK: usize = 8192; + +struct Controller { + tree: Arc>, + counters: Arc, + opts: Options, + excludes: Vec<(bool, Vec)>, // (is path pattern, pattern) + pending: Vec<(NodeId, u64)>, // FIFO of (dir, dev); head advances, compacted occasionally + head: usize, + links: HashSet, + visited: Option>, // only when following symlinks +} + +fn worker(cancel: &Cancel, jobs: Receiver>, results: Sender, chunk: usize) { + for j in jobs { + let err = if cancel.is_cancelled() { + Some(ReadErr::Cancelled) + } else { + platform::read_dir(&j.path, chunk, |ms| { + if cancel.is_cancelled() { + return Err(ReadErr::Cancelled); + } + results.send(ScanResult { job: j.clone(), entries: ms, is_final: false, err: None }).map_err(|_| ReadErr::Cancelled) + }) + .err() + }; + let _ = results.send(ScanResult { job: j, entries: Vec::new(), is_final: true, err }); + } +} + +impl Controller { + fn run(mut self, cancel: Cancel) { + let (jobs_tx, jobs_rx) = crossbeam_channel::bounded::>(self.opts.job_buffer); + let (res_tx, res_rx) = crossbeam_channel::bounded::(self.opts.result_buf); + let mut handles = Vec::with_capacity(self.opts.workers); + for i in 0..self.opts.workers { + let (jobs, results, cancel, chunk) = (jobs_rx.clone(), res_tx.clone(), cancel.clone(), self.opts.chunk_size); + handles.push( + std::thread::Builder::new() + .name(format!("scan-worker-{i}")) + .spawn(move || guard(|| worker(&cancel, jobs, results, chunk))) + .expect("spawn scan worker"), + ); + } + drop((jobs_rx, res_tx)); + + let mut inflight = 0i64; + let mut cancelled = false; + let mut next: Option> = None; + let mut batch: Vec = Vec::with_capacity(self.opts.apply_batch); + loop { + if next.is_none() && !cancelled && self.head < self.pending.len() { + next = Some(self.make_job(self.pending[self.head])); + } + if inflight == 0 && next.is_none() { + break; + } + let mut sel = Select::new(); + let send_i = next.as_ref().map(|_| sel.send(&jobs_tx)); + let recv_i = sel.recv(&res_rx); + let cancel_i = (!cancelled).then(|| sel.recv(cancel.done())); + let op = sel.select(); + let i = op.index(); + if Some(i) == send_i { + op.send(&jobs_tx, next.take().unwrap()).expect("workers alive"); + self.head += 1; + if self.head > 4096 && self.head * 2 > self.pending.len() { + self.pending.drain(..self.head); + self.head = 0; + } + inflight += 1; + } else if i == recv_i { + let r = op.recv(&res_rx).expect("workers alive"); + let mut n = r.entries.len(); + batch.push(r); + while batch.len() < self.opts.apply_batch && n < MAX_ENTRIES_PER_LOCK { + match res_rx.try_recv() { + Ok(r) => { + n += r.entries.len(); + batch.push(r); + } + Err(_) => break, + } + } + let tree = self.tree.clone(); + let mut t = tree.write().unwrap(); + for r in batch.drain(..) { + if self.apply(&mut t, r) { + inflight -= 1; + } + } + } else { + debug_assert_eq!(Some(i), cancel_i); + let _ = op.recv(cancel.done()); + cancelled = true; + next = None; + } + self.counters.inflight.store(inflight, Ordering::Relaxed); + self.counters.pending.store((self.pending.len() - self.head) as i64, Ordering::Relaxed); + } + drop(jobs_tx); + for h in handles { + let _ = h.join(); + } + + let mut t = self.tree.write().unwrap(); + if cancelled { + t.stats.cancelled = true; + t.stats.unscanned += (self.pending.len() - self.head) as i64; + for &(id, _) in &self.pending[self.head..] { + t.node_mut(id).flags |= inv::FLAG_INCOMPLETE; + } + } + t.stats.complete = true; + t.stats.end = Some(SystemTime::now()); + drop(t); + self.counters.pending.store(0, Ordering::Relaxed); + self.counters.inflight.store(0, Ordering::Relaxed); + } + + fn make_job(&self, (id, dev): (NodeId, u64)) -> Arc { + let path = self.tree.read().unwrap().path_bytes(id); + let (in_cache, in_log) = inv::path_hints(&path); + Arc::new(Job { id, path, dev, in_cache, in_log }) + } + + /// Merges one result into the tree. Reports whether the result completed + /// its directory. Called with the tree write-locked. + fn apply(&mut self, t: &mut Tree, r: ScanResult) -> bool { + let j = &*r.job; + let mut d = Delta::default(); + self.counters.entries.fetch_add(r.entries.len() as i64, Ordering::Relaxed); + for m in r.entries { + self.add_entry(t, j, m, &mut d); + } + if r.is_final { + let dn = t.node_mut(j.id); + match r.err { + None => dn.flags |= inv::FLAG_SCANNED, + Some(ReadErr::Cancelled) => { + dn.flags |= inv::FLAG_INCOMPLETE; + t.stats.unscanned += 1; + } + Some(ReadErr::Io(e)) => { + dn.flags |= inv::FLAG_ERROR | inv::FLAG_INCOMPLETE; + t.stats.add_error(ErrorRecord { node: j.id, kind: inv::classify(&e), msg: format!("open: {}", err_msg(&e)), ..Default::default() }); + d.errors += 1; + } + } + } + if d != Delta::default() { + t.propagate(j.id, &d); + } + r.is_final + } + + fn excluded(&self, path: &[u8], name: &[u8]) -> bool { + self.excludes.iter().any(|(is_path, p)| { + if *is_path { + path == p.as_slice() || path.starts_with(p) && path.get(p.len()) == Some(&b'/') || platform::glob_match(p, path) + } else { + platform::glob_match(p, name) + } + }) + } + + fn add_entry(&mut self, t: &mut Tree, j: &Job, m: Meta, d: &mut Delta) { + if let Some(e) = &m.err { + let k = inv::classify(e); + if k == inv::ErrKind::Vanished { + t.stats.add_error(ErrorRecord { node: j.id, name: m.name, kind: k, msg: "vanished between listing and stat".into() }); + d.errors += 1; + return; + } + let id = t.add(j.id, &m.name, Kind::Other); + let n = t.node_mut(id); + n.flags |= inv::FLAG_ERROR; + n.errors = 1; + t.stats.add_error(ErrorRecord { node: id, kind: k, msg: format!("stat: {}", err_msg(e)), ..Default::default() }); + d.errors += 1; + d.files += 1; + t.stats.others += 1; + return; + } + let mut path = Vec::new(); + if !self.excludes.is_empty() { + path = platform::join_path(&j.path, &m.name); + if self.excluded(&path, &m.name) { + t.stats.excluded += 1; + return; + } + } + let mut kind = kind_of(&m); + let mut target = None; + let mut broken = false; + if kind == Kind::Symlink { + if path.is_empty() { + path = platform::join_path(&j.path, &m.name); + } + match platform::stat(&path) { + Err(_) => { + t.stats.broken_links += 1; + broken = true; + } + Ok(tm) if platform::mode::is_dir(tm.mode) && tm.has_ino && (self.opts.follow == FollowMode::All || self.opts.follow == FollowMode::SameFs && tm.dev == j.dev) => { + target = Some(tm); + kind = Kind::Dir; + } + Ok(_) => {} + } + } + + let id = t.add(j.id, &m.name, kind); + let n = t.node_mut(id); + set_meta(n, &m); + n.tot_size = n.size; + n.tot_alloc = n.alloc; + let ext = n.ext; + let mut flags = if broken { inv::FLAG_BROKEN_LINK } else { 0 }; + let st = &mut t.stats; + + match kind { + Kind::Dir => { + st.dirs += 1; + d.dirs += 1; + d.size += m.size; + d.alloc += m.alloc; + let mut dev = m.dev; + let mut key = (m.dev, m.ino); + if let Some(tm) = &target { + flags |= inv::FLAG_FOLLOWED; + dev = tm.dev; + key = (tm.dev, tm.ino); + } + let descend = 'check: { + if let Some(v) = &mut self.visited { + if !v.insert(key) { + flags |= inv::FLAG_LOOP; + st.loops_skipped += 1; + break 'check false; + } + } + if m.has_ino && dev != j.dev { + flags |= inv::FLAG_MOUNT_POINT; + if self.opts.one_file_system { + flags |= inv::FLAG_SKIPPED_FS; + st.skipped_mounts += 1; + break 'check false; + } + if path.is_empty() { + path = platform::join_path(&j.path, &m.name); + } + if platform::is_virtual_fs(&path) { + flags |= inv::FLAG_VIRTUAL_FS; + st.virtual_skipped += 1; + break 'check false; + } + } + true + }; + if descend { + self.pending.push((id, dev)); + } + } + Kind::File => { + st.files += 1; + d.files += 1; + let dup = 'check: { + if m.nlink > 1 && m.has_ino { + flags |= inv::FLAG_HARDLINKED; + if !self.links.insert((m.dev, m.ino)) { + flags |= inv::FLAG_HARDLINK_DUP; + st.hardlink_dups += 1; + break 'check true; + } + } + false + }; + if !dup { + if m.alloc_known && m.alloc < m.size && m.size - m.alloc >= 4096 { + flags |= inv::FLAG_SPARSE; + } + let cat = inv::category_for(t.ext_by_index(ext as usize), j.in_cache, j.in_log); + t.node_mut(id).cat = cat; + d.size += m.size; + d.alloc += m.alloc; + d.cat[cat as usize] += m.alloc; + } else { + // Duplicates contribute zero to aggregates. + let n = t.node_mut(id); + n.tot_size = 0; + n.tot_alloc = 0; + } + } + _ => { + if kind == Kind::Symlink { + st.symlinks += 1; + } else { + st.others += 1; + } + d.files += 1; + d.size += m.size; + d.alloc += m.alloc; + } + } + t.node_mut(id).flags |= flags; + } +} + +/// The OS error text without Rust's " (os error N)" suffix, matching Go. +fn err_msg(e: &io::Error) -> String { + let s = e.to_string(); + match s.rfind(" (os error ") { + Some(i) => s[..i].to_lowercase(), + None => s, + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use std::os::unix::ffi::OsStrExt; + use std::path::{Path, PathBuf}; + + pub struct TempDir(pub PathBuf); + impl TempDir { + pub fn new(tag: &str) -> TempDir { + static N: AtomicI64 = AtomicI64::new(0); + let p = std::env::temp_dir().join(format!("mapsize-{tag}-{}-{}", std::process::id(), N.fetch_add(1, Ordering::Relaxed))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } + } + impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } + } + + /// Creates a fan-out tree: (files, dirs, bytes). + pub fn mk_tree(dir: &Path, fanout: usize, depth: usize, files: usize) -> (i64, i64, i64) { + fn rec(dir: &Path, d: usize, fanout: usize, depth: usize, files: usize, acc: &mut (i64, i64, i64)) { + for i in 0..files { + let sz = (i + 1) * 100; + std::fs::write(dir.join(format!("f{i}.txt")), vec![0u8; sz]).unwrap(); + acc.0 += 1; + acc.2 += sz as i64; + } + if d == depth { + return; + } + for i in 0..fanout { + let sub = dir.join(format!("d{i}")); + std::fs::create_dir(&sub).unwrap(); + acc.1 += 1; + rec(&sub, d + 1, fanout, depth, files, acc); + } + } + let mut acc = (0, 0, 0); + rec(dir, 0, fanout, depth, files, &mut acc); + acc + } + + pub fn run_scan(root: &Path, o: Options) -> Tree { + let s = start(root.as_os_str().as_bytes(), o, Cancel::new()).unwrap(); + s.done().recv_timeout(std::time::Duration::from_secs(60)).expect_err("scan did not finish (deadlock?)"); + let t = s.tree.read().unwrap().clone(); + t + } + + fn totals(t: &Tree) -> Vec<[i64; 4]> { + (0..t.len() as NodeId).map(|i| t.node(i)).map(|n| [n.tot_size, n.tot_alloc, n.files as i64, n.dirs as i64]).collect() + } + + #[test] + fn counts_and_aggregates() { + let root = TempDir::new("scan"); + let (nf, nd, bytes) = mk_tree(&root.0, 3, 3, 4); + let mut t = run_scan(&root.0, Options { workers: 4, ..Default::default() }); + let r = t.node(0); + assert_eq!((r.files as i64, r.dirs as i64), (nf, nd)); + let dir_bytes: i64 = (0..t.len() as NodeId).map(|i| t.node(i)).filter(|n| n.is_dir()).map(|n| n.size).sum(); + assert_eq!(r.tot_size, bytes + dir_bytes); + assert!(t.stats.complete && !t.stats.incomplete()); + for i in 0..t.len() as NodeId { + let n = t.node(i); + if n.is_dir() { + let sum: i64 = n.size + t.children(i).map(|(_, c)| c.tot_size).sum::(); + assert_eq!(sum, n.tot_size, "{}", t.path_string(i)); + } + } + let before = totals(&t); + t.recompute(); + assert_eq!(before, totals(&t), "incremental aggregates differ from recompute"); + } + + #[test] + fn backpressure_no_deadlock() { + let root = TempDir::new("bp"); + let (nf, nd, _) = mk_tree(&root.0, 6, 4, 3); + for w in [1, 2, 16] { + let t = run_scan(&root.0, Options { workers: w, job_buffer: 1, result_buf: 1, chunk_size: 1, apply_batch: 1, ..Default::default() }); + assert_eq!((t.node(0).files as i64, t.node(0).dirs as i64), (nf, nd), "workers={w}"); + } + } + + #[test] + fn cancel_stops() { + let root = TempDir::new("cancel"); + mk_tree(&root.0, 8, 3, 2); + let c = Cancel::new(); + let s = start(root.0.as_os_str().as_bytes(), Options { workers: 2, job_buffer: 1, result_buf: 1, chunk_size: 1, ..Default::default() }, c.clone()).unwrap(); + c.cancel(); + s.done().recv_timeout(std::time::Duration::from_secs(10)).expect_err("cancel did not stop scan"); + let t = s.tree.read().unwrap(); + assert!(t.stats.cancelled && t.stats.incomplete()); + } + + #[test] + fn symlink_loops() { + let root = TempDir::new("loop"); + let a = root.0.join("a"); + std::fs::create_dir_all(a.join("b")).unwrap(); + std::fs::write(a.join("b/x"), vec![0u8; 5000]).unwrap(); + std::os::unix::fs::symlink("..", a.join("b/up")).unwrap(); + std::os::unix::fs::symlink("../a", a.join("self")).unwrap(); + std::os::unix::fs::symlink("nowhere", root.0.join("broken")).unwrap(); + for f in [FollowMode::None, FollowMode::SameFs, FollowMode::All] { + let t = run_scan(&root.0, Options { workers: 4, follow: f, ..Default::default() }); + let r = t.node(0); + assert!(r.files > 0 && r.tot_size > 5000, "{f:?}"); + assert_eq!(t.stats.broken_links, 1); + assert!(f == FollowMode::None || t.stats.loops_skipped > 0); + let count = (0..t.len() as NodeId).filter(|&i| &*t.node(i).name == b"x").count(); + assert_eq!(count, 1, "{f:?}"); + } + } + + #[test] + fn hardlinks_counted_once() { + let root = TempDir::new("hl"); + let p = root.0.join("orig"); + std::fs::write(&p, vec![0u8; 100000]).unwrap(); + std::fs::create_dir(root.0.join("d")).unwrap(); + std::fs::hard_link(&p, root.0.join("d/link")).unwrap(); + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + assert_eq!(t.stats.hardlink_dups, 1); + assert!(t.node(0).tot_size < 200000); + } + + #[test] + fn permission_errors_surface() { + if unsafe { libc::geteuid() } == 0 { + return; + } + use std::os::unix::fs::PermissionsExt; + let root = TempDir::new("perm"); + let locked = root.0.join("locked"); + std::fs::create_dir(&locked).unwrap(); + std::fs::write(locked.join("f"), b"x").unwrap(); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0)).unwrap(); + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + std::fs::set_permissions(&locked, std::fs::Permissions::from_mode(0o755)).unwrap(); + assert_eq!(t.stats.err_counts[inv::ErrKind::Permission as usize], 1); + assert!(t.stats.incomplete()); + assert_eq!(t.node(0).errors, 1); + } + + #[test] + fn excludes() { + let root = TempDir::new("excl"); + std::fs::create_dir_all(root.0.join("keep")).unwrap(); + std::fs::create_dir_all(root.0.join("skip")).unwrap(); + std::fs::write(root.0.join("keep/a.tmp"), b"x").unwrap(); + std::fs::write(root.0.join("keep/b.txt"), b"x").unwrap(); + std::fs::write(root.0.join("skip/c.txt"), b"x").unwrap(); + let skip = root.0.join("skip").to_string_lossy().into_owned(); + let t = run_scan(&root.0, Options { workers: 2, excludes: vec!["*.tmp".into(), skip], ..Default::default() }); + assert_eq!((t.stats.excluded, t.node(0).files), (2, 1)); + } + + #[test] + fn hostile_names_preserved() { + let root = TempDir::new("names"); + let names: [&[u8]; 6] = ["日本語.txt".as_bytes(), "emoji-🎉.png".as_bytes(), b"esc\x1b[31mred", b"nl\nname", b"sp ace", b"bad\xffutf8"]; + for n in names { + std::fs::write(root.0.join(std::ffi::OsStr::from_bytes(n)), b"x").unwrap(); + } + let t = run_scan(&root.0, Options { workers: 2, ..Default::default() }); + let got: HashSet<&[u8]> = t.children(0).map(|(_, n)| &*n.name).collect(); + for n in names { + assert!(got.contains(n), "missing {n:?}"); + } + } + + #[test] + fn single_file_root() { + let root = TempDir::new("single"); + let p = root.0.join("file"); + std::fs::write(&p, vec![0u8; 1234]).unwrap(); + let t = run_scan(&p, Options::default()); + assert_eq!(t.node(0).tot_size, 1234); + assert_eq!(t.node(0).kind, Kind::File); + } +} diff --git a/rust/src/textutil.rs b/rust/src/textutil.rs new file mode 100644 index 0000000..a4c496e --- /dev/null +++ b/rust/src/textutil.rs @@ -0,0 +1,368 @@ +//! Display helpers: sanitising untrusted strings for the terminal, +//! width-aware truncation, and human-readable units. + +use std::fmt::Write; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Duration; +use unicode_segmentation::UnicodeSegmentation; +use unicode_width::{UnicodeWidthChar, UnicodeWidthStr}; + +/// Makes an untrusted string (a filename, snapshot data) safe to write to a +/// terminal. Control characters (C0, DEL, C1) and bidi overrides become +/// visible escapes; invalid UTF-8 becomes U+FFFD. Keep the original bytes +/// for filesystem operations. +pub fn sanitize_bytes(s: &[u8]) -> String { + if s.iter().all(|&c| (0x20..0x7f).contains(&c)) { + // SAFETY-free: printable ASCII is valid UTF-8. + return String::from_utf8(s.to_vec()).unwrap(); + } + let mut b = String::with_capacity(s.len() + 8); + for chunk in s.utf8_chunks() { + for r in chunk.valid().chars() { + let c = r as u32; + if c < 0x20 || c == 0x7f || (0x80..=0x9f).contains(&c) { + let _ = write!(b, "\\x{c:02x}"); + } else if is_bidi_control(c) { + let _ = write!(b, "\\u{c:04x}"); + } else { + b.push(r); + } + } + for _ in chunk.invalid() { + b.push('\u{FFFD}'); + } + } + b +} + +/// sanitize_bytes for strings. +pub fn sanitize(s: &str) -> String { + sanitize_bytes(s.as_bytes()) +} + +fn is_bidi_control(r: u32) -> bool { + (0x202a..=0x202e).contains(&r) || (0x2066..=0x2069).contains(&r) || r == 0x200e || r == 0x200f || r == 0x061c +} + +/// Calls f for each grapheme cluster of s with its display width. Clusters +/// whose width terminals disagree on (per-char wcwidth versus grapheme +/// width, e.g. emoji ZWJ sequences) are replaced by U+FFFD with width 1, and +/// zero-width clusters are dropped, so measuring and drawing always agree +/// with the terminal. f returns false to stop. +pub fn clusters(s: &str, mut f: impl FnMut(&str, usize) -> bool) { + for c in s.graphemes(true) { + let mut chars = c.chars(); + let first = chars.next().unwrap(); + let w = if chars.next().is_none() { first.width().unwrap_or(0) } else { c.width() }; + if w == 0 { + continue; + } + let ok = if c.len() > 1 && c.chars().map(|r| r.width().unwrap_or(0)).sum::() != w { + f("\u{FFFD}", 1) + } else { + f(c, w) + }; + if !ok { + return; + } + } +} + +/// Display width of s in terminal cells. +pub fn width(s: &str) -> usize { + if s.bytes().all(|c| (0x20..0x7f).contains(&c)) { + return s.len(); + } + let mut n = 0; + clusters(s, |_, w| { + n += w; + true + }); + n +} + +/// Clips s to at most w cells, appending "…" if anything was cut. +pub fn truncate(s: &str, w: usize) -> String { + if w == 0 { + return String::new(); + } + if width(s) <= w { + return s.to_string(); + } + clip(s, w - 1) + "…" +} + +/// Clips from the left, prefixing "…" (for paths and breadcrumbs where the +/// tail is the informative part). +pub fn truncate_left(s: &str, w: usize) -> String { + if w == 0 { + return String::new(); + } + let total = width(s); + if total <= w { + return s.to_string(); + } + let mut skip = (total - (w - 1)) as isize; + let mut b = String::from("…"); + clusters(s, |c, cw| { + if skip > 0 { + skip -= cw as isize; + if skip < 0 { + // A wide cluster overshot by one cell; pad to keep width exact. + b.push_str(&" ".repeat((-skip) as usize)); + } + return true; + } + b.push_str(c); + true + }); + b +} + +/// The longest prefix of s that fits in w cells. +fn clip(s: &str, w: usize) -> String { + let mut b = String::new(); + let mut used = 0; + clusters(s, |c, cw| { + if used + cw > w { + return false; + } + used += cw; + b.push_str(c); + true + }); + b +} + +/// Pads s with spaces to exactly w cells (truncating if longer). +pub fn pad_right(s: &str, w: usize) -> String { + let mut s = truncate(s, w); + let sw = width(&s); + if sw < w { + s.push_str(&" ".repeat(w - sw)); + } + s +} + +/// Right-aligns s in w cells. +pub fn pad_left(s: &str, w: usize) -> String { + let s = truncate(s, w); + let sw = width(&s); + if sw < w { + return " ".repeat(w - sw) + &s; + } + s +} + +static SI: AtomicBool = AtomicBool::new(false); + +/// Selects decimal units (kB, MB, …) instead of IEC (KiB, MiB, …). Set once +/// at start-up. +pub fn set_si(v: bool) { + SI.store(v, Ordering::Relaxed); +} + +const IEC_UNITS: [&str; 7] = ["B", "KiB", "MiB", "GiB", "TiB", "PiB", "EiB"]; +const SI_UNITS: [&str; 7] = ["B", "kB", "MB", "GB", "TB", "PB", "EB"]; + +fn scaled(n: i64) -> (f64, &'static str) { + let (base, units) = if SI.load(Ordering::Relaxed) { (1000.0, &SI_UNITS) } else { (1024.0, &IEC_UNITS) }; + let neg = n < 0; + let mut v = (n as f64).abs(); + let mut i = 0; + while v >= base && i < units.len() - 1 { + v /= base; + i += 1; + } + (if neg { -v } else { v }, units[i]) +} + +fn num(v: f64, unit: &str) -> String { + let a = v.abs(); + if unit == "B" { + format!("{v:.0}") + } else if a < 10.0 { + format!("{v:.2}") + } else if a < 100.0 { + format!("{v:.1}") + } else { + format!("{v:.0}") + } +} + +/// Formats a byte count, e.g. "86.4 GiB". +pub fn size(n: i64) -> String { + let (v, u) = scaled(n); + num(v, u) + " " + u +} + +/// Formats a byte count in at most ~7 cells, e.g. "86GiB". +pub fn size_compact(n: i64) -> String { + let (v, u) = scaled(n); + if u != "B" && v.abs() < 10.0 { + format!("{v:.1}{u}") + } else { + format!("{v:.0}{u}") + } +} + +/// Formats a delta with an explicit sign. +pub fn signed_size(n: i64) -> String { + match n { + n if n > 0 => "+".to_string() + &size(n), + n if n < 0 => "-".to_string() + &size(n.saturating_neg()), + _ => "±0 B".to_string(), + } +} + +/// Formats an integer with thousands separators. +pub fn count(n: i64) -> String { + let s = n.unsigned_abs().to_string(); + let mut out = String::with_capacity(s.len() + s.len() / 3 + 1); + if n < 0 { + out.push('-'); + } + for (i, c) in s.chars().enumerate() { + if i > 0 && (s.len() - i) % 3 == 0 { + out.push(','); + } + out.push(c); + } + out +} + +/// Formats an elapsed time as mm:ss or h:mm:ss. +pub fn duration(d: Duration) -> String { + let s = d.as_secs(); + if s >= 3600 { + format!("{}:{:02}:{:02}", s / 3600, s / 60 % 60, s % 60) + } else { + format!("{:02}:{:02}", s / 60, s % 60) + } +} + +/// Formats part/whole as a percentage. +pub fn percent(part: i64, whole: i64) -> String { + if whole <= 0 { + return "–".to_string(); + } + let p = part as f64 * 100.0 / whole as f64; + if p < 10.0 { + format!("{p:.1}%") + } else { + format!("{p:.0}%") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn sanitize_cases() { + let cases: [(&[u8], &str); 8] = [ + (b"plain.txt", "plain.txt"), + (b"evil\x1b[2Jname", r"evil\x1b[2Jname"), + (b"bell\x07", r"bell\x07"), + ("c1\u{009b}x".as_bytes(), r"c1\x9bx"), + (b"bad\xffutf8", "bad\u{FFFD}utf8"), + ("rtl\u{202e}gnp.exe".as_bytes(), "rtl\\u202egnp.exe"), + ("日本語".as_bytes(), "日本語"), + (b"tab\tnew\nline\rdel\x7f", r"tab\x09new\x0aline\x0ddel\x7f"), + ]; + for (input, want) in cases { + let got = sanitize_bytes(input); + assert_eq!(got, want); + assert!(!got.contains(['\x1b', '\x07', '\u{009b}'])); + } + } + + #[test] + fn truncation() { + assert_eq!(truncate("hello world", 5), "hell…"); + assert!(width(&truncate("日本語テキスト", 5)) <= 5); + assert_eq!(truncate("abc", 3), "abc"); + assert_eq!(truncate("abc", 0), ""); + let got = truncate_left("/a/very/long/path", 8); + assert!(width(&got) == 8 && got.ends_with("path"), "{got}"); + assert_eq!(width(&truncate_left("ab日本語", 4)), 4); + } + + #[test] + fn sizes() { + for (n, want) in [(0, "0 B"), (1023, "1023 B"), (1024, "1.00 KiB"), (1536, "1.50 KiB"), (181 << 30, "181 GiB"), (1 << 62, "4.00 EiB")] { + assert_eq!(size(n), want); + } + assert_eq!(count(2184392), "2,184,392"); + assert_eq!(signed_size(-2048), "-2.00 KiB"); + } + + #[test] + fn clusters_agree_with_width() { + for s in ["abc", "日本", "é", "👨‍👩‍👧", "🎉x", "\u{200b}"] { + let mut n = 0; + clusters(s, |_, w| { + n += w; + true + }); + assert_eq!(n, width(s), "{s:?}"); + } + } +} + +/// Formats unix nanoseconds as an RFC 3339 UTC timestamp, e.g. +/// "2026-10-08T09:30:00Z". +pub fn rfc3339_utc(unix_nanos: i64) -> String { + let secs = unix_nanos.div_euclid(1_000_000_000); + let (days, rem) = (secs.div_euclid(86400), secs.rem_euclid(86400)); + // Civil-from-days (Howard Hinnant). + let z = days + 719468; + let era = z.div_euclid(146097); + let doe = z - era * 146097; + let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let d = doy - (153 * mp + 2) / 5 + 1; + let m = if mp < 10 { mp + 3 } else { mp - 9 }; + let y = yoe + era * 400 + (m <= 2) as i64; + format!("{y:04}-{m:02}-{d:02}T{:02}:{:02}:{:02}Z", rem / 3600, rem / 60 % 60, rem % 60) +} + +/// Formats a duration like Go's time.Duration.String after rounding to +/// milliseconds, e.g. "1.234s", "250ms", "2m3.5s". +pub fn go_duration(d: Duration) -> String { + let ms = (d.as_nanos() + 500_000) / 1_000_000; + if ms == 0 { + return "0s".into(); + } + if ms < 1000 { + return format!("{ms}ms"); + } + let (h, m, s) = (ms / 3_600_000, ms / 60_000 % 60, ms % 60_000); + let mut secs = format!("{}.{:03}", s / 1000, s % 1000); + while secs.ends_with('0') { + secs.pop(); + } + if secs.ends_with('.') { + secs.pop(); + } + match (h, m) { + (0, 0) => format!("{secs}s"), + (0, m) => format!("{m}m{secs}s"), + (h, m) => format!("{h}h{m}m{secs}s"), + } +} + +#[cfg(test)] +mod time_tests { + use super::*; + + #[test] + fn formats() { + assert_eq!(rfc3339_utc(0), "1970-01-01T00:00:00Z"); + assert_eq!(rfc3339_utc(1_791_456_000 * 1_000_000_000), "2026-10-08T10:40:00Z"); + assert_eq!(go_duration(Duration::from_micros(1_234_400)), "1.234s"); + assert_eq!(go_duration(Duration::from_millis(250)), "250ms"); + assert_eq!(go_duration(Duration::from_millis(123_500)), "2m3.5s"); + } +}