From 476311b583ed6c58f4f12d2ba84945c266a63069 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Thu, 8 Oct 2026 17:16:06 +0100 Subject: [PATCH] rust: port filter --- rust/Cargo.lock | 39 +++ rust/Cargo.toml | 1 + rust/src/filter/apply.rs | 80 ++++++ rust/src/filter/eval.rs | 133 ++++++++++ rust/src/filter/lexer.rs | 95 +++++++ rust/src/filter/mod.rs | 127 +++++++++ rust/src/filter/parser.rs | 532 ++++++++++++++++++++++++++++++++++++++ rust/src/lib.rs | 1 + 8 files changed, 1008 insertions(+) create mode 100644 rust/src/filter/apply.rs create mode 100644 rust/src/filter/eval.rs create mode 100644 rust/src/filter/lexer.rs create mode 100644 rust/src/filter/mod.rs create mode 100644 rust/src/filter/parser.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index e96506f..4150d0d 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -8,6 +8,15 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "aho-corasick" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c982642fa9e8606056828ee9a8505737230110bb1099153c79efe865c59d12ba" +dependencies = [ + "memchr", +] + [[package]] name = "bitflags" version = "2.13.2" @@ -179,6 +188,7 @@ dependencies = [ "crossterm", "flate2", "libc", + "regex", "serde", "serde_json", "sha2", @@ -264,6 +274,35 @@ dependencies = [ "bitflags", ] +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad8553b9b26413251cbf30e620595c7a41b3887f03da04579c0e6b0d6a06b4b2" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + [[package]] name = "rustix" version = "0.38.44" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index efd2908..044ac2c 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -7,6 +7,7 @@ description = "Interactive terminal disk usage analyser (Rust port)" [dependencies] libc = "0.2" +regex = "1" crossbeam-channel = "0.5" crossterm = "0.28" flate2 = "1" diff --git a/rust/src/filter/apply.rs b/rust/src/filter/apply.rs new file mode 100644 index 0000000..6b9b4d5 --- /dev/null +++ b/rust/src/filter/apply.rs @@ -0,0 +1,80 @@ +use super::eval::Ctx; +use super::parser::Query; +use crate::cancel::Cancel; +use crate::inventory::{NodeId, SizeMode, Tree, FLAG_HARDLINK_DUP}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; + +/// Per-node filtered sizes: the bytes of matching content beneath each node. +/// A matching directory contributes its whole subtree (once, even if +/// descendants match too). matched marks every node satisfying the query. +/// Nodes added after the result was computed (live scan) have no entry and +/// are treated as not matching until the filter is re-applied. +pub struct FilterResult { + pub query: Arc, + pub mode: SizeMode, + pub sizes: Vec, // per NodeId; len = tree size at evaluation time + pub matched: Vec, // node itself matched + pub count: i64, // number of matching nodes + pub total: i64, + pub elapsed: Duration, +} + +impl FilterResult { + /// The filtered size of id (0 if out of range). + pub fn size(&self, id: NodeId) -> i64 { + self.sizes.get(id as usize).copied().unwrap_or(0) + } + + /// Reports whether id itself matched. + pub fn is_match(&self, id: NodeId) -> bool { + self.matched.get(id as usize).copied().unwrap_or(false) + } +} + +/// Evaluates q over the whole tree. The caller holds the read lock. It relies +/// on parents having smaller IDs than their children. None if cancelled. +pub fn apply(cancel: &Cancel, t: &Tree, q: Arc, m: SizeMode) -> Option { + let start = Instant::now(); + let n = t.len(); + let mut sizes = vec![0i64; n]; + let mut matched = vec![false; n]; + let mut covered = vec![false; n]; + let mut count = 0; + let now = SystemTime::now().duration_since(UNIX_EPOCH).map_or(0, |d| d.as_nanos() as i64); + let mut c = Ctx::new(t, 0, now); + if q.uses_path { + c.dir_paths = Some(HashMap::new()); + } + for i in 0..n { + if i & 0xffff == 0 && cancel.is_cancelled() { + return None; + } + let id = i as NodeId; + let nd = t.node(id); + (c.id, c.node, c.path) = (id, nd, None); + if c.dir_paths.is_some() && nd.is_dir() { + let p = c.lower_path(); + c.dir_paths.as_mut().unwrap().insert(id, p.clone()); + c.path = Some(p); + } + if i > 0 && q.matches(&mut c) { + matched[i] = true; + covered[i] = true; + count += 1; + } + if i > 0 && covered[nd.parent as usize] { + covered[i] = true; + } + if covered[i] && nd.flags & FLAG_HARDLINK_DUP == 0 { + sizes[i] = nd.own(m); + } + } + for i in (1..n).rev() { + let p = t.node(i as NodeId).parent as usize; + sizes[p] = sizes[p].saturating_add(sizes[i]); + } + let total = sizes.first().copied().unwrap_or(0); + Some(FilterResult { query: q, mode: m, sizes, matched, count, total, elapsed: start.elapsed() }) +} diff --git a/rust/src/filter/eval.rs b/rust/src/filter/eval.rs new file mode 100644 index 0000000..5f112c1 --- /dev/null +++ b/rust/src/filter/eval.rs @@ -0,0 +1,133 @@ +use super::parser::{Field, Query}; +use crate::inventory::{Flags, Node, NodeId, Tree}; +use crate::platform; +use regex::Regex; +use std::borrow::Cow; +use std::collections::HashMap; + +/// The evaluation context for one node. The tree must be read-locked. +pub struct Ctx<'a> { + pub tree: &'a Tree, + pub id: NodeId, + pub node: &'a Node, + pub now: i64, // unix nanoseconds + pub(super) path: Option, + // dir_paths caches lower-cased directory paths during apply, which visits + // parents before children, so a node's path is one concatenation. + pub(super) dir_paths: Option>, +} + +impl<'a> Ctx<'a> { + pub fn new(tree: &'a Tree, id: NodeId, now: i64) -> Ctx<'a> { + Ctx { tree, id, node: tree.node(id), now, path: None, dir_paths: None } + } + + fn str(&mut self, f: Field) -> Cow<'_, str> { + let n = self.node; + match f { + Field::Name => n.name_str().to_lowercase().into(), + Field::Path => { + if self.path.is_none() { + self.path = Some(self.lower_path()); + } + self.path.as_deref().unwrap().into() + } + Field::Ext => self.tree.ext_name(n).into(), + Field::Type => n.kind.name().to_lowercase().into(), + Field::Category if n.is_dir() => "directory".into(), + Field::Category => n.cat.name().to_lowercase().into(), + Field::Owner => platform::user_name(n.uid).to_lowercase().into(), + Field::Group => platform::group_name(n.gid).to_lowercase().into(), + _ => "".into(), + } + } + + fn num(&self, f: Field) -> i64 { + let n = self.node; + match f { + Field::Size => n.tot_size, + Field::Allocated => n.tot_alloc, + Field::Age => self.now.saturating_sub(n.mtime), + Field::Modified => n.mtime, + Field::Files => n.files as i64, + _ => 0, + } + } + + pub(super) fn lower_path(&self) -> String { + if let Some(dp) = &self.dir_paths { + if let Some(p) = dp.get(&self.id) { + return p.clone(); + } + if let Some(pp) = dp.get(&self.node.parent) { + let sep = if pp.ends_with('/') { "" } else { "/" }; + return format!("{pp}{sep}{}", self.node.name_str().to_lowercase()); + } + } + self.tree.path_string(self.id).to_lowercase() + } +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum NumOp { + Eq, + Ne, + Gt, + Ge, + Lt, + Le, +} + +/// A compiled query expression. +pub(super) enum Expr { + And(Box, Box), + Or(Box, Box), + Not(Box), + Contains(Field, String), + Glob(Field, String), + Re(Field, Regex), + Eq(Field, Vec), + Flag(Flags), // any of these flags set + Num(Field, NumOp, Vec), + False, // malformed bare glob +} + +impl Expr { + fn eval(&self, c: &mut Ctx) -> bool { + match self { + Expr::And(l, r) => l.eval(c) && r.eval(c), + Expr::Or(l, r) => l.eval(c) || r.eval(c), + Expr::Not(n) => !n.eval(c), + Expr::Contains(f, sub) => c.str(*f).contains(sub.as_str()), + Expr::Glob(f, pat) => platform::glob_match(pat.as_bytes(), c.str(*f).as_bytes()), + Expr::Re(f, re) => re.is_match(&c.str(*f)), + Expr::Eq(f, set) => { + let v = c.str(*f); + set.iter().any(|s| *s == v) + } + Expr::Flag(mask) => c.node.flags & mask != 0, + Expr::Num(f, op, vals) => { + if matches!(f, Field::Age | Field::Modified) && c.node.mtime == 0 { + return false; // unknown time never matches + } + let v = c.num(*f); + vals.iter().any(|&x| match op { + NumOp::Eq => v == x, + NumOp::Ne => v != x, + NumOp::Gt => v > x, + NumOp::Ge => v >= x, + NumOp::Lt => v < x, + NumOp::Le => v <= x, + }) + } + Expr::False => false, + } + } +} + +impl Query { + /// Evaluates the query against one node. + pub fn matches(&self, c: &mut Ctx) -> bool { + self.root.eval(c) + } +} diff --git a/rust/src/filter/lexer.rs b/rust/src/filter/lexer.rs new file mode 100644 index 0000000..f39f5f7 --- /dev/null +++ b/rust/src/filter/lexer.rs @@ -0,0 +1,95 @@ +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub(super) enum TokKind { + Eof, + Word, + Str, + Op, + LParen, + RParen, + Comma, +} + +#[derive(Clone, Debug)] +pub(super) struct Token { + pub kind: TokKind, + pub val: String, + pub pos: usize, // byte offset +} + +fn tok(kind: TokKind, val: &str, pos: usize) -> Token { + Token { kind, val: val.to_string(), pos } +} + +/// ASCII white space as Go's unicode.IsSpace sees it. +fn is_space(c: u8) -> bool { + matches!(c, b' ' | b'\t' | b'\n' | b'\x0b' | b'\x0c' | b'\r') +} + +pub(super) fn lex(s: &str) -> Result, String> { + use TokKind::*; + let b = s.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while i < b.len() { + let c = b[i]; + match c { + // All of is_space, not just " \t\n\r": a word can't start on a + // space byte, so anything else here would never advance. + _ if is_space(c) => i += 1, + b'(' => { + out.push(tok(LParen, "(", i)); + i += 1; + } + b')' => { + out.push(tok(RParen, ")", i)); + i += 1; + } + b',' => { + out.push(tok(Comma, ",", i)); + i += 1; + } + b'"' | b'\'' => { + let start = i; + i += 1; + let mut v = Vec::new(); + while i < b.len() && b[i] != c { + if b[i] == b'\\' && i + 1 < b.len() { + i += 1; + } + v.push(b[i]); + i += 1; + } + if i >= b.len() { + return Err(format!("unterminated string at {}", start + 1)); + } + i += 1; + // Only ASCII bytes were dropped, so v is still valid UTF-8. + out.push(Token { kind: Str, val: String::from_utf8_lossy(&v).into_owned(), pos: start }); + } + b'=' | b'!' | b'<' | b'>' | b'~' | b'&' | b'|' => { + let start = i; + let n = b.get(i + 1).copied(); + let two = n == Some(b'=') || (c == b'&' && n == Some(b'&')) || (c == b'|' && n == Some(b'|')); + let op = &s[i..i + 1 + two as usize]; + i += op.len(); + match op { + "&&" => out.push(tok(Word, "AND", start)), + "||" => out.push(tok(Word, "OR", start)), + "!" => out.push(tok(Word, "NOT", start)), + "&" | "|" => return Err(format!("unexpected {op:?} at {}", start + 1)), + "==" => out.push(tok(Op, "=", start)), + _ => out.push(tok(Op, op, start)), + } + } + _ => { + let start = i; + while i < b.len() && !is_space(b[i]) && !b"()=!<>~,\"'&|".contains(&b[i]) { + i += 1; + } + out.push(tok(Word, &s[start..i], start)); + } + } + } + out.push(tok(Eof, "", s.len())); + Ok(out) +} diff --git a/rust/src/filter/mod.rs b/rust/src/filter/mod.rs new file mode 100644 index 0000000..77dcfb2 --- /dev/null +++ b/rust/src/filter/mod.rs @@ -0,0 +1,127 @@ +//! The search/filter query language: +//! +//! ```text +//! query := or +//! or := and { OR and } +//! and := unary { [AND] unary } (juxtaposition means AND) +//! unary := NOT unary | "(" query ")" | cmp | word +//! cmp := field op value +//! op := = | != | > | >= | < | <= | contains | matches | in | ~ +//! value := word | "quoted string" | "(" value { "," value } ")" +//! ``` +//! +//! A bare word matches the name: substring (case-insensitive), or a glob if it +//! contains * ? or [. Parsing is independent of the UI and fully testable. + +mod apply; +mod eval; +mod lexer; +mod parser; + +pub use apply::{apply, FilterResult}; +pub use eval::Ctx; +pub use parser::{parse, parse_age, parse_size, Field, Query, FIELDS}; + +#[cfg(test)] +mod tests { + use super::*; + use crate::cancel::Cancel; + use crate::inventory::{category_for, ext, Kind, NodeId, SizeMode, Tree}; + use std::collections::HashSet; + use std::sync::Arc; + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + const HOUR: Duration = Duration::from_secs(3600); + const DAY: Duration = Duration::from_secs(86400); + + fn tree() -> Tree { + let mut t = Tree::new(b"/srv", Kind::Dir); + let vms = t.add(0, b"vms", Kind::Dir); + let cache = t.add(0, b"cache", Kind::Dir); + let mut add = |p: NodeId, name: &str, size: i64, age: Duration| { + let id = t.add(p, name.as_bytes(), Kind::File); + let n = t.node_mut(id); + n.size = size; + n.alloc = size; + n.mtime = (SystemTime::now() - age).duration_since(UNIX_EPOCH).unwrap().as_nanos() as i64; + n.cat = category_for(&ext(name.as_bytes()), false, false); + }; + add(vms, "ubuntu.qcow2", 80 << 30, 400 * DAY); + add(vms, "win.vmdk", 40 << 30, HOUR); + add(vms, "notes.txt", 1000, HOUR); + add(cache, "blob.iso", 700 << 20, 10 * DAY); + add(cache, "x.tmp", 10, HOUR); + t.recompute(); + t + } + + fn names(tr: &Tree, q: &str) -> HashSet { + let pq = parse(q, SystemTime::now()).unwrap_or_else(|e| panic!("parse({q:?}): {e}")); + let r = apply(&Cancel::new(), tr, Arc::new(pq), SizeMode::Logical).unwrap(); + (0..tr.len() as NodeId).filter(|&i| r.is_match(i)).map(|i| tr.node(i).name_str().into_owned()).collect() + } + + #[test] + fn queries() { + let tr = tree(); + let cases: &[(&str, &[&str])] = &[ + ("ubuntu", &["ubuntu.qcow2"]), + ("*.iso", &["blob.iso"]), + ("size > 1GB", &["vms", "ubuntu.qcow2", "win.vmdk"]), + ("size > 1GB AND type = file", &["ubuntu.qcow2", "win.vmdk"]), + ("ext = qcow2", &["ubuntu.qcow2"]), + ("size > 500MB AND ext IN (iso,qcow2,vmdk)", &["ubuntu.qcow2", "win.vmdk", "blob.iso"]), + ("path contains cache", &["cache", "blob.iso", "x.tmp"]), + ("age > 365d", &["ubuntu.qcow2"]), + ("NOT type = dir AND size < 1k", &["notes.txt", "x.tmp"]), + ("(ext = iso OR ext = tmp) type=file", &["blob.iso", "x.tmp"]), + ("category = vm", &["ubuntu.qcow2", "win.vmdk", "blob.iso"]), + ("name matches '^w.*k$'", &["win.vmdk"]), + ("ext != qcow2 && type = file && size > 1GB", &["win.vmdk"]), + ("name = 'notes.txt'", &["notes.txt"]), + ]; + for (q, want) in cases { + let want: HashSet = want.iter().map(|s| s.to_string()).collect(); + assert_eq!(names(&tr, q), want, "{q:?}"); + } + } + + #[test] + fn filtered_sizes_no_double_count() { + let tr = tree(); + let q = parse("name = vms OR ext = qcow2", SystemTime::now()).unwrap(); + let r = apply(&Cancel::new(), &tr, Arc::new(q), SizeMode::Logical).unwrap(); + assert_eq!(r.total, tr.node(1).tot_size, "matching dir must not double count matching child"); + } + + #[test] + fn parse_errors() { + for q in [ + "", "size >", "size > banana", "(ext = iso", "ext = iso)", "size contains 5", + "name matches '['", "type = blob", "'unterminated", "a & b", "category = nope", + ] { + assert!(parse(q, SystemTime::now()).is_err(), "parse({q:?}) should fail"); + } + } + + #[test] + fn dates_globs_and_non_ascii() { + let tr = tree(); + assert_eq!(names(&tr, "modified < 2000-01-01"), HashSet::new()); + assert_eq!(names(&tr, "modified > '2000-01-01 9:30'").len(), 5); + assert_eq!(names(&tr, "modified > 2000-01-01T00:00:00.5+01:00").len(), 5); + assert_eq!(names(&tr, "modified > 30d").len(), 4); + for bad in ["modified > 2024-02-30", "modified > 2024-01-31T10:00", "ext = 'a['"] { + assert!(parse(bad, SystemTime::now()).is_ok() == bad.ends_with("10:00"), "{bad}"); + } + assert!(names(&tr, "[").is_empty()); // malformed bare glob matches nothing + assert!(names(&tr, "voilà\u{a0}x").is_empty()); // Go's lexer hangs on byte 0xA0 + } + + #[test] + fn parse_size_units() { + for (input, want) in [("1GB", 1_000_000_000), ("1GiB", 1 << 30), ("1.5k", 1536), ("500MB", 500_000_000), ("10", 10)] { + assert_eq!(parse_size(input), Ok(want), "{input}"); + } + } +} diff --git a/rust/src/filter/parser.rs b/rust/src/filter/parser.rs new file mode 100644 index 0000000..4e9b08e --- /dev/null +++ b/rust/src/filter/parser.rs @@ -0,0 +1,532 @@ +use super::eval::{Expr, NumOp}; +use super::lexer::{lex, TokKind, Token}; +use crate::inventory::{self as inv, Flags}; +use regex::Regex; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; + +/// A filterable attribute. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum Field { + Name, + Path, + Ext, + Type, + Category, + Size, + Allocated, + Age, + Modified, + Owner, + Group, + Files, + Flag, +} + +fn field_by_name(s: &str) -> Option { + use Field::*; + Some(match s { + "name" => Name, + "path" => Path, + "ext" | "extension" => Ext, + "type" | "kind" => Type, + "category" | "cat" => Category, + "size" => Size, + "allocated" | "alloc" => Allocated, + "age" => Age, + "modified" | "mtime" => Modified, + "owner" | "user" => Owner, + "group" => Group, + "files" => Files, + "flag" | "is" => Flag, + _ => return None, + }) +} + +/// Field names for help text. +pub const FIELDS: &str = "name path ext type category size allocated age modified owner group files flag"; + +/// A compiled filter. +pub struct Query { + src: String, + pub(super) root: Expr, + pub uses_path: bool, +} + +impl std::fmt::Display for Query { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.src) + } +} + +struct Parser { + toks: Vec, + i: usize, + uses_path: bool, + now: SystemTime, +} + +/// Compiles a query. now anchors relative ages (pass SystemTime::now()). +pub fn parse(s: &str, now: SystemTime) -> Result { + let toks = lex(s)?; + let mut p = Parser { toks, i: 0, uses_path: false, now }; + if p.peek().kind == TokKind::Eof { + return Err("empty query".into()); + } + let root = p.or()?; + let t = p.peek(); + if t.kind != TokKind::Eof { + return Err(format!("unexpected {:?} at {}", t.val, t.pos + 1)); + } + Ok(Query { src: s.to_string(), root, uses_path: p.uses_path }) +} + +fn is_kw(t: &Token, kw: &str) -> bool { + t.kind == TokKind::Word && t.val.eq_ignore_ascii_case(kw) +} + +fn is_value(t: &Token) -> bool { + t.kind == TokKind::Word || t.kind == TokKind::Str +} + +impl Parser { + fn peek(&self) -> &Token { + &self.toks[self.i] + } + + fn next(&mut self) -> Token { + let t = self.toks[self.i].clone(); + if t.kind != TokKind::Eof { + self.i += 1; + } + t + } + + fn or(&mut self) -> Result { + let mut l = self.and()?; + while is_kw(self.peek(), "OR") { + self.next(); + let r = self.and()?; + l = Expr::Or(Box::new(l), Box::new(r)); + } + Ok(l) + } + + fn and(&mut self) -> Result { + let mut l = self.unary()?; + loop { + let t = self.peek(); + if is_kw(t, "AND") { + self.next(); + } else if matches!(t.kind, TokKind::Eof | TokKind::RParen) || is_kw(t, "OR") { + return Ok(l); + } + let r = self.unary()?; + l = Expr::And(Box::new(l), Box::new(r)); + } + } + + fn unary(&mut self) -> Result { + let t = self.peek().clone(); + if is_kw(&t, "NOT") { + self.next(); + return Ok(Expr::Not(Box::new(self.unary()?))); + } + if t.kind == TokKind::LParen { + self.next(); + let n = self.or()?; + if self.next().kind != TokKind::RParen { + return Err(format!("missing ) for ( at {}", t.pos + 1)); + } + return Ok(n); + } + if is_value(&t) { + self.next(); + if let Some(f) = field_by_name(&t.val.to_lowercase()) { + if t.kind == TokKind::Word && self.is_op(self.peek()) { + let op = self.next(); + return self.cmp(f, op); + } + } + return Ok(name_match(&t.val)); + } + Err(format!("unexpected {:?} at {}", t.val, t.pos + 1)) + } + + fn is_op(&self, t: &Token) -> bool { + t.kind == TokKind::Op || is_kw(t, "contains") || is_kw(t, "matches") || is_kw(t, "in") + } + + fn values(&mut self) -> Result, String> { + if self.peek().kind != TokKind::LParen { + let t = self.next(); + if !is_value(&t) { + return Err(format!("expected value at {}", t.pos + 1)); + } + return Ok(vec![t]); + } + self.next(); + let mut out = Vec::new(); + loop { + let t = self.next(); + if !is_value(&t) { + return Err(format!("expected value at {}", t.pos + 1)); + } + out.push(t); + let n = self.next(); + match n.kind { + TokKind::Comma => {} + TokKind::RParen => return Ok(out), + _ => return Err(format!("expected , or ) at {}", n.pos + 1)), + } + } + } + + fn cmp(&mut self, f: Field, op_tok: Token) -> Result { + let mut op = op_tok.val.to_lowercase(); + if op == "~" { + op = "matches".into(); + } + let vals = self.values()?; + if op != "in" && vals.len() != 1 { + return Err(format!("operator {op} takes one value")); + } + if f == Field::Path { + self.uses_path = true; + } + if matches!(f, Field::Size | Field::Allocated | Field::Age | Field::Modified | Field::Files) { + let nop = match op.as_str() { + "=" | "in" => NumOp::Eq, + "!=" => NumOp::Ne, + ">" => NumOp::Gt, + ">=" => NumOp::Ge, + "<" => NumOp::Lt, + "<=" => NumOp::Le, + _ => return Err(format!("{op} cannot be used with numeric field")), + }; + let nums = vals.iter().map(|v| self.number(f, &v.val)).collect::, _>>()?; + return Ok(Expr::Num(f, nop, nums)); + } + // String-like fields. + match op.as_str() { + "contains" => Ok(Expr::Contains(f, vals[0].val.to_lowercase())), + "matches" => match Regex::new(&format!("(?i){}", vals[0].val)) { + Ok(re) => Ok(Expr::Re(f, re)), + Err(e) => Err(format!("bad regular expression: {e}")), + }, + "=" | "!=" | "in" => { + let mut set = Vec::new(); + for v in &vals { + let mut s = v.val.to_lowercase(); + match f { + Field::Ext => s = s.strip_prefix('.').unwrap_or(&s).to_string(), + Field::Category => match inv::parse_category(&s) { + Some(c) => s = c.name().to_lowercase(), + None => return Err(format!("unknown category {:?}", v.val)), + }, + Field::Type => s = norm_type(&s)?.to_string(), + Field::Flag if flag_bit(&s).is_none() => { + return Err(format!( + "unknown flag {:?} (sparse, hardlink, error, mount, incomplete, followed, broken)", + v.val + )) + } + _ => {} + } + set.push(s); + } + let n = if f == Field::Flag { + Expr::Flag(set.iter().filter_map(|s| flag_bit(s)).fold(0, |a, b| a | b)) + } else if set.len() == 1 && has_glob(&set[0]) { + if !glob_ok(set[0].as_bytes()) { + return Err(format!("bad pattern {:?}", vals[0].val)); + } + Expr::Glob(f, set.pop().unwrap()) + } else { + Expr::Eq(f, set) + }; + Ok(if op == "!=" { Expr::Not(Box::new(n)) } else { n }) + } + _ => Err(format!("operator {op} not valid for {}", op_tok.val)), + } + } + + /// Parses sizes ("1.5GiB", "500MB", "10k"), ages ("365d", "2w") and dates + /// ("2024-01-31"). Ages and dates are converted to modification times + /// (unix nanoseconds) relative to self.now. + fn number(&self, f: Field, s: &str) -> Result { + match f { + Field::Size | Field::Allocated => parse_size(s), + Field::Files => s.parse::().map_err(|e| format!("bad number {s:?}: {e}")), + Field::Age => parse_age(s).map(nanos), + Field::Modified => { + if let Some(t) = parse_date(s) { + return Ok(t); + } + if let Ok(d) = parse_age(s) { + let now = self.now.duration_since(UNIX_EPOCH).map(nanos).unwrap_or(0); + return Ok(now.saturating_sub(nanos(d))); + } + Err(format!("bad date {s:?} (use YYYY-MM-DD or an age like 30d)")) + } + _ => Err("not numeric".into()), + } + } +} + +fn nanos(d: Duration) -> i64 { + d.as_nanos().min(i64::MAX as u128) as i64 +} + +fn has_glob(s: &str) -> bool { + s.contains(['*', '?', '[']) +} + +pub(super) fn name_match(v: &str) -> Expr { + let lv = v.to_lowercase(); + if has_glob(v) { + // Go's filepath.Match never matches a malformed pattern. + return if glob_ok(lv.as_bytes()) { Expr::Glob(Field::Name, lv) } else { Expr::False }; + } + Expr::Contains(Field::Name, lv) +} + +/// Reports whether pattern is well formed by Go's filepath.Match rules +/// (fnmatch, which does the matching, accepts anything). +fn glob_ok(p: &[u8]) -> bool { + // One possibly escaped character of a range; None if malformed. + let esc = |mut i: usize| -> Option { + match p.get(i)? { + b'-' | b']' => return None, + b'\\' => { + i += 1; + p.get(i)?; + } + _ => {} + } + Some(i + 1) + }; + let mut i = 0; + while i < p.len() { + match p[i] { + b'\\' => { + if i + 1 >= p.len() { + return false; + } + i += 2; + } + b'[' => { + i += 1; + if p.get(i) == Some(&b'^') { + i += 1; + } + let mut n = 0; + loop { + if p.get(i) == Some(&b']') && n > 0 { + i += 1; + break; + } + let Some(j) = esc(i) else { return false }; + i = j; + if p.get(i) == Some(&b'-') { + let Some(j) = esc(i + 1) else { return false }; + i = j; + } + n += 1; + } + } + _ => i += 1, + } + } + true +} + +fn norm_type(s: &str) -> Result<&'static str, String> { + match s { + "f" | "file" => Ok("file"), + "d" | "dir" | "directory" => Ok("directory"), + "l" | "link" | "symlink" => Ok("symlink"), + "other" | "special" => Ok("special"), + _ => Err(format!("unknown type {s:?} (file, dir, symlink, other)")), + } +} + +fn flag_bit(s: &str) -> Option { + Some(match s { + "sparse" => inv::FLAG_SPARSE, + "hardlink" | "hardlinked" => inv::FLAG_HARDLINKED, + "error" => inv::FLAG_ERROR, + "mount" => inv::FLAG_MOUNT_POINT, + "incomplete" => inv::FLAG_INCOMPLETE, + "followed" => inv::FLAG_FOLLOWED, + "broken" => inv::FLAG_BROKEN_LINK, + _ => return None, + }) +} + +/// Splits a leading run of digits and dots from s. +fn split_num(s: &str) -> (&str, &str) { + let i = s.bytes().position(|c| !(c.is_ascii_digit() || c == b'.')).unwrap_or(s.len()); + s.split_at(i) +} + +/// Parses a size with optional unit. kB/MB/GB/TB are SI (powers of 1000); +/// KiB/MiB/GiB/TiB are IEC (1024). Bare K/M/G/T follow the display setting +/// (textutil::set_si). +pub fn parse_size(s: &str) -> Result { + let (num, unit_raw) = split_num(s); + let Ok(v) = num.parse::() else { return Err(format!("bad size {s:?}")) }; + // ponytail: textutil exposes no SI getter; its formatting reveals the setting. + let bare: f64 = if crate::textutil::size(1000).ends_with("kB") { 1000.0 } else { 1024.0 }; + let m = match unit_raw.trim().to_lowercase().as_str() { + "" | "b" => 1.0, + "k" => bare, + "m" => bare.powi(2), + "g" => bare.powi(3), + "t" => bare.powi(4), + "p" => bare.powi(5), + "kb" => 1e3, + "mb" => 1e6, + "gb" => 1e9, + "tb" => 1e12, + "pb" => 1e15, + "kib" => (1u64 << 10) as f64, + "mib" => (1u64 << 20) as f64, + "gib" => (1u64 << 30) as f64, + "tib" => (1u64 << 40) as f64, + "pib" => (1u64 << 50) as f64, + _ => return Err(format!("unknown size unit {unit_raw:?}")), + }; + let r = v * m; + if r > 9e18 { + return Err(format!("size {s:?} too large")); + } + Ok(r as i64) +} + +/// Parses durations like 90s, 30min, 12h, 7d, 2w, 6mo, 1y. +pub fn parse_age(s: &str) -> Result { + let (num, unit) = split_num(s); + let Ok(v) = num.parse::() else { return Err(format!("bad age {s:?}")) }; + const DAY: f64 = 86400.0; + let secs = match unit.to_lowercase().as_str() { + "s" | "sec" => 1.0, + "min" => 60.0, + "h" => 3600.0, + "d" | "" => DAY, + "w" => 7.0 * DAY, + "m" | "mo" => 30.0 * DAY, + "y" => 365.0 * DAY, + _ => return Err(format!("unknown age unit {unit:?} (s, min, h, d, w, mo, y)")), + }; + // Saturates where Go's time.Duration would overflow. + Ok(Duration::from_nanos((v * secs * 1e9) as u64)) +} + +/// Parses "2006-01-02", "2006-01-02T15:04", "2006-01-02 15:04" (local time) +/// or RFC 3339, as Go's time.ParseInLocation with time.Local would, into unix +/// nanoseconds. +fn parse_date(s: &str) -> Option { + let b = s.as_bytes(); + let fixed = |i: usize, n: usize| -> Option { + let d = b.get(i..i + n)?; + d.iter().all(u8::is_ascii_digit).then(|| d.iter().fold(0, |a, &c| a * 10 + (c - b'0') as i64)) + }; + let (y, mo, d) = (fixed(0, 4)?, fixed(5, 2)?, fixed(8, 2)?); + if b.get(4) != Some(&b'-') || b.get(7) != Some(&b'-') || !(1..=12).contains(&mo) || d < 1 || d > days_in(y, mo) { + return None; + } + if b.len() == 10 { + return local_nanos(y, mo, d, 0, 0); + } + let sep = *b.get(10)?; + if sep != b'T' && sep != b' ' { + return None; + } + // Go's "15" hour accepts one or two digits. + let mut i = 11; + let hl = if fixed(i, 2).is_some() { 2 } else { 1 }; + let h = fixed(i, hl)?; + i += hl; + if b.get(i) != Some(&b':') || h > 23 { + return None; + } + let mi = fixed(i + 1, 2)?; + i += 3; + if mi > 59 { + return None; + } + if i == b.len() { + return local_nanos(y, mo, d, h, mi); + } + // RFC 3339: :05[.frac](Z|±07:00) + if sep != b'T' || b.get(i) != Some(&b':') { + return None; + } + let sec = fixed(i + 1, 2)?; + i += 3; + if sec > 59 { + return None; + } + let mut ns = 0i64; + if matches!(b.get(i), Some(b'.' | b',')) { + let n = b[i + 1..].iter().take_while(|c| c.is_ascii_digit()).count(); + if n == 0 { + return None; + } + for k in 0..9 { + ns = ns * 10 + b.get(i + 1 + k).filter(|_| k < n).map_or(0, |c| (c - b'0') as i64); + } + i += 1 + n; + } + let off = match b.get(i)? { + b'Z' => { + i += 1; + 0 + } + &c @ (b'+' | b'-') => { + let (oh, om) = (fixed(i + 1, 2)?, fixed(i + 4, 2)?); + if b.get(i + 3) != Some(&b':') || oh > 23 || om > 59 { + return None; + } + i += 6; + (oh * 3600 + om * 60) * if c == b'-' { -1 } else { 1 } + } + _ => return None, + }; + if i != b.len() { + return None; + } + let secs = days_from_civil(y, mo, d) * 86400 + h * 3600 + mi * 60 + sec - off; + Some(secs * 1_000_000_000 + ns) +} + +fn days_in(y: i64, m: i64) -> i64 { + match m { + 2 if y % 4 == 0 && (y % 100 != 0 || y % 400 == 0) => 29, + 2 => 28, + 4 | 6 | 9 | 11 => 30, + _ => 31, + } +} + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * (m + if m > 2 { -3 } else { 9 }) + 2) / 5 + d - 1; + let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy; + era * 146097 + doe - 719468 +} + +fn local_nanos(y: i64, mo: i64, d: i64, h: i64, mi: i64) -> Option { + let mut tm: libc::tm = unsafe { std::mem::zeroed() }; + tm.tm_year = (y - 1900) as i32; + tm.tm_mon = (mo - 1) as i32; + tm.tm_mday = d as i32; + tm.tm_hour = h as i32; + tm.tm_min = mi as i32; + tm.tm_isdst = -1; + let t = unsafe { libc::mktime(&mut tm) }; + (t != -1).then(|| t as i64 * 1_000_000_000) +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index ae752ec..9d3839d 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -4,6 +4,7 @@ pub mod brand; pub mod cancel; pub mod config; pub mod export; +pub mod filter; pub mod inventory; pub mod platform; pub mod scan;