diff --git a/internal/filter/apply.go b/internal/filter/apply.go new file mode 100644 index 0000000..e405abe --- /dev/null +++ b/internal/filter/apply.go @@ -0,0 +1,71 @@ +package filter + +import ( + "context" + "time" + + "github.com/andydixon/mapsize/internal/inventory" +) + +// Result holds per-node filtered sizes: the bytes of matching content beneath +// each node. A matching directory contributes its whole subtree (once, even +// if descendants match too). Matched marks every node satisfying the query. Nodes added +// after the result was computed (live scan) have no entry and are treated as +// not matching until the filter is re-applied. +type Result struct { + Query *Query + Mode inventory.SizeMode + Sizes []int64 // per NodeID; len = tree size at evaluation time + Matched []bool // node itself matched + Count int64 // number of matching nodes + Total int64 + Elapsed time.Duration +} + +// Size returns the filtered size of id (0 if out of range). +func (r *Result) Size(id inventory.NodeID) int64 { + if int(id) < len(r.Sizes) { + return r.Sizes[id] + } + return 0 +} + +// IsMatch reports whether id itself matched. +func (r *Result) IsMatch(id inventory.NodeID) bool { + return int(id) < len(r.Matched) && r.Matched[id] +} + +// Apply evaluates q over the whole tree. The tree must be read-locked by the +// caller. It relies on parents having smaller IDs than their children. +func Apply(ctx context.Context, t *inventory.Tree, q *Query, m inventory.SizeMode) (*Result, error) { + start := time.Now() + n := t.Len() + r := &Result{Query: q, Mode: m, Sizes: make([]int64, n), Matched: make([]bool, n)} + covered := make([]bool, n) + c := &Ctx{Tree: t, Now: time.Now().UnixNano()} + for i := range n { + if i&0xffff == 0 && ctx.Err() != nil { + return nil, ctx.Err() + } + id := inventory.NodeID(i) + nd := t.Node(id) + c.ID, c.Node = id, nd + if i > 0 && q.Match(c) { + r.Matched[i] = true + covered[i] = true + r.Count++ + } + if i > 0 && covered[nd.Parent] { + covered[i] = true + } + if covered[i] && nd.Flags&inventory.FlagHardlinkDup == 0 { + r.Sizes[i] = nd.Own(m) + } + } + for i := n - 1; i > 0; i-- { + r.Sizes[t.Node(inventory.NodeID(i)).Parent] += r.Sizes[i] + } + r.Total = r.Sizes[0] + r.Elapsed = time.Since(start) + return r, nil +} diff --git a/internal/filter/eval.go b/internal/filter/eval.go new file mode 100644 index 0000000..e04cc94 --- /dev/null +++ b/internal/filter/eval.go @@ -0,0 +1,160 @@ +package filter + +import ( + "path/filepath" + "regexp" + "strings" + + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/platform" +) + +// Ctx is the evaluation context for one node. The tree must be read-locked. +type Ctx struct { + Tree *inventory.Tree + ID inventory.NodeID + Node *inventory.Node + Now int64 // unix nanoseconds + path string +} + +func (c *Ctx) str(f Field) string { + n := c.Node + switch f { + case FName: + return strings.ToLower(n.Name) + case FPath: + if c.path == "" { + c.path = strings.ToLower(c.Tree.Path(c.ID)) + } + return c.path + case FExt: + return c.Tree.ExtName(n) + case FType: + return strings.ToLower(n.Kind.String()) + case FCategory: + if n.IsDir() { + return "directory" + } + return strings.ToLower(n.Cat.String()) + case FOwner: + return strings.ToLower(platform.UserName(n.UID)) + case FGroup: + return strings.ToLower(platform.GroupName(n.GID)) + } + return "" +} + +func (c *Ctx) num(f Field) int64 { + n := c.Node + switch f { + case FSize: + return n.TotSize + case FAllocated: + return n.TotAlloc + case FAge: + return c.Now - n.MTime + case FModified: + return n.MTime + case FFiles: + return int64(n.Files) + } + return 0 +} + +type node interface{ eval(*Ctx) bool } + +type andNode struct{ l, r node } +type orNode struct{ l, r node } +type notNode struct{ n node } + +func (n andNode) eval(c *Ctx) bool { return n.l.eval(c) && n.r.eval(c) } +func (n orNode) eval(c *Ctx) bool { return n.l.eval(c) || n.r.eval(c) } +func (n notNode) eval(c *Ctx) bool { return !n.n.eval(c) } + +type containsNode struct { + field Field + sub string +} + +func (n containsNode) eval(c *Ctx) bool { return strings.Contains(c.str(n.field), n.sub) } + +type globNode struct { + field Field + pat string +} + +func (n globNode) eval(c *Ctx) bool { + ok, _ := filepath.Match(n.pat, c.str(n.field)) + return ok +} + +type reNode struct { + field Field + re *regexp.Regexp +} + +func (n reNode) eval(c *Ctx) bool { return n.re.MatchString(c.str(n.field)) } + +type eqNode struct { + field Field + set []string +} + +func (n eqNode) eval(c *Ctx) bool { + if n.field == FFlag { + for _, s := range n.set { + if c.Node.Flags&flagNames[s] != 0 { + return true + } + } + return false + } + v := c.str(n.field) + for _, s := range n.set { + if v == s { + return true + } + } + return false +} + +type numNode struct { + field Field + op string + vals []int64 +} + +func (n numNode) eval(c *Ctx) bool { + if (n.field == FAge || n.field == FModified) && c.Node.MTime == 0 { + return false // unknown time never matches + } + v := c.num(n.field) + for _, x := range n.vals { + var ok bool + switch n.op { + case "=": + ok = v == x + case "!=": + ok = v != x + case ">": + ok = v > x + case ">=": + ok = v >= x + case "<": + ok = v < x + case "<=": + ok = v <= x + } + if ok { + return true + } + } + return false +} + +// Match evaluates the query against one node. +func (q *Query) Match(c *Ctx) bool { + c.path = "" + return q.root.eval(c) +} diff --git a/internal/filter/filter_test.go b/internal/filter/filter_test.go new file mode 100644 index 0000000..c6a4d43 --- /dev/null +++ b/internal/filter/filter_test.go @@ -0,0 +1,122 @@ +package filter + +import ( + "context" + "testing" + "time" + + "github.com/andydixon/mapsize/internal/inventory" +) + +func tree() *inventory.Tree { + t := inventory.New("/srv", inventory.KindDir) + vms := t.Add(0, "vms", inventory.KindDir) + cache := t.Add(0, "cache", inventory.KindDir) + add := func(p inventory.NodeID, name string, size int64, age time.Duration) inventory.NodeID { + id := t.Add(p, name, inventory.KindFile) + n := t.Node(id) + n.Size, n.Alloc = size, size + n.MTime = time.Now().Add(-age).UnixNano() + n.Cat = inventory.CategoryFor(inventory.Ext(name), false, false) + return id + } + add(vms, "ubuntu.qcow2", 80<<30, 400*24*time.Hour) + add(vms, "win.vmdk", 40<<30, time.Hour) + add(vms, "notes.txt", 1000, time.Hour) + add(cache, "blob.iso", 700<<20, 10*24*time.Hour) + add(cache, "x.tmp", 10, time.Hour) + t.Recompute() + return t +} + +func names(t *testing.T, tr *inventory.Tree, q string) map[string]bool { + t.Helper() + pq, err := Parse(q, time.Now()) + if err != nil { + t.Fatalf("Parse(%q): %v", q, err) + } + r, err := Apply(context.Background(), tr, pq, inventory.SizeLogical) + if err != nil { + t.Fatal(err) + } + out := map[string]bool{} + for i := range tr.Len() { + if r.IsMatch(inventory.NodeID(i)) { + out[tr.Node(inventory.NodeID(i)).Name] = true + } + } + return out +} + +func TestQueries(t *testing.T) { + tr := tree() + cases := map[string][]string{ + "ubuntu": {"ubuntu.qcow2"}, + "*.iso": {"blob.iso"}, + "size > 1GB": {"vms", "ubuntu.qcow2", "win.vmdk"}, + "size > 1GB AND type = file": {"ubuntu.qcow2", "win.vmdk"}, + "ext = qcow2": {"ubuntu.qcow2"}, + "size > 500MB AND ext IN (iso,qcow2,vmdk)": {"ubuntu.qcow2", "win.vmdk", "blob.iso"}, + "path contains cache": {"cache", "blob.iso", "x.tmp"}, + "age > 365d": {"ubuntu.qcow2"}, + "NOT type = dir AND size < 1k": {"notes.txt", "x.tmp"}, + "(ext = iso OR ext = tmp) type=file": {"blob.iso", "x.tmp"}, + "category = vm": {"ubuntu.qcow2", "win.vmdk", "blob.iso"}, + "name matches '^w.*k$'": {"win.vmdk"}, + "ext != qcow2 && type = file && size > 1GB": {"win.vmdk"}, + "name = 'notes.txt'": {"notes.txt"}, + } + for q, want := range cases { + got := names(t, tr, q) + if len(got) != len(want) { + t.Errorf("%q: got %v want %v", q, got, want) + continue + } + for _, w := range want { + if !got[w] { + t.Errorf("%q: got %v want %v", q, got, want) + } + } + } +} + +func TestFilteredSizesNoDoubleCount(t *testing.T) { + tr := tree() + q, _ := Parse("name = vms OR ext = qcow2", time.Now()) + r, _ := Apply(context.Background(), tr, q, inventory.SizeLogical) + if want := tr.Node(1).TotSize; r.Total != want { + t.Fatalf("total %d want %d (matching dir must not double count matching child)", r.Total, want) + } +} + +func TestParseErrors(t *testing.T) { + for _, q := range []string{"", "size >", "size > banana", "(ext = iso", "ext = iso)", "size contains 5", + "name matches '['", "type = blob", "'unterminated", "a & b", "category = nope"} { + if _, err := Parse(q, time.Now()); err == nil { + t.Errorf("Parse(%q) should fail", q) + } + } +} + +func TestParseSize(t *testing.T) { + cases := map[string]int64{"1GB": 1e9, "1GiB": 1 << 30, "1.5k": 1536, "500MB": 5e8, "10": 10} + for in, want := range cases { + if got, err := ParseSize(in); err != nil || got != want { + t.Errorf("ParseSize(%q)=%d,%v want %d", in, got, err, want) + } + } +} + +func FuzzParse(f *testing.F) { + for _, s := range []string{"size > 1GB", "(a OR b) AND NOT c", "ext IN (a,b)", "'x", "name matches '.*'"} { + f.Add(s) + } + tr := tree() + f.Fuzz(func(t *testing.T, s string) { + q, err := Parse(s, time.Now()) + if err != nil { + return + } + Apply(context.Background(), tr, q, inventory.SizeAllocated) + }) +} diff --git a/internal/filter/lexer.go b/internal/filter/lexer.go new file mode 100644 index 0000000..bf72624 --- /dev/null +++ b/internal/filter/lexer.go @@ -0,0 +1,103 @@ +// Package filter implements the search/filter query language: +// +// query := or +// or := and { OR and } +// and := unary { [AND] unary } (juxtaposition means AND) +// unary := NOT unary | "(" query ")" | cmp | word +// cmp := field op value +// op := = | != | > | >= | < | <= | contains | matches | in | ~ +// value := word | "quoted string" | "(" value { "," value } ")" +// +// A bare word matches the name: substring (case-insensitive), or a glob if it +// contains * ? or [. Parsing is independent of the UI and fully testable. +package filter + +import ( + "fmt" + "strings" + "unicode" +) + +type tokKind int + +const ( + tEOF tokKind = iota + tWord + tString + tOp + tLParen + tRParen + tComma +) + +type token struct { + kind tokKind + val string + pos int +} + +func lex(s string) ([]token, error) { + var out []token + i := 0 + for i < len(s) { + c := s[i] + switch { + case c == ' ' || c == '\t' || c == '\n' || c == '\r': + i++ + case c == '(': + out = append(out, token{tLParen, "(", i}) + i++ + case c == ')': + out = append(out, token{tRParen, ")", i}) + i++ + case c == ',': + out = append(out, token{tComma, ",", i}) + i++ + case c == '"' || c == '\'': + q := c + start := i + i++ + var b strings.Builder + for i < len(s) && s[i] != q { + if s[i] == '\\' && i+1 < len(s) { + i++ + } + b.WriteByte(s[i]) + i++ + } + if i >= len(s) { + return nil, fmt.Errorf("unterminated string at %d", start+1) + } + i++ + out = append(out, token{tString, b.String(), start}) + case strings.ContainsRune("=!<>~&|", rune(c)): + start := i + op := string(c) + if i+1 < len(s) && (s[i+1] == '=' || (c == '&' && s[i+1] == '&') || (c == '|' && s[i+1] == '|')) { + op += string(s[i+1]) + } + i += len(op) + switch op { + case "&&": + out = append(out, token{tWord, "AND", start}) + case "||": + out = append(out, token{tWord, "OR", start}) + case "!": + out = append(out, token{tWord, "NOT", start}) + case "&", "|": + return nil, fmt.Errorf("unexpected %q at %d", op, start+1) + case "==": + out = append(out, token{tOp, "=", start}) + default: + out = append(out, token{tOp, op, start}) + } + default: + start := i + for i < len(s) && !unicode.IsSpace(rune(s[i])) && !strings.ContainsRune("()=!<>~,\"'&|", rune(s[i])) { + i++ + } + out = append(out, token{tWord, s[start:i], start}) + } + } + return append(out, token{tEOF, "", len(s)}), nil +} diff --git a/internal/filter/parser.go b/internal/filter/parser.go new file mode 100644 index 0000000..3e89cb5 --- /dev/null +++ b/internal/filter/parser.go @@ -0,0 +1,382 @@ +package filter + +import ( + "fmt" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/textutil" +) + +// Field is a filterable attribute. +type Field int + +const ( + FName Field = iota + FPath + FExt + FType + FCategory + FSize + FAllocated + FAge + FModified + FOwner + FGroup + FFiles + FFlag +) + +var fieldNames = map[string]Field{ + "name": FName, "path": FPath, "ext": FExt, "extension": FExt, "type": FType, "kind": FType, + "category": FCategory, "cat": FCategory, "size": FSize, "allocated": FAllocated, "alloc": FAllocated, + "age": FAge, "modified": FModified, "mtime": FModified, "owner": FOwner, "user": FOwner, "group": FGroup, + "files": FFiles, "flag": FFlag, "is": FFlag, +} + +// Fields lists the field names for help text. +const Fields = "name path ext type category size allocated age modified owner group files flag" + +// Query is a compiled filter. +type Query struct { + src string + root node + UsesPath bool +} + +func (q *Query) String() string { return q.src } + +type parser struct { + toks []token + i int + q *Query + now time.Time +} + +// Parse compiles a query. now anchors relative ages (pass time.Now()). +func Parse(s string, now time.Time) (*Query, error) { + toks, err := lex(s) + if err != nil { + return nil, err + } + q := &Query{src: s} + p := &parser{toks: toks, q: q, now: now} + if p.peek().kind == tEOF { + return nil, fmt.Errorf("empty query") + } + q.root, err = p.or() + if err != nil { + return nil, err + } + if t := p.peek(); t.kind != tEOF { + return nil, fmt.Errorf("unexpected %q at %d", t.val, t.pos+1) + } + return q, nil +} + +func (p *parser) peek() token { return p.toks[p.i] } +func (p *parser) next() token { + t := p.toks[p.i] + if t.kind != tEOF { + p.i++ + } + return t +} + +func isKw(t token, kw string) bool { return t.kind == tWord && strings.EqualFold(t.val, kw) } + +func (p *parser) or() (node, error) { + l, err := p.and() + if err != nil { + return nil, err + } + for isKw(p.peek(), "OR") { + p.next() + r, err := p.and() + if err != nil { + return nil, err + } + l = orNode{l, r} + } + return l, nil +} + +func (p *parser) and() (node, error) { + l, err := p.unary() + if err != nil { + return nil, err + } + for { + t := p.peek() + if isKw(t, "AND") { + p.next() + } else if t.kind == tEOF || t.kind == tRParen || isKw(t, "OR") { + return l, nil + } + r, err := p.unary() + if err != nil { + return nil, err + } + l = andNode{l, r} + } +} + +func (p *parser) unary() (node, error) { + t := p.peek() + switch { + case isKw(t, "NOT"): + p.next() + n, err := p.unary() + if err != nil { + return nil, err + } + return notNode{n}, nil + case t.kind == tLParen: + p.next() + n, err := p.or() + if err != nil { + return nil, err + } + if p.next().kind != tRParen { + return nil, fmt.Errorf("missing ) for ( at %d", t.pos+1) + } + return n, nil + case t.kind == tWord || t.kind == tString: + p.next() + if f, ok := fieldNames[strings.ToLower(t.val)]; ok && t.kind == tWord && p.isOp(p.peek()) { + return p.cmp(f, p.next()) + } + return nameMatch(t.val), nil + } + return nil, fmt.Errorf("unexpected %q at %d", t.val, t.pos+1) +} + +func (p *parser) isOp(t token) bool { + return t.kind == tOp || isKw(t, "contains") || isKw(t, "matches") || isKw(t, "in") +} + +func nameMatch(v string) node { + lv := strings.ToLower(v) + if strings.ContainsAny(v, "*?[") { + return globNode{field: FName, pat: lv} + } + return containsNode{field: FName, sub: lv} +} + +func (p *parser) values() ([]token, error) { + if p.peek().kind != tLParen { + t := p.next() + if t.kind != tWord && t.kind != tString { + return nil, fmt.Errorf("expected value at %d", t.pos+1) + } + return []token{t}, nil + } + p.next() + var out []token + for { + t := p.next() + if t.kind != tWord && t.kind != tString { + return nil, fmt.Errorf("expected value at %d", t.pos+1) + } + out = append(out, t) + switch n := p.next(); n.kind { + case tComma: + case tRParen: + return out, nil + default: + return nil, fmt.Errorf("expected , or ) at %d", n.pos+1) + } + } +} + +func (p *parser) cmp(f Field, opTok token) (node, error) { + op := strings.ToLower(opTok.val) + if op == "~" { + op = "matches" + } + vals, err := p.values() + if err != nil { + return nil, err + } + if op != "in" && len(vals) != 1 { + return nil, fmt.Errorf("operator %s takes one value", op) + } + if f == FPath { + p.q.UsesPath = true + } + switch f { + case FSize, FAllocated, FAge, FModified, FFiles: + if op == "contains" || op == "matches" { + return nil, fmt.Errorf("%s cannot be used with numeric field", op) + } + var nums []int64 + for _, v := range vals { + n, err := p.number(f, v.val) + if err != nil { + return nil, err + } + nums = append(nums, n) + } + if op == "in" { + op = "=" + } + return numNode{field: f, op: op, vals: nums}, nil + } + // String-like fields. + switch op { + case "contains": + return containsNode{field: f, sub: strings.ToLower(vals[0].val)}, nil + case "matches": + re, err := regexp.Compile("(?i)" + vals[0].val) + if err != nil { + return nil, fmt.Errorf("bad regular expression: %v", err) + } + return reNode{field: f, re: re}, nil + case "=", "!=", "in": + var n node + var set []string + for _, v := range vals { + s := strings.ToLower(v.val) + if f == FExt { + s = strings.TrimPrefix(s, ".") + } + if f == FCategory { + c, ok := inventory.ParseCategory(s) + if !ok { + return nil, fmt.Errorf("unknown category %q", v.val) + } + s = strings.ToLower(c.String()) + } + if f == FType { + s, err = normType(s) + if err != nil { + return nil, err + } + } + if f == FFlag { + if _, ok := flagNames[s]; !ok { + return nil, fmt.Errorf("unknown flag %q (sparse, hardlink, error, mount, incomplete, followed, broken)", v.val) + } + } + set = append(set, s) + } + if len(set) == 1 && strings.ContainsAny(set[0], "*?[") { + if _, err := filepath.Match(set[0], ""); err != nil { + return nil, fmt.Errorf("bad pattern %q", vals[0].val) + } + n = globNode{field: f, pat: set[0]} + } else { + n = eqNode{field: f, set: set} + } + if op == "!=" { + n = notNode{n} + } + return n, nil + } + return nil, fmt.Errorf("operator %s not valid for %s", op, opTok.val) +} + +func normType(s string) (string, error) { + switch s { + case "f", "file": + return "file", nil + case "d", "dir", "directory": + return "directory", nil + case "l", "link", "symlink": + return "symlink", nil + case "other", "special": + return "special", nil + } + return "", fmt.Errorf("unknown type %q (file, dir, symlink, other)", s) +} + +var flagNames = map[string]inventory.Flags{ + "sparse": inventory.FlagSparse, "hardlink": inventory.FlagHardlinked, "hardlinked": inventory.FlagHardlinked, + "error": inventory.FlagError, "mount": inventory.FlagMountPoint, "incomplete": inventory.FlagIncomplete, + "followed": inventory.FlagFollowed, "broken": inventory.FlagBrokenLink, +} + +// number parses sizes ("1.5GiB", "500MB", "10k"), ages ("365d", "2w") and +// dates ("2024-01-31"). Ages and dates are converted to modification times +// (unix nanoseconds) relative to p.now. +func (p *parser) number(f Field, s string) (int64, error) { + switch f { + case FSize, FAllocated: + return ParseSize(s) + case FFiles: + return strconv.ParseInt(s, 10, 64) + case FAge: + d, err := ParseAge(s) + return int64(d), err + case FModified: + for _, layout := range []string{"2006-01-02", "2006-01-02T15:04", "2006-01-02 15:04", time.RFC3339} { + if t, err := time.ParseInLocation(layout, s, time.Local); err == nil { + return t.UnixNano(), nil + } + } + if d, err := ParseAge(s); err == nil { + return p.now.Add(-d).UnixNano(), nil + } + return 0, fmt.Errorf("bad date %q (use YYYY-MM-DD or an age like 30d)", s) + } + return 0, fmt.Errorf("not numeric") +} + +// ParseSize parses a size with optional unit. kB/MB/GB/TB are SI (powers of +// 1000); KiB/MiB/GiB/TiB are IEC (1024). Bare K/M/G/T follow the display +// setting (textutil.SI). +func ParseSize(s string) (int64, error) { + i := 0 + for i < len(s) && (s[i] >= '0' && s[i] <= '9' || s[i] == '.') { + i++ + } + if i == 0 { + return 0, fmt.Errorf("bad size %q", s) + } + v, err := strconv.ParseFloat(s[:i], 64) + if err != nil { + return 0, fmt.Errorf("bad size %q", s) + } + unit := strings.ToLower(strings.TrimSpace(s[i:])) + bare := 1024.0 + if textutil.SI { + bare = 1000 + } + mult := map[string]float64{"": 1, "b": 1, + "k": bare, "m": bare * bare, "g": bare * bare * bare, "t": bare * bare * bare * bare, "p": bare * bare * bare * bare * bare, + "kb": 1e3, "mb": 1e6, "gb": 1e9, "tb": 1e12, "pb": 1e15, + "kib": 1 << 10, "mib": 1 << 20, "gib": 1 << 30, "tib": 1 << 40, "pib": 1 << 50} + m, ok := mult[unit] + if !ok { + return 0, fmt.Errorf("unknown size unit %q", s[i:]) + } + r := v * m + if r > 9e18 { + return 0, fmt.Errorf("size %q too large", s) + } + return int64(r), nil +} + +// ParseAge parses durations like 90s, 30min, 12h, 7d, 2w, 6mo, 1y. +func ParseAge(s string) (time.Duration, error) { + i := 0 + for i < len(s) && (s[i] >= '0' && s[i] <= '9' || s[i] == '.') { + i++ + } + v, err := strconv.ParseFloat(s[:i], 64) + if err != nil { + return 0, fmt.Errorf("bad age %q", s) + } + day := 24 * time.Hour + units := map[string]time.Duration{"s": time.Second, "sec": time.Second, "min": time.Minute, + "h": time.Hour, "d": day, "": day, "w": 7 * day, "m": 30 * day, "mo": 30 * day, "y": 365 * day} + u, ok := units[strings.ToLower(s[i:])] + if !ok { + return 0, fmt.Errorf("unknown age unit %q (s, min, h, d, w, mo, y)", s[i:]) + } + return time.Duration(v * float64(u)), nil +}