diff --git a/internal/inventory/category.go b/internal/inventory/category.go new file mode 100644 index 0000000..17f1ff7 --- /dev/null +++ b/internal/inventory/category.go @@ -0,0 +1,117 @@ +package inventory + +import "strings" + +// Category is a coarse file classification used for colouring and stats. +type Category uint8 + +const ( + CatOther Category = iota + CatImage + CatVideo + CatAudio + CatArchive + CatDiskImage + CatDatabase + CatCode + CatDocument + CatExecutable + CatLog + CatCache + NumCategories +) + +var categoryNames = [NumCategories]string{ + "Other", "Image", "Video", "Audio", "Archive", "Disk image", "Database", + "Source code", "Document", "Executable", "Log", "Cache", +} + +func (c Category) String() string { + if c < NumCategories { + return categoryNames[c] + } + return "Other" +} + +// ParseCategory resolves a category by (case-insensitive) name or prefix. +func ParseCategory(s string) (Category, bool) { + s = strings.ToLower(strings.TrimSpace(s)) + for i, n := range categoryNames { + ln := strings.ToLower(n) + if ln == s || strings.ReplaceAll(ln, " ", "") == s || strings.HasPrefix(ln, s) && len(s) >= 3 { + return Category(i), true + } + } + switch s { + case "vm", "virtual machine", "disk": + return CatDiskImage, true + case "db": + return CatDatabase, true + case "src", "source": + return CatCode, true + case "exe", "binary": + return CatExecutable, true + } + return 0, false +} + +var extCategory = map[string]Category{} + +func init() { + add := func(c Category, exts string) { + for _, e := range strings.Fields(exts) { + extCategory[e] = c + } + } + add(CatImage, "jpg jpeg png gif bmp tif tiff webp heic heif raw cr2 cr3 nef arw dng svg ico psd xcf avif jxl orf rw2") + add(CatVideo, "mp4 mkv mov avi wmv flv webm m4v mpg mpeg ts m2ts vob 3gp ogv mts") + add(CatAudio, "mp3 flac wav ogg oga opus m4a aac wma aiff aif alac mid midi") + add(CatArchive, "zip tar gz tgz bz2 tbz xz txz zst zstd 7z rar lz lz4 lzma cab deb rpm apk jar war cpio pkg snap whl gem") + add(CatDiskImage, "iso img qcow qcow2 vmdk vdi vhd vhdx hdd ova ovf dmg raw vmem vmsn nvram sav") + add(CatDatabase, "db sqlite sqlite3 db3 mdb accdb frm ibd myd myi ldf mdf ndf dbf wal kdbx realm lmdb rdb aof") + add(CatCode, "c h cc cpp cxx hpp hh go rs py pyc js mjs cjs ts tsx jsx java kt kts scala rb php pl pm sh bash zsh fish lua swift m mm cs fs vb r jl hs ml ex exs erl clj dart sql css scss sass less html htm vue svelte json yaml yml toml xml proto gradle cmake mk ipynb o a lib obj class") + add(CatDocument, "pdf doc docx odt rtf txt md rst tex xls xlsx ods csv tsv ppt pptx odp epub mobi djvu pages numbers key org adoc") + add(CatExecutable, "exe dll so dylib bin msi app elf ko sys efi appimage com") + add(CatLog, "log journal") + add(CatCache, "cache tmp temp swp bak old part crdownload") +} + +// CategoryFor classifies a file by its (lower-case) extension and by +// whether it lives beneath a cache or log directory. +func CategoryFor(ext string, inCache, inLog bool) Category { + c, ok := extCategory[ext] + switch { + case inCache && (!ok || c == CatOther || c == CatCode): + return CatCache + case inLog && (!ok || c == CatArchive || c == CatOther): + return CatLog + case ok: + return c + } + return CatOther +} + +// PathHints reports whether a directory path contains a cache or log +// component. It is evaluated once per directory, not per file. +func PathHints(path string) (inCache, inLog bool) { + lp := strings.ToLower(path) + for _, part := range strings.FieldsFunc(lp, func(r rune) bool { return r == '/' || r == '\\' }) { + switch part { + case "cache", ".cache", "caches", "__pycache__", ".npm", ".gradle", "inetcache", "temp", "tmp": + inCache = true + case "log", "logs", ".logs", "journal": + inLog = true + } + } + return +} + +// Ext returns the lower-case extension of name without the dot ("" if none, +// or if the name is a dotfile like ".bashrc"). +func Ext(name string) string { + i := strings.LastIndexByte(name, '.') + if i <= 0 || i == len(name)-1 || len(name)-i > 16 { + return "" + } + return strings.ToLower(name[i+1:]) +} diff --git a/internal/inventory/query.go b/internal/inventory/query.go new file mode 100644 index 0000000..26ef11a --- /dev/null +++ b/internal/inventory/query.go @@ -0,0 +1,176 @@ +package inventory + +import ( + "container/heap" + "sort" +) + +// SizeMode selects which size drives layout and sorting. +type SizeMode uint8 + +const ( + SizeAllocated SizeMode = iota // disk usage (default) + SizeLogical // apparent size +) + +func (m SizeMode) String() string { + if m == SizeLogical { + return "logical" + } + return "allocated" +} + +// Total returns the node's aggregate size in the given mode. +func (n *Node) Total(m SizeMode) int64 { + if m == SizeLogical { + return n.TotSize + } + return n.TotAlloc +} + +// Own returns the node's own size in the given mode. +func (n *Node) Own(m SizeMode) int64 { + if m == SizeLogical { + return n.Size + } + return n.Alloc +} + +type ranked struct { + id NodeID + key int64 +} + +type minHeap []ranked + +func (h minHeap) Len() int { return len(h) } +func (h minHeap) Less(i, j int) bool { + if h[i].key != h[j].key { + return h[i].key < h[j].key + } + return h[i].id > h[j].id +} +func (h minHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } +func (h *minHeap) Push(x any) { *h = append(*h, x.(ranked)) } +func (h *minHeap) Pop() any { + old := *h + x := old[len(old)-1] + *h = old[:len(old)-1] + return x +} + +// TopK keeps the k entries with the largest keys; O(n log k). +type TopK struct { + k int + h minHeap +} + +// NewTopK creates a collector for the k largest keys. +func NewTopK(k int) *TopK { return &TopK{k: k} } + +// Offer considers one candidate. +func (t *TopK) Offer(id NodeID, key int64) { + if t.k <= 0 { + return + } + if len(t.h) < t.k { + heap.Push(&t.h, ranked{id, key}) + return + } + if r := t.h[0]; key > r.key || key == r.key && id < r.id { + t.h[0] = ranked{id, key} + heap.Fix(&t.h, 0) + } +} + +// Sorted returns the collected IDs, largest key first (ties by ID). +func (t *TopK) Sorted() []NodeID { + rs := append([]ranked(nil), t.h...) + sort.Slice(rs, func(i, j int) bool { + if rs[i].key != rs[j].key { + return rs[i].key > rs[j].key + } + return rs[i].id < rs[j].id + }) + out := make([]NodeID, len(rs)) + for i, r := range rs { + out[i] = r.id + } + return out +} + +// Walk visits id and all its descendants depth-first (iteratively). +func (t *Tree) Walk(id NodeID, fn func(NodeID, *Node) bool) { + stack := []NodeID{id} + for len(stack) > 0 { + cur := stack[len(stack)-1] + stack = stack[:len(stack)-1] + n := t.Node(cur) + if !fn(cur, n) { + continue + } + for c := n.FirstChild; c != NoNode; c = t.Node(c).NextSibling { + stack = append(stack, c) + } + } +} + +// Top returns the k nodes beneath (and including) root with the largest key, +// considering only nodes accepted by keep. +func (t *Tree) Top(root NodeID, k int, keep func(*Node) bool, key func(*Node) int64) []NodeID { + tk := NewTopK(k) + t.Walk(root, func(id NodeID, n *Node) bool { + if keep(n) { + tk.Offer(id, key(n)) + } + return true + }) + return tk.Sorted() +} + +// SortedChildren returns the children of id ordered by size descending. +func (t *Tree) SortedChildren(id NodeID, m SizeMode) []NodeID { + var out []NodeID + t.Children(id, func(c NodeID, _ *Node) { out = append(out, c) }) + sort.Slice(out, func(i, j int) bool { + a, b := t.Node(out[i]).Total(m), t.Node(out[j]).Total(m) + if a != b { + return a > b + } + return out[i] < out[j] + }) + return out +} + +// ExtStat summarises one extension. +type ExtStat struct { + Ext string + Cat Category + Count int64 + Size int64 // logical + Alloc int64 +} + +// ExtStats aggregates file statistics by extension beneath root. +func (t *Tree) ExtStats(root NodeID) []ExtStat { + stats := make([]ExtStat, t.ExtCount()) + t.Walk(root, func(_ NodeID, n *Node) bool { + if n.Kind == KindFile && n.Flags&FlagHardlinkDup == 0 { + s := &stats[n.Ext] + s.Count++ + s.Size += n.Size + s.Alloc += n.Alloc + s.Cat = n.Cat + } + return true + }) + out := stats[:0] + for i, s := range stats { + if s.Count > 0 { + s.Ext = t.exts[i] + s.Cat = CategoryFor(s.Ext, false, false) + out = append(out, s) + } + } + return out +} diff --git a/internal/inventory/query_test.go b/internal/inventory/query_test.go new file mode 100644 index 0000000..8cd96f1 --- /dev/null +++ b/internal/inventory/query_test.go @@ -0,0 +1,42 @@ +package inventory + +import "testing" + +func TestTopKAndRecompute(t *testing.T) { + tr := New("/r", KindDir) + d := tr.Add(0, "d", KindDir) + for i := range 100 { + id := tr.Add(d, "f", KindFile) + n := tr.Node(id) + n.Size, n.Alloc = int64(i), int64(i) + } + tr.Recompute() + if got := tr.Node(0).TotSize; got != 4950 { + t.Fatalf("total %d", got) + } + if tr.Node(0).Files != 100 || tr.Node(0).Dirs != 1 { + t.Fatalf("counts %d %d", tr.Node(0).Files, tr.Node(0).Dirs) + } + top := tr.Top(0, 3, func(n *Node) bool { return n.Kind == KindFile }, func(n *Node) int64 { return n.Size }) + if len(top) != 3 || tr.Node(top[0]).Size != 99 || tr.Node(top[2]).Size != 97 { + t.Fatalf("top %v", top) + } + if p := tr.Path(top[0]); p != "/r/d/f" { + t.Fatalf("path %q", p) + } +} + +func TestExtAndCategory(t *testing.T) { + cases := map[string]string{"a.QCOW2": "qcow2", ".bashrc": "", "noext": "", "x.tar.gz": "gz", "trail.": ""} + for in, want := range cases { + if got := Ext(in); got != want { + t.Errorf("Ext(%q)=%q want %q", in, got, want) + } + } + if CategoryFor("qcow2", false, false) != CatDiskImage || CategoryFor("bin", true, false) != CatExecutable { + t.Fatal("category mapping") + } + if CategoryFor("", true, false) != CatCache { + t.Fatal("cache hint") + } +} diff --git a/internal/inventory/stats.go b/internal/inventory/stats.go new file mode 100644 index 0000000..bce1573 --- /dev/null +++ b/internal/inventory/stats.go @@ -0,0 +1,106 @@ +package inventory + +import ( + "errors" + "io/fs" + "syscall" + "time" +) + +// ErrKind classifies scan errors for reporting. +type ErrKind uint8 + +const ( + ErrPermission ErrKind = iota + ErrVanished + ErrIO + ErrLoop + ErrOther + NumErrKinds +) + +func (k ErrKind) String() string { + return [...]string{"Permission denied", "Vanished during scan", "I/O error", "Symlink loop", "Other error"}[k] +} + +// Classify maps an OS error to an ErrKind. +func Classify(err error) ErrKind { + switch { + case errors.Is(err, fs.ErrPermission): + return ErrPermission + case errors.Is(err, fs.ErrNotExist): + return ErrVanished + case errors.Is(err, syscall.ELOOP): + return ErrLoop + case errors.Is(err, syscall.EIO): + return ErrIO + } + return ErrOther +} + +// ErrorRecord is one recorded scan error. +type ErrorRecord struct { + Node NodeID // node the error is attached to (the directory for listing failures) + Name string // entry name when no node was created (e.g. vanished) + Kind ErrKind + Msg string +} + +// MaxErrorRecords caps the detailed error list; counts are always exact. +const MaxErrorRecords = 100_000 + +// Stats describes a scan as a whole. +type Stats struct { + Root string + Start, End time.Time + Complete bool // scan finished (possibly with errors) + Cancelled bool + + Files, Dirs, Symlinks, Others int64 + + ErrCounts [NumErrKinds]int64 + Errors []ErrorRecord + Excluded int64 + SkippedMounts int64 + VirtualSkipped int64 + LoopsSkipped int64 + BrokenLinks int64 + HardlinkDups int64 + Unscanned int64 // directories never read because the scan was cancelled + + Excludes []string + OneFileSystem bool + Follow string + Workers int + FromSnapshot string +} + +// TotalErrors sums all error kinds. +func (s *Stats) TotalErrors() int64 { + var t int64 + for _, c := range s.ErrCounts { + t += c + } + return t +} + +// AddError records an error (the detailed list is capped). +func (s *Stats) AddError(r ErrorRecord) { + s.ErrCounts[r.Kind]++ + if len(s.Errors) < MaxErrorRecords { + s.Errors = append(s.Errors, r) + } +} + +// Elapsed is the scan duration so far (or total once finished). +func (s *Stats) Elapsed() time.Duration { + if !s.End.IsZero() { + return s.End.Sub(s.Start) + } + return time.Since(s.Start) +} + +// Incomplete reports whether totals may under-count real usage. +func (s *Stats) Incomplete() bool { + return !s.Complete || s.Cancelled || s.TotalErrors() > 0 || s.Unscanned > 0 +} diff --git a/internal/inventory/tree.go b/internal/inventory/tree.go new file mode 100644 index 0000000..3f2a71d --- /dev/null +++ b/internal/inventory/tree.go @@ -0,0 +1,326 @@ +// Package inventory is the in-memory filesystem model: an append-only arena +// of nodes addressed by stable NodeIDs, with incrementally maintained +// directory aggregates. +// +// Concurrency: a Tree has exactly one writer (the scan controller or a +// snapshot loader). Other goroutines must hold RLock while reading node data. +// Nodes are never removed, so a NodeID stays valid for the Tree's lifetime. +package inventory + +import ( + "path/filepath" + "slices" + "sync" + "time" +) + +// NodeID addresses a node within one Tree. The root is always 0. +type NodeID uint32 + +// NoNode is the nil NodeID. +const NoNode NodeID = ^NodeID(0) + +// Kind is the node type. +type Kind uint8 + +const ( + KindFile Kind = iota + KindDir + KindSymlink + KindOther // devices, sockets, fifos, reparse points +) + +func (k Kind) String() string { + return [...]string{"File", "Directory", "Symlink", "Special"}[k&3] +} + +// Flags record per-node facts. +type Flags uint16 + +const ( + FlagHardlinkDup Flags = 1 << iota // additional link to an inode counted elsewhere + FlagMountPoint // directory on a different device than its parent + FlagVirtualFS // virtual filesystem, not descended + FlagSkippedFS // other filesystem, not descended (--one-file-system) + FlagSparse // allocated < logical (sparse or compressed) + FlagError // stat or read failed + FlagIncomplete // directory contents not (fully) read + FlagScanned // directory fully read + FlagLoop // directory already visited via another path + FlagFollowed // reached through a followed symlink + FlagAllocUnknown // platform cannot report allocated size + FlagBrokenLink // symlink target does not exist + FlagHardlinked // regular file with link count > 1 +) + +// Node is one filesystem object. Sizes are own sizes; Tot* fields are +// aggregates including the node itself and (for directories) everything +// beneath it. +type Node struct { + Parent, FirstChild, NextSibling NodeID + dir uint32 // index into dir side table (directories only) + + Name string // single path component; the root holds the full root path + + Size, Alloc int64 // own logical / allocated bytes + TotSize, TotAlloc int64 // aggregates (hard-link duplicates contribute 0) + MTime int64 // unix nanoseconds + + Files, Dirs uint32 // recursive counts beneath (dirs excludes self) + Errors uint32 // recursive error count beneath and including self + Mode uint32 // fs.FileMode bits + UID, GID uint32 + Nlink uint32 + + Ext uint16 + Flags Flags + Kind Kind + Cat Category +} + +// IsDir reports whether the node is a directory. +func (n *Node) IsDir() bool { return n.Kind == KindDir } + +// Delta is an aggregate change applied to a node and all its ancestors. +type Delta struct { + Size, Alloc int64 + Files, Dirs, Errors int64 + Cat [NumCategories]int64 +} + +const chunkBits = 16 +const chunkSize = 1 << chunkBits + +// chunked is a growable array that never moves existing elements. +type chunked[T any] struct { + c [][]T + n int +} + +func (c *chunked[T]) add(v T) uint32 { + if c.n&(chunkSize-1) == 0 && c.n>>chunkBits == len(c.c) { + c.c = append(c.c, make([]T, 0, chunkSize)) + } + ch := &c.c[c.n>>chunkBits] + *ch = append(*ch, v) + c.n++ + return uint32(c.n - 1) +} + +func (c *chunked[T]) at(i uint32) *T { return &c.c[i>>chunkBits][i&(chunkSize-1)] } + +// Tree is the inventory for one scan root. +type Tree struct { + sync.RWMutex + + nodes chunked[Node] + dirs chunked[[NumCategories]int64] + + exts []string + extIdx map[string]uint16 + + Stats Stats +} + +// New creates a tree whose root has the given full path and kind. +func New(rootPath string, kind Kind) *Tree { + t := &Tree{exts: []string{""}, extIdx: map[string]uint16{"": 0}} + t.Stats.Root = rootPath + t.Stats.Start = time.Now() + t.Add(NoNode, rootPath, kind) + return t +} + +// Len returns the number of nodes. +func (t *Tree) Len() int { return t.nodes.n } + +// Node returns the node for id. The pointer stays valid; reading its fields +// concurrently with the writer requires RLock. +func (t *Tree) Node(id NodeID) *Node { return t.nodes.at(uint32(id)) } + +// Root returns the root NodeID (always 0). +func (t *Tree) Root() NodeID { return 0 } + +// Add appends a child node and links it under parent. Only the writer may +// call it. It does not touch aggregates; use Propagate. +func (t *Tree) Add(parent NodeID, name string, kind Kind) NodeID { + n := Node{Parent: parent, FirstChild: NoNode, NextSibling: NoNode, Name: name, Kind: kind} + if kind == KindDir { + n.dir = t.dirs.add([NumCategories]int64{}) + } else { + n.Ext = t.internExt(Ext(name)) + } + id := NodeID(t.nodes.add(n)) + if parent != NoNode { + p := t.Node(parent) + t.Node(id).NextSibling = p.FirstChild + p.FirstChild = id + } + return id +} + +func (t *Tree) internExt(e string) uint16 { + if i, ok := t.extIdx[e]; ok { + return i + } + if len(t.exts) >= 1<<16-1 { + return 0 + } + t.exts = append(t.exts, e) + t.extIdx[e] = uint16(len(t.exts) - 1) + return uint16(len(t.exts) - 1) +} + +// ExtName returns the extension string for a node ("" if none). +func (t *Tree) ExtName(n *Node) string { return t.exts[n.Ext] } + +// ExtCount returns the number of interned extensions (including ""). +func (t *Tree) ExtCount() int { return len(t.exts) } + +// ExtByIndex returns an interned extension. +func (t *Tree) ExtByIndex(i int) string { return t.exts[i] } + +// CatSizes returns the per-category allocated bytes beneath a directory. +func (t *Tree) CatSizes(id NodeID) *[NumCategories]int64 { + n := t.Node(id) + if n.Kind != KindDir { + return nil + } + return t.dirs.at(n.dir) +} + +// Propagate adds d to id and every ancestor. Cost is O(depth); callers batch +// many entries into one Delta. +func (t *Tree) Propagate(id NodeID, d Delta) { + for id != NoNode { + n := t.Node(id) + n.TotSize += d.Size + n.TotAlloc += d.Alloc + n.Files = addU32(n.Files, d.Files) + n.Dirs = addU32(n.Dirs, d.Dirs) + n.Errors = addU32(n.Errors, d.Errors) + if n.Kind == KindDir { + cs := t.dirs.at(n.dir) + for i, v := range d.Cat { + cs[i] += v + } + } + id = n.Parent + } +} + +func addU32(a uint32, d int64) uint32 { + v := int64(a) + d + if v < 0 { + return 0 + } + if v > 1<<32-1 { + return 1<<32 - 1 + } + return uint32(v) +} + +// Children calls fn for each child of id (in reverse insertion order). +func (t *Tree) Children(id NodeID, fn func(NodeID, *Node)) { + for c := t.Node(id).FirstChild; c != NoNode; { + n := t.Node(c) + fn(c, n) + c = n.NextSibling + } +} + +// ChildCount returns the number of direct children. +func (t *Tree) ChildCount(id NodeID) int { + k := 0 + for c := t.Node(id).FirstChild; c != NoNode; c = t.Node(c).NextSibling { + k++ + } + return k +} + +// Path reconstructs the full path of id. +func (t *Tree) Path(id NodeID) string { + var parts []string + for id != NoNode { + n := t.Node(id) + parts = append(parts, n.Name) + id = n.Parent + } + slices.Reverse(parts) + return filepath.Join(parts...) +} + +// Ancestors returns the chain root..id inclusive. +func (t *Tree) Ancestors(id NodeID) []NodeID { + var out []NodeID + for id != NoNode { + out = append(out, id) + id = t.Node(id).Parent + } + slices.Reverse(out) + return out +} + +// IsAncestor reports whether a is id or an ancestor of id. +func (t *Tree) IsAncestor(a, id NodeID) bool { + for id != NoNode { + if id == a { + return true + } + id = t.Node(id).Parent + } + return false +} + +// Depth returns the number of ancestors of id. +func (t *Tree) Depth(id NodeID) int { + d := 0 + for id = t.Node(id).Parent; id != NoNode; id = t.Node(id).Parent { + d++ + } + return d +} + +// Recompute rebuilds all aggregates bottom-up. It relies on the invariant +// that a parent's ID is smaller than its children's, which both the scanner +// and the snapshot loader guarantee. Used after loading a snapshot. +func (t *Tree) Recompute() { + for i := 0; i < t.Len(); i++ { + n := t.Node(NodeID(i)) + n.TotSize, n.TotAlloc, n.Files, n.Dirs = 0, 0, 0, 0 + n.Errors = 0 + if n.Flags&FlagError != 0 { + n.Errors = 1 + } + if n.Flags&FlagHardlinkDup == 0 { + n.TotSize, n.TotAlloc = n.Size, n.Alloc + } + if n.Kind == KindDir { + *t.dirs.at(n.dir) = [NumCategories]int64{} + } + } + for i := t.Len() - 1; i > 0; i-- { + n := t.Node(NodeID(i)) + p := t.Node(n.Parent) + p.TotSize += n.TotSize + p.TotAlloc += n.TotAlloc + p.Errors += n.Errors + cs := t.dirs.at(p.dir) + if n.Kind == KindDir { + p.Files += n.Files + p.Dirs += n.Dirs + 1 + for c, v := range t.dirs.at(n.dir) { + cs[c] += v + } + } else { + p.Files++ + if n.Flags&FlagHardlinkDup == 0 { + cs[n.Cat] += n.Alloc + } + } + } +} + +// ExtIndexer returns a function interning extension strings; used by +// loaders that rebuild a tree from external data. +func (t *Tree) ExtIndexer() func(string) uint16 { return t.internExt }