Add filesystem inventory model
Append-only chunked node arena with stable NodeIDs, intrusive child lists, per-directory category totals and batched ancestor propagation.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,117 @@
|
||||
package inventory
|
||||
|
||||
import "strings"
|
||||
|
||||
// Category is a coarse file classification used for colouring and stats.
|
||||
type Category uint8
|
||||
|
||||
const (
|
||||
CatOther Category = iota
|
||||
CatImage
|
||||
CatVideo
|
||||
CatAudio
|
||||
CatArchive
|
||||
CatDiskImage
|
||||
CatDatabase
|
||||
CatCode
|
||||
CatDocument
|
||||
CatExecutable
|
||||
CatLog
|
||||
CatCache
|
||||
NumCategories
|
||||
)
|
||||
|
||||
var categoryNames = [NumCategories]string{
|
||||
"Other", "Image", "Video", "Audio", "Archive", "Disk image", "Database",
|
||||
"Source code", "Document", "Executable", "Log", "Cache",
|
||||
}
|
||||
|
||||
func (c Category) String() string {
|
||||
if c < NumCategories {
|
||||
return categoryNames[c]
|
||||
}
|
||||
return "Other"
|
||||
}
|
||||
|
||||
// ParseCategory resolves a category by (case-insensitive) name or prefix.
|
||||
func ParseCategory(s string) (Category, bool) {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
for i, n := range categoryNames {
|
||||
ln := strings.ToLower(n)
|
||||
if ln == s || strings.ReplaceAll(ln, " ", "") == s || strings.HasPrefix(ln, s) && len(s) >= 3 {
|
||||
return Category(i), true
|
||||
}
|
||||
}
|
||||
switch s {
|
||||
case "vm", "virtual machine", "disk":
|
||||
return CatDiskImage, true
|
||||
case "db":
|
||||
return CatDatabase, true
|
||||
case "src", "source":
|
||||
return CatCode, true
|
||||
case "exe", "binary":
|
||||
return CatExecutable, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
var extCategory = map[string]Category{}
|
||||
|
||||
func init() {
|
||||
add := func(c Category, exts string) {
|
||||
for _, e := range strings.Fields(exts) {
|
||||
extCategory[e] = c
|
||||
}
|
||||
}
|
||||
add(CatImage, "jpg jpeg png gif bmp tif tiff webp heic heif raw cr2 cr3 nef arw dng svg ico psd xcf avif jxl orf rw2")
|
||||
add(CatVideo, "mp4 mkv mov avi wmv flv webm m4v mpg mpeg ts m2ts vob 3gp ogv mts")
|
||||
add(CatAudio, "mp3 flac wav ogg oga opus m4a aac wma aiff aif alac mid midi")
|
||||
add(CatArchive, "zip tar gz tgz bz2 tbz xz txz zst zstd 7z rar lz lz4 lzma cab deb rpm apk jar war cpio pkg snap whl gem")
|
||||
add(CatDiskImage, "iso img qcow qcow2 vmdk vdi vhd vhdx hdd ova ovf dmg raw vmem vmsn nvram sav")
|
||||
add(CatDatabase, "db sqlite sqlite3 db3 mdb accdb frm ibd myd myi ldf mdf ndf dbf wal kdbx realm lmdb rdb aof")
|
||||
add(CatCode, "c h cc cpp cxx hpp hh go rs py pyc js mjs cjs ts tsx jsx java kt kts scala rb php pl pm sh bash zsh fish lua swift m mm cs fs vb r jl hs ml ex exs erl clj dart sql css scss sass less html htm vue svelte json yaml yml toml xml proto gradle cmake mk ipynb o a lib obj class")
|
||||
add(CatDocument, "pdf doc docx odt rtf txt md rst tex xls xlsx ods csv tsv ppt pptx odp epub mobi djvu pages numbers key org adoc")
|
||||
add(CatExecutable, "exe dll so dylib bin msi app elf ko sys efi appimage com")
|
||||
add(CatLog, "log journal")
|
||||
add(CatCache, "cache tmp temp swp bak old part crdownload")
|
||||
}
|
||||
|
||||
// CategoryFor classifies a file by its (lower-case) extension and by
|
||||
// whether it lives beneath a cache or log directory.
|
||||
func CategoryFor(ext string, inCache, inLog bool) Category {
|
||||
c, ok := extCategory[ext]
|
||||
switch {
|
||||
case inCache && (!ok || c == CatOther || c == CatCode):
|
||||
return CatCache
|
||||
case inLog && (!ok || c == CatArchive || c == CatOther):
|
||||
return CatLog
|
||||
case ok:
|
||||
return c
|
||||
}
|
||||
return CatOther
|
||||
}
|
||||
|
||||
// PathHints reports whether a directory path contains a cache or log
|
||||
// component. It is evaluated once per directory, not per file.
|
||||
func PathHints(path string) (inCache, inLog bool) {
|
||||
lp := strings.ToLower(path)
|
||||
for _, part := range strings.FieldsFunc(lp, func(r rune) bool { return r == '/' || r == '\\' }) {
|
||||
switch part {
|
||||
case "cache", ".cache", "caches", "__pycache__", ".npm", ".gradle", "inetcache", "temp", "tmp":
|
||||
inCache = true
|
||||
case "log", "logs", ".logs", "journal":
|
||||
inLog = true
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Ext returns the lower-case extension of name without the dot ("" if none,
|
||||
// or if the name is a dotfile like ".bashrc").
|
||||
func Ext(name string) string {
|
||||
i := strings.LastIndexByte(name, '.')
|
||||
if i <= 0 || i == len(name)-1 || len(name)-i > 16 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToLower(name[i+1:])
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"container/heap"
|
||||
"sort"
|
||||
)
|
||||
|
||||
// SizeMode selects which size drives layout and sorting.
|
||||
type SizeMode uint8
|
||||
|
||||
const (
|
||||
SizeAllocated SizeMode = iota // disk usage (default)
|
||||
SizeLogical // apparent size
|
||||
)
|
||||
|
||||
func (m SizeMode) String() string {
|
||||
if m == SizeLogical {
|
||||
return "logical"
|
||||
}
|
||||
return "allocated"
|
||||
}
|
||||
|
||||
// Total returns the node's aggregate size in the given mode.
|
||||
func (n *Node) Total(m SizeMode) int64 {
|
||||
if m == SizeLogical {
|
||||
return n.TotSize
|
||||
}
|
||||
return n.TotAlloc
|
||||
}
|
||||
|
||||
// Own returns the node's own size in the given mode.
|
||||
func (n *Node) Own(m SizeMode) int64 {
|
||||
if m == SizeLogical {
|
||||
return n.Size
|
||||
}
|
||||
return n.Alloc
|
||||
}
|
||||
|
||||
type ranked struct {
|
||||
id NodeID
|
||||
key int64
|
||||
}
|
||||
|
||||
type minHeap []ranked
|
||||
|
||||
func (h minHeap) Len() int { return len(h) }
|
||||
func (h minHeap) Less(i, j int) bool {
|
||||
if h[i].key != h[j].key {
|
||||
return h[i].key < h[j].key
|
||||
}
|
||||
return h[i].id > h[j].id
|
||||
}
|
||||
func (h minHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] }
|
||||
func (h *minHeap) Push(x any) { *h = append(*h, x.(ranked)) }
|
||||
func (h *minHeap) Pop() any {
|
||||
old := *h
|
||||
x := old[len(old)-1]
|
||||
*h = old[:len(old)-1]
|
||||
return x
|
||||
}
|
||||
|
||||
// TopK keeps the k entries with the largest keys; O(n log k).
|
||||
type TopK struct {
|
||||
k int
|
||||
h minHeap
|
||||
}
|
||||
|
||||
// NewTopK creates a collector for the k largest keys.
|
||||
func NewTopK(k int) *TopK { return &TopK{k: k} }
|
||||
|
||||
// Offer considers one candidate.
|
||||
func (t *TopK) Offer(id NodeID, key int64) {
|
||||
if t.k <= 0 {
|
||||
return
|
||||
}
|
||||
if len(t.h) < t.k {
|
||||
heap.Push(&t.h, ranked{id, key})
|
||||
return
|
||||
}
|
||||
if r := t.h[0]; key > r.key || key == r.key && id < r.id {
|
||||
t.h[0] = ranked{id, key}
|
||||
heap.Fix(&t.h, 0)
|
||||
}
|
||||
}
|
||||
|
||||
// Sorted returns the collected IDs, largest key first (ties by ID).
|
||||
func (t *TopK) Sorted() []NodeID {
|
||||
rs := append([]ranked(nil), t.h...)
|
||||
sort.Slice(rs, func(i, j int) bool {
|
||||
if rs[i].key != rs[j].key {
|
||||
return rs[i].key > rs[j].key
|
||||
}
|
||||
return rs[i].id < rs[j].id
|
||||
})
|
||||
out := make([]NodeID, len(rs))
|
||||
for i, r := range rs {
|
||||
out[i] = r.id
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// Walk visits id and all its descendants depth-first (iteratively).
|
||||
func (t *Tree) Walk(id NodeID, fn func(NodeID, *Node) bool) {
|
||||
stack := []NodeID{id}
|
||||
for len(stack) > 0 {
|
||||
cur := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
n := t.Node(cur)
|
||||
if !fn(cur, n) {
|
||||
continue
|
||||
}
|
||||
for c := n.FirstChild; c != NoNode; c = t.Node(c).NextSibling {
|
||||
stack = append(stack, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Top returns the k nodes beneath (and including) root with the largest key,
|
||||
// considering only nodes accepted by keep.
|
||||
func (t *Tree) Top(root NodeID, k int, keep func(*Node) bool, key func(*Node) int64) []NodeID {
|
||||
tk := NewTopK(k)
|
||||
t.Walk(root, func(id NodeID, n *Node) bool {
|
||||
if keep(n) {
|
||||
tk.Offer(id, key(n))
|
||||
}
|
||||
return true
|
||||
})
|
||||
return tk.Sorted()
|
||||
}
|
||||
|
||||
// SortedChildren returns the children of id ordered by size descending.
|
||||
func (t *Tree) SortedChildren(id NodeID, m SizeMode) []NodeID {
|
||||
var out []NodeID
|
||||
t.Children(id, func(c NodeID, _ *Node) { out = append(out, c) })
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := t.Node(out[i]).Total(m), t.Node(out[j]).Total(m)
|
||||
if a != b {
|
||||
return a > b
|
||||
}
|
||||
return out[i] < out[j]
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
// ExtStat summarises one extension.
|
||||
type ExtStat struct {
|
||||
Ext string
|
||||
Cat Category
|
||||
Count int64
|
||||
Size int64 // logical
|
||||
Alloc int64
|
||||
}
|
||||
|
||||
// ExtStats aggregates file statistics by extension beneath root.
|
||||
func (t *Tree) ExtStats(root NodeID) []ExtStat {
|
||||
stats := make([]ExtStat, t.ExtCount())
|
||||
t.Walk(root, func(_ NodeID, n *Node) bool {
|
||||
if n.Kind == KindFile && n.Flags&FlagHardlinkDup == 0 {
|
||||
s := &stats[n.Ext]
|
||||
s.Count++
|
||||
s.Size += n.Size
|
||||
s.Alloc += n.Alloc
|
||||
s.Cat = n.Cat
|
||||
}
|
||||
return true
|
||||
})
|
||||
out := stats[:0]
|
||||
for i, s := range stats {
|
||||
if s.Count > 0 {
|
||||
s.Ext = t.exts[i]
|
||||
s.Cat = CategoryFor(s.Ext, false, false)
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package inventory
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestTopKAndRecompute(t *testing.T) {
|
||||
tr := New("/r", KindDir)
|
||||
d := tr.Add(0, "d", KindDir)
|
||||
for i := range 100 {
|
||||
id := tr.Add(d, "f", KindFile)
|
||||
n := tr.Node(id)
|
||||
n.Size, n.Alloc = int64(i), int64(i)
|
||||
}
|
||||
tr.Recompute()
|
||||
if got := tr.Node(0).TotSize; got != 4950 {
|
||||
t.Fatalf("total %d", got)
|
||||
}
|
||||
if tr.Node(0).Files != 100 || tr.Node(0).Dirs != 1 {
|
||||
t.Fatalf("counts %d %d", tr.Node(0).Files, tr.Node(0).Dirs)
|
||||
}
|
||||
top := tr.Top(0, 3, func(n *Node) bool { return n.Kind == KindFile }, func(n *Node) int64 { return n.Size })
|
||||
if len(top) != 3 || tr.Node(top[0]).Size != 99 || tr.Node(top[2]).Size != 97 {
|
||||
t.Fatalf("top %v", top)
|
||||
}
|
||||
if p := tr.Path(top[0]); p != "/r/d/f" {
|
||||
t.Fatalf("path %q", p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtAndCategory(t *testing.T) {
|
||||
cases := map[string]string{"a.QCOW2": "qcow2", ".bashrc": "", "noext": "", "x.tar.gz": "gz", "trail.": ""}
|
||||
for in, want := range cases {
|
||||
if got := Ext(in); got != want {
|
||||
t.Errorf("Ext(%q)=%q want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
if CategoryFor("qcow2", false, false) != CatDiskImage || CategoryFor("bin", true, false) != CatExecutable {
|
||||
t.Fatal("category mapping")
|
||||
}
|
||||
if CategoryFor("", true, false) != CatCache {
|
||||
t.Fatal("cache hint")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io/fs"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
// ErrKind classifies scan errors for reporting.
|
||||
type ErrKind uint8
|
||||
|
||||
const (
|
||||
ErrPermission ErrKind = iota
|
||||
ErrVanished
|
||||
ErrIO
|
||||
ErrLoop
|
||||
ErrOther
|
||||
NumErrKinds
|
||||
)
|
||||
|
||||
func (k ErrKind) String() string {
|
||||
return [...]string{"Permission denied", "Vanished during scan", "I/O error", "Symlink loop", "Other error"}[k]
|
||||
}
|
||||
|
||||
// Classify maps an OS error to an ErrKind.
|
||||
func Classify(err error) ErrKind {
|
||||
switch {
|
||||
case errors.Is(err, fs.ErrPermission):
|
||||
return ErrPermission
|
||||
case errors.Is(err, fs.ErrNotExist):
|
||||
return ErrVanished
|
||||
case errors.Is(err, syscall.ELOOP):
|
||||
return ErrLoop
|
||||
case errors.Is(err, syscall.EIO):
|
||||
return ErrIO
|
||||
}
|
||||
return ErrOther
|
||||
}
|
||||
|
||||
// ErrorRecord is one recorded scan error.
|
||||
type ErrorRecord struct {
|
||||
Node NodeID // node the error is attached to (the directory for listing failures)
|
||||
Name string // entry name when no node was created (e.g. vanished)
|
||||
Kind ErrKind
|
||||
Msg string
|
||||
}
|
||||
|
||||
// MaxErrorRecords caps the detailed error list; counts are always exact.
|
||||
const MaxErrorRecords = 100_000
|
||||
|
||||
// Stats describes a scan as a whole.
|
||||
type Stats struct {
|
||||
Root string
|
||||
Start, End time.Time
|
||||
Complete bool // scan finished (possibly with errors)
|
||||
Cancelled bool
|
||||
|
||||
Files, Dirs, Symlinks, Others int64
|
||||
|
||||
ErrCounts [NumErrKinds]int64
|
||||
Errors []ErrorRecord
|
||||
Excluded int64
|
||||
SkippedMounts int64
|
||||
VirtualSkipped int64
|
||||
LoopsSkipped int64
|
||||
BrokenLinks int64
|
||||
HardlinkDups int64
|
||||
Unscanned int64 // directories never read because the scan was cancelled
|
||||
|
||||
Excludes []string
|
||||
OneFileSystem bool
|
||||
Follow string
|
||||
Workers int
|
||||
FromSnapshot string
|
||||
}
|
||||
|
||||
// TotalErrors sums all error kinds.
|
||||
func (s *Stats) TotalErrors() int64 {
|
||||
var t int64
|
||||
for _, c := range s.ErrCounts {
|
||||
t += c
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// AddError records an error (the detailed list is capped).
|
||||
func (s *Stats) AddError(r ErrorRecord) {
|
||||
s.ErrCounts[r.Kind]++
|
||||
if len(s.Errors) < MaxErrorRecords {
|
||||
s.Errors = append(s.Errors, r)
|
||||
}
|
||||
}
|
||||
|
||||
// Elapsed is the scan duration so far (or total once finished).
|
||||
func (s *Stats) Elapsed() time.Duration {
|
||||
if !s.End.IsZero() {
|
||||
return s.End.Sub(s.Start)
|
||||
}
|
||||
return time.Since(s.Start)
|
||||
}
|
||||
|
||||
// Incomplete reports whether totals may under-count real usage.
|
||||
func (s *Stats) Incomplete() bool {
|
||||
return !s.Complete || s.Cancelled || s.TotalErrors() > 0 || s.Unscanned > 0
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
// Package inventory is the in-memory filesystem model: an append-only arena
|
||||
// of nodes addressed by stable NodeIDs, with incrementally maintained
|
||||
// directory aggregates.
|
||||
//
|
||||
// Concurrency: a Tree has exactly one writer (the scan controller or a
|
||||
// snapshot loader). Other goroutines must hold RLock while reading node data.
|
||||
// Nodes are never removed, so a NodeID stays valid for the Tree's lifetime.
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NodeID addresses a node within one Tree. The root is always 0.
|
||||
type NodeID uint32
|
||||
|
||||
// NoNode is the nil NodeID.
|
||||
const NoNode NodeID = ^NodeID(0)
|
||||
|
||||
// Kind is the node type.
|
||||
type Kind uint8
|
||||
|
||||
const (
|
||||
KindFile Kind = iota
|
||||
KindDir
|
||||
KindSymlink
|
||||
KindOther // devices, sockets, fifos, reparse points
|
||||
)
|
||||
|
||||
func (k Kind) String() string {
|
||||
return [...]string{"File", "Directory", "Symlink", "Special"}[k&3]
|
||||
}
|
||||
|
||||
// Flags record per-node facts.
|
||||
type Flags uint16
|
||||
|
||||
const (
|
||||
FlagHardlinkDup Flags = 1 << iota // additional link to an inode counted elsewhere
|
||||
FlagMountPoint // directory on a different device than its parent
|
||||
FlagVirtualFS // virtual filesystem, not descended
|
||||
FlagSkippedFS // other filesystem, not descended (--one-file-system)
|
||||
FlagSparse // allocated < logical (sparse or compressed)
|
||||
FlagError // stat or read failed
|
||||
FlagIncomplete // directory contents not (fully) read
|
||||
FlagScanned // directory fully read
|
||||
FlagLoop // directory already visited via another path
|
||||
FlagFollowed // reached through a followed symlink
|
||||
FlagAllocUnknown // platform cannot report allocated size
|
||||
FlagBrokenLink // symlink target does not exist
|
||||
FlagHardlinked // regular file with link count > 1
|
||||
)
|
||||
|
||||
// Node is one filesystem object. Sizes are own sizes; Tot* fields are
|
||||
// aggregates including the node itself and (for directories) everything
|
||||
// beneath it.
|
||||
type Node struct {
|
||||
Parent, FirstChild, NextSibling NodeID
|
||||
dir uint32 // index into dir side table (directories only)
|
||||
|
||||
Name string // single path component; the root holds the full root path
|
||||
|
||||
Size, Alloc int64 // own logical / allocated bytes
|
||||
TotSize, TotAlloc int64 // aggregates (hard-link duplicates contribute 0)
|
||||
MTime int64 // unix nanoseconds
|
||||
|
||||
Files, Dirs uint32 // recursive counts beneath (dirs excludes self)
|
||||
Errors uint32 // recursive error count beneath and including self
|
||||
Mode uint32 // fs.FileMode bits
|
||||
UID, GID uint32
|
||||
Nlink uint32
|
||||
|
||||
Ext uint16
|
||||
Flags Flags
|
||||
Kind Kind
|
||||
Cat Category
|
||||
}
|
||||
|
||||
// IsDir reports whether the node is a directory.
|
||||
func (n *Node) IsDir() bool { return n.Kind == KindDir }
|
||||
|
||||
// Delta is an aggregate change applied to a node and all its ancestors.
|
||||
type Delta struct {
|
||||
Size, Alloc int64
|
||||
Files, Dirs, Errors int64
|
||||
Cat [NumCategories]int64
|
||||
}
|
||||
|
||||
const chunkBits = 16
|
||||
const chunkSize = 1 << chunkBits
|
||||
|
||||
// chunked is a growable array that never moves existing elements.
|
||||
type chunked[T any] struct {
|
||||
c [][]T
|
||||
n int
|
||||
}
|
||||
|
||||
func (c *chunked[T]) add(v T) uint32 {
|
||||
if c.n&(chunkSize-1) == 0 && c.n>>chunkBits == len(c.c) {
|
||||
c.c = append(c.c, make([]T, 0, chunkSize))
|
||||
}
|
||||
ch := &c.c[c.n>>chunkBits]
|
||||
*ch = append(*ch, v)
|
||||
c.n++
|
||||
return uint32(c.n - 1)
|
||||
}
|
||||
|
||||
func (c *chunked[T]) at(i uint32) *T { return &c.c[i>>chunkBits][i&(chunkSize-1)] }
|
||||
|
||||
// Tree is the inventory for one scan root.
|
||||
type Tree struct {
|
||||
sync.RWMutex
|
||||
|
||||
nodes chunked[Node]
|
||||
dirs chunked[[NumCategories]int64]
|
||||
|
||||
exts []string
|
||||
extIdx map[string]uint16
|
||||
|
||||
Stats Stats
|
||||
}
|
||||
|
||||
// New creates a tree whose root has the given full path and kind.
|
||||
func New(rootPath string, kind Kind) *Tree {
|
||||
t := &Tree{exts: []string{""}, extIdx: map[string]uint16{"": 0}}
|
||||
t.Stats.Root = rootPath
|
||||
t.Stats.Start = time.Now()
|
||||
t.Add(NoNode, rootPath, kind)
|
||||
return t
|
||||
}
|
||||
|
||||
// Len returns the number of nodes.
|
||||
func (t *Tree) Len() int { return t.nodes.n }
|
||||
|
||||
// Node returns the node for id. The pointer stays valid; reading its fields
|
||||
// concurrently with the writer requires RLock.
|
||||
func (t *Tree) Node(id NodeID) *Node { return t.nodes.at(uint32(id)) }
|
||||
|
||||
// Root returns the root NodeID (always 0).
|
||||
func (t *Tree) Root() NodeID { return 0 }
|
||||
|
||||
// Add appends a child node and links it under parent. Only the writer may
|
||||
// call it. It does not touch aggregates; use Propagate.
|
||||
func (t *Tree) Add(parent NodeID, name string, kind Kind) NodeID {
|
||||
n := Node{Parent: parent, FirstChild: NoNode, NextSibling: NoNode, Name: name, Kind: kind}
|
||||
if kind == KindDir {
|
||||
n.dir = t.dirs.add([NumCategories]int64{})
|
||||
} else {
|
||||
n.Ext = t.internExt(Ext(name))
|
||||
}
|
||||
id := NodeID(t.nodes.add(n))
|
||||
if parent != NoNode {
|
||||
p := t.Node(parent)
|
||||
t.Node(id).NextSibling = p.FirstChild
|
||||
p.FirstChild = id
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (t *Tree) internExt(e string) uint16 {
|
||||
if i, ok := t.extIdx[e]; ok {
|
||||
return i
|
||||
}
|
||||
if len(t.exts) >= 1<<16-1 {
|
||||
return 0
|
||||
}
|
||||
t.exts = append(t.exts, e)
|
||||
t.extIdx[e] = uint16(len(t.exts) - 1)
|
||||
return uint16(len(t.exts) - 1)
|
||||
}
|
||||
|
||||
// ExtName returns the extension string for a node ("" if none).
|
||||
func (t *Tree) ExtName(n *Node) string { return t.exts[n.Ext] }
|
||||
|
||||
// ExtCount returns the number of interned extensions (including "").
|
||||
func (t *Tree) ExtCount() int { return len(t.exts) }
|
||||
|
||||
// ExtByIndex returns an interned extension.
|
||||
func (t *Tree) ExtByIndex(i int) string { return t.exts[i] }
|
||||
|
||||
// CatSizes returns the per-category allocated bytes beneath a directory.
|
||||
func (t *Tree) CatSizes(id NodeID) *[NumCategories]int64 {
|
||||
n := t.Node(id)
|
||||
if n.Kind != KindDir {
|
||||
return nil
|
||||
}
|
||||
return t.dirs.at(n.dir)
|
||||
}
|
||||
|
||||
// Propagate adds d to id and every ancestor. Cost is O(depth); callers batch
|
||||
// many entries into one Delta.
|
||||
func (t *Tree) Propagate(id NodeID, d Delta) {
|
||||
for id != NoNode {
|
||||
n := t.Node(id)
|
||||
n.TotSize += d.Size
|
||||
n.TotAlloc += d.Alloc
|
||||
n.Files = addU32(n.Files, d.Files)
|
||||
n.Dirs = addU32(n.Dirs, d.Dirs)
|
||||
n.Errors = addU32(n.Errors, d.Errors)
|
||||
if n.Kind == KindDir {
|
||||
cs := t.dirs.at(n.dir)
|
||||
for i, v := range d.Cat {
|
||||
cs[i] += v
|
||||
}
|
||||
}
|
||||
id = n.Parent
|
||||
}
|
||||
}
|
||||
|
||||
func addU32(a uint32, d int64) uint32 {
|
||||
v := int64(a) + d
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1<<32-1 {
|
||||
return 1<<32 - 1
|
||||
}
|
||||
return uint32(v)
|
||||
}
|
||||
|
||||
// Children calls fn for each child of id (in reverse insertion order).
|
||||
func (t *Tree) Children(id NodeID, fn func(NodeID, *Node)) {
|
||||
for c := t.Node(id).FirstChild; c != NoNode; {
|
||||
n := t.Node(c)
|
||||
fn(c, n)
|
||||
c = n.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
// ChildCount returns the number of direct children.
|
||||
func (t *Tree) ChildCount(id NodeID) int {
|
||||
k := 0
|
||||
for c := t.Node(id).FirstChild; c != NoNode; c = t.Node(c).NextSibling {
|
||||
k++
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// Path reconstructs the full path of id.
|
||||
func (t *Tree) Path(id NodeID) string {
|
||||
var parts []string
|
||||
for id != NoNode {
|
||||
n := t.Node(id)
|
||||
parts = append(parts, n.Name)
|
||||
id = n.Parent
|
||||
}
|
||||
slices.Reverse(parts)
|
||||
return filepath.Join(parts...)
|
||||
}
|
||||
|
||||
// Ancestors returns the chain root..id inclusive.
|
||||
func (t *Tree) Ancestors(id NodeID) []NodeID {
|
||||
var out []NodeID
|
||||
for id != NoNode {
|
||||
out = append(out, id)
|
||||
id = t.Node(id).Parent
|
||||
}
|
||||
slices.Reverse(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsAncestor reports whether a is id or an ancestor of id.
|
||||
func (t *Tree) IsAncestor(a, id NodeID) bool {
|
||||
for id != NoNode {
|
||||
if id == a {
|
||||
return true
|
||||
}
|
||||
id = t.Node(id).Parent
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Depth returns the number of ancestors of id.
|
||||
func (t *Tree) Depth(id NodeID) int {
|
||||
d := 0
|
||||
for id = t.Node(id).Parent; id != NoNode; id = t.Node(id).Parent {
|
||||
d++
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Recompute rebuilds all aggregates bottom-up. It relies on the invariant
|
||||
// that a parent's ID is smaller than its children's, which both the scanner
|
||||
// and the snapshot loader guarantee. Used after loading a snapshot.
|
||||
func (t *Tree) Recompute() {
|
||||
for i := 0; i < t.Len(); i++ {
|
||||
n := t.Node(NodeID(i))
|
||||
n.TotSize, n.TotAlloc, n.Files, n.Dirs = 0, 0, 0, 0
|
||||
n.Errors = 0
|
||||
if n.Flags&FlagError != 0 {
|
||||
n.Errors = 1
|
||||
}
|
||||
if n.Flags&FlagHardlinkDup == 0 {
|
||||
n.TotSize, n.TotAlloc = n.Size, n.Alloc
|
||||
}
|
||||
if n.Kind == KindDir {
|
||||
*t.dirs.at(n.dir) = [NumCategories]int64{}
|
||||
}
|
||||
}
|
||||
for i := t.Len() - 1; i > 0; i-- {
|
||||
n := t.Node(NodeID(i))
|
||||
p := t.Node(n.Parent)
|
||||
p.TotSize += n.TotSize
|
||||
p.TotAlloc += n.TotAlloc
|
||||
p.Errors += n.Errors
|
||||
cs := t.dirs.at(p.dir)
|
||||
if n.Kind == KindDir {
|
||||
p.Files += n.Files
|
||||
p.Dirs += n.Dirs + 1
|
||||
for c, v := range t.dirs.at(n.dir) {
|
||||
cs[c] += v
|
||||
}
|
||||
} else {
|
||||
p.Files++
|
||||
if n.Flags&FlagHardlinkDup == 0 {
|
||||
cs[n.Cat] += n.Alloc
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExtIndexer returns a function interning extension strings; used by
|
||||
// loaders that rebuild a tree from external data.
|
||||
func (t *Tree) ExtIndexer() func(string) uint16 { return t.internExt }
|
||||
Reference in New Issue
Block a user