Add filesystem inventory model
Append-only chunked node arena with stable NodeIDs, intrusive child lists, per-directory category totals and batched ancestor propagation.
This commit is contained in:
Andy Dixon
@@ -0,0 +1,326 @@
|
||||
// Package inventory is the in-memory filesystem model: an append-only arena
|
||||
// of nodes addressed by stable NodeIDs, with incrementally maintained
|
||||
// directory aggregates.
|
||||
//
|
||||
// Concurrency: a Tree has exactly one writer (the scan controller or a
|
||||
// snapshot loader). Other goroutines must hold RLock while reading node data.
|
||||
// Nodes are never removed, so a NodeID stays valid for the Tree's lifetime.
|
||||
package inventory
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// NodeID addresses a node within one Tree. The root is always 0.
|
||||
type NodeID uint32
|
||||
|
||||
// NoNode is the nil NodeID.
|
||||
const NoNode NodeID = ^NodeID(0)
|
||||
|
||||
// Kind is the node type.
|
||||
type Kind uint8
|
||||
|
||||
const (
|
||||
KindFile Kind = iota
|
||||
KindDir
|
||||
KindSymlink
|
||||
KindOther // devices, sockets, fifos, reparse points
|
||||
)
|
||||
|
||||
func (k Kind) String() string {
|
||||
return [...]string{"File", "Directory", "Symlink", "Special"}[k&3]
|
||||
}
|
||||
|
||||
// Flags record per-node facts.
|
||||
type Flags uint16
|
||||
|
||||
const (
|
||||
FlagHardlinkDup Flags = 1 << iota // additional link to an inode counted elsewhere
|
||||
FlagMountPoint // directory on a different device than its parent
|
||||
FlagVirtualFS // virtual filesystem, not descended
|
||||
FlagSkippedFS // other filesystem, not descended (--one-file-system)
|
||||
FlagSparse // allocated < logical (sparse or compressed)
|
||||
FlagError // stat or read failed
|
||||
FlagIncomplete // directory contents not (fully) read
|
||||
FlagScanned // directory fully read
|
||||
FlagLoop // directory already visited via another path
|
||||
FlagFollowed // reached through a followed symlink
|
||||
FlagAllocUnknown // platform cannot report allocated size
|
||||
FlagBrokenLink // symlink target does not exist
|
||||
FlagHardlinked // regular file with link count > 1
|
||||
)
|
||||
|
||||
// Node is one filesystem object. Sizes are own sizes; Tot* fields are
|
||||
// aggregates including the node itself and (for directories) everything
|
||||
// beneath it.
|
||||
type Node struct {
|
||||
Parent, FirstChild, NextSibling NodeID
|
||||
dir uint32 // index into dir side table (directories only)
|
||||
|
||||
Name string // single path component; the root holds the full root path
|
||||
|
||||
Size, Alloc int64 // own logical / allocated bytes
|
||||
TotSize, TotAlloc int64 // aggregates (hard-link duplicates contribute 0)
|
||||
MTime int64 // unix nanoseconds
|
||||
|
||||
Files, Dirs uint32 // recursive counts beneath (dirs excludes self)
|
||||
Errors uint32 // recursive error count beneath and including self
|
||||
Mode uint32 // fs.FileMode bits
|
||||
UID, GID uint32
|
||||
Nlink uint32
|
||||
|
||||
Ext uint16
|
||||
Flags Flags
|
||||
Kind Kind
|
||||
Cat Category
|
||||
}
|
||||
|
||||
// IsDir reports whether the node is a directory.
|
||||
func (n *Node) IsDir() bool { return n.Kind == KindDir }
|
||||
|
||||
// Delta is an aggregate change applied to a node and all its ancestors.
|
||||
type Delta struct {
|
||||
Size, Alloc int64
|
||||
Files, Dirs, Errors int64
|
||||
Cat [NumCategories]int64
|
||||
}
|
||||
|
||||
const chunkBits = 16
|
||||
const chunkSize = 1 << chunkBits
|
||||
|
||||
// chunked is a growable array that never moves existing elements.
|
||||
type chunked[T any] struct {
|
||||
c [][]T
|
||||
n int
|
||||
}
|
||||
|
||||
func (c *chunked[T]) add(v T) uint32 {
|
||||
if c.n&(chunkSize-1) == 0 && c.n>>chunkBits == len(c.c) {
|
||||
c.c = append(c.c, make([]T, 0, chunkSize))
|
||||
}
|
||||
ch := &c.c[c.n>>chunkBits]
|
||||
*ch = append(*ch, v)
|
||||
c.n++
|
||||
return uint32(c.n - 1)
|
||||
}
|
||||
|
||||
func (c *chunked[T]) at(i uint32) *T { return &c.c[i>>chunkBits][i&(chunkSize-1)] }
|
||||
|
||||
// Tree is the inventory for one scan root.
|
||||
type Tree struct {
|
||||
sync.RWMutex
|
||||
|
||||
nodes chunked[Node]
|
||||
dirs chunked[[NumCategories]int64]
|
||||
|
||||
exts []string
|
||||
extIdx map[string]uint16
|
||||
|
||||
Stats Stats
|
||||
}
|
||||
|
||||
// New creates a tree whose root has the given full path and kind.
|
||||
func New(rootPath string, kind Kind) *Tree {
|
||||
t := &Tree{exts: []string{""}, extIdx: map[string]uint16{"": 0}}
|
||||
t.Stats.Root = rootPath
|
||||
t.Stats.Start = time.Now()
|
||||
t.Add(NoNode, rootPath, kind)
|
||||
return t
|
||||
}
|
||||
|
||||
// Len returns the number of nodes.
|
||||
func (t *Tree) Len() int { return t.nodes.n }
|
||||
|
||||
// Node returns the node for id. The pointer stays valid; reading its fields
|
||||
// concurrently with the writer requires RLock.
|
||||
func (t *Tree) Node(id NodeID) *Node { return t.nodes.at(uint32(id)) }
|
||||
|
||||
// Root returns the root NodeID (always 0).
|
||||
func (t *Tree) Root() NodeID { return 0 }
|
||||
|
||||
// Add appends a child node and links it under parent. Only the writer may
|
||||
// call it. It does not touch aggregates; use Propagate.
|
||||
func (t *Tree) Add(parent NodeID, name string, kind Kind) NodeID {
|
||||
n := Node{Parent: parent, FirstChild: NoNode, NextSibling: NoNode, Name: name, Kind: kind}
|
||||
if kind == KindDir {
|
||||
n.dir = t.dirs.add([NumCategories]int64{})
|
||||
} else {
|
||||
n.Ext = t.internExt(Ext(name))
|
||||
}
|
||||
id := NodeID(t.nodes.add(n))
|
||||
if parent != NoNode {
|
||||
p := t.Node(parent)
|
||||
t.Node(id).NextSibling = p.FirstChild
|
||||
p.FirstChild = id
|
||||
}
|
||||
return id
|
||||
}
|
||||
|
||||
func (t *Tree) internExt(e string) uint16 {
|
||||
if i, ok := t.extIdx[e]; ok {
|
||||
return i
|
||||
}
|
||||
if len(t.exts) >= 1<<16-1 {
|
||||
return 0
|
||||
}
|
||||
t.exts = append(t.exts, e)
|
||||
t.extIdx[e] = uint16(len(t.exts) - 1)
|
||||
return uint16(len(t.exts) - 1)
|
||||
}
|
||||
|
||||
// ExtName returns the extension string for a node ("" if none).
|
||||
func (t *Tree) ExtName(n *Node) string { return t.exts[n.Ext] }
|
||||
|
||||
// ExtCount returns the number of interned extensions (including "").
|
||||
func (t *Tree) ExtCount() int { return len(t.exts) }
|
||||
|
||||
// ExtByIndex returns an interned extension.
|
||||
func (t *Tree) ExtByIndex(i int) string { return t.exts[i] }
|
||||
|
||||
// CatSizes returns the per-category allocated bytes beneath a directory.
|
||||
func (t *Tree) CatSizes(id NodeID) *[NumCategories]int64 {
|
||||
n := t.Node(id)
|
||||
if n.Kind != KindDir {
|
||||
return nil
|
||||
}
|
||||
return t.dirs.at(n.dir)
|
||||
}
|
||||
|
||||
// Propagate adds d to id and every ancestor. Cost is O(depth); callers batch
|
||||
// many entries into one Delta.
|
||||
func (t *Tree) Propagate(id NodeID, d Delta) {
|
||||
for id != NoNode {
|
||||
n := t.Node(id)
|
||||
n.TotSize += d.Size
|
||||
n.TotAlloc += d.Alloc
|
||||
n.Files = addU32(n.Files, d.Files)
|
||||
n.Dirs = addU32(n.Dirs, d.Dirs)
|
||||
n.Errors = addU32(n.Errors, d.Errors)
|
||||
if n.Kind == KindDir {
|
||||
cs := t.dirs.at(n.dir)
|
||||
for i, v := range d.Cat {
|
||||
cs[i] += v
|
||||
}
|
||||
}
|
||||
id = n.Parent
|
||||
}
|
||||
}
|
||||
|
||||
func addU32(a uint32, d int64) uint32 {
|
||||
v := int64(a) + d
|
||||
if v < 0 {
|
||||
return 0
|
||||
}
|
||||
if v > 1<<32-1 {
|
||||
return 1<<32 - 1
|
||||
}
|
||||
return uint32(v)
|
||||
}
|
||||
|
||||
// Children calls fn for each child of id (in reverse insertion order).
|
||||
func (t *Tree) Children(id NodeID, fn func(NodeID, *Node)) {
|
||||
for c := t.Node(id).FirstChild; c != NoNode; {
|
||||
n := t.Node(c)
|
||||
fn(c, n)
|
||||
c = n.NextSibling
|
||||
}
|
||||
}
|
||||
|
||||
// ChildCount returns the number of direct children.
|
||||
func (t *Tree) ChildCount(id NodeID) int {
|
||||
k := 0
|
||||
for c := t.Node(id).FirstChild; c != NoNode; c = t.Node(c).NextSibling {
|
||||
k++
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// Path reconstructs the full path of id.
|
||||
func (t *Tree) Path(id NodeID) string {
|
||||
var parts []string
|
||||
for id != NoNode {
|
||||
n := t.Node(id)
|
||||
parts = append(parts, n.Name)
|
||||
id = n.Parent
|
||||
}
|
||||
slices.Reverse(parts)
|
||||
return filepath.Join(parts...)
|
||||
}
|
||||
|
||||
// Ancestors returns the chain root..id inclusive.
|
||||
func (t *Tree) Ancestors(id NodeID) []NodeID {
|
||||
var out []NodeID
|
||||
for id != NoNode {
|
||||
out = append(out, id)
|
||||
id = t.Node(id).Parent
|
||||
}
|
||||
slices.Reverse(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// IsAncestor reports whether a is id or an ancestor of id.
|
||||
func (t *Tree) IsAncestor(a, id NodeID) bool {
|
||||
for id != NoNode {
|
||||
if id == a {
|
||||
return true
|
||||
}
|
||||
id = t.Node(id).Parent
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Depth returns the number of ancestors of id.
|
||||
func (t *Tree) Depth(id NodeID) int {
|
||||
d := 0
|
||||
for id = t.Node(id).Parent; id != NoNode; id = t.Node(id).Parent {
|
||||
d++
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// Recompute rebuilds all aggregates bottom-up. It relies on the invariant
|
||||
// that a parent's ID is smaller than its children's, which both the scanner
|
||||
// and the snapshot loader guarantee. Used after loading a snapshot.
|
||||
func (t *Tree) Recompute() {
|
||||
for i := 0; i < t.Len(); i++ {
|
||||
n := t.Node(NodeID(i))
|
||||
n.TotSize, n.TotAlloc, n.Files, n.Dirs = 0, 0, 0, 0
|
||||
n.Errors = 0
|
||||
if n.Flags&FlagError != 0 {
|
||||
n.Errors = 1
|
||||
}
|
||||
if n.Flags&FlagHardlinkDup == 0 {
|
||||
n.TotSize, n.TotAlloc = n.Size, n.Alloc
|
||||
}
|
||||
if n.Kind == KindDir {
|
||||
*t.dirs.at(n.dir) = [NumCategories]int64{}
|
||||
}
|
||||
}
|
||||
for i := t.Len() - 1; i > 0; i-- {
|
||||
n := t.Node(NodeID(i))
|
||||
p := t.Node(n.Parent)
|
||||
p.TotSize += n.TotSize
|
||||
p.TotAlloc += n.TotAlloc
|
||||
p.Errors += n.Errors
|
||||
cs := t.dirs.at(p.dir)
|
||||
if n.Kind == KindDir {
|
||||
p.Files += n.Files
|
||||
p.Dirs += n.Dirs + 1
|
||||
for c, v := range t.dirs.at(n.dir) {
|
||||
cs[c] += v
|
||||
}
|
||||
} else {
|
||||
p.Files++
|
||||
if n.Flags&FlagHardlinkDup == 0 {
|
||||
cs[n.Cat] += n.Alloc
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ExtIndexer returns a function interning extension strings; used by
|
||||
// loaders that rebuild a tree from external data.
|
||||
func (t *Tree) ExtIndexer() func(string) uint16 { return t.internExt }
|
||||
Reference in New Issue
Block a user