Add CLI reports, snapshot format and duplicate detection
This commit is contained in:
Andy Dixon
@@ -0,0 +1,169 @@
|
||||
// Package app wires the command-line modes together: interactive TUI,
|
||||
// non-interactive reports, snapshots and comparisons. TUI and CLI share the
|
||||
// same scanner and inventory packages.
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"os"
|
||||
"os/signal"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/config"
|
||||
"github.com/andydixon/mapsize/internal/export"
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
"github.com/andydixon/mapsize/internal/scan"
|
||||
"github.com/andydixon/mapsize/internal/textutil"
|
||||
)
|
||||
|
||||
// Config is the fully-resolved command line.
|
||||
type Config struct {
|
||||
Paths []string
|
||||
ScanOpts scan.Options
|
||||
NoUI bool
|
||||
JSON, CSV bool
|
||||
Depth int
|
||||
Top int
|
||||
LargestFiles int
|
||||
Save string
|
||||
Load string
|
||||
Compare [2]string
|
||||
Duplicates bool
|
||||
ReadOnly bool
|
||||
Theme string
|
||||
SizeMode inventory.SizeMode
|
||||
Mouse bool
|
||||
Settings config.Settings
|
||||
Stdout io.Writer
|
||||
Stderr io.Writer
|
||||
}
|
||||
|
||||
// Report reports whether any non-interactive output was requested.
|
||||
func (c *Config) Report() bool {
|
||||
return c.NoUI || c.JSON || c.CSV || c.Top > 0 || c.LargestFiles > 0 || c.Duplicates
|
||||
}
|
||||
|
||||
// Run executes the configured mode.
|
||||
func Run(cfg *Config) error {
|
||||
if cfg.Stdout == nil {
|
||||
cfg.Stdout = os.Stdout
|
||||
}
|
||||
if cfg.Stderr == nil {
|
||||
cfg.Stderr = os.Stderr
|
||||
}
|
||||
if cfg.Compare[0] != "" {
|
||||
return runCompare(cfg)
|
||||
}
|
||||
if !cfg.Report() {
|
||||
return runTUI(cfg)
|
||||
}
|
||||
return runReport(cfg)
|
||||
}
|
||||
|
||||
// startSource begins a scan (or loads a snapshot). The scanner is nil when
|
||||
// the tree came from a snapshot.
|
||||
func startSource(ctx context.Context, cfg *Config) (*scan.Scanner, *inventory.Tree, error) {
|
||||
if cfg.Load != "" {
|
||||
t, err := loadSnapshot(cfg.Load)
|
||||
return nil, t, err
|
||||
}
|
||||
root := "."
|
||||
if len(cfg.Paths) > 0 {
|
||||
root = cfg.Paths[0]
|
||||
}
|
||||
slog.Info("scan start", "workers", cfg.ScanOpts.Workers, "follow", cfg.ScanOpts.Follow.String())
|
||||
s, err := scan.Start(ctx, root, cfg.ScanOpts)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return s, s.Tree, nil
|
||||
}
|
||||
|
||||
func runReport(cfg *Config) error {
|
||||
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer stop()
|
||||
s, t, err := startSource(ctx, cfg)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if s != nil {
|
||||
waitWithProgress(s, cfg.Stderr)
|
||||
}
|
||||
t.RLock()
|
||||
defer t.RUnlock()
|
||||
if s != nil && cfg.Save != "" {
|
||||
if err := saveSnapshot(t, cfg.Save); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Fprintf(cfg.Stderr, "Saved snapshot %s\n", cfg.Save)
|
||||
}
|
||||
out := cfg.Stdout
|
||||
switch {
|
||||
case cfg.JSON:
|
||||
return export.JSON(out, t, t.Root(), cfg.Depth)
|
||||
case cfg.CSV:
|
||||
return export.CSV(out, t, t.Root(), cfg.Depth)
|
||||
}
|
||||
if cfg.Duplicates {
|
||||
if err := runDuplicates(ctx, cfg, t); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if cfg.Top > 0 {
|
||||
ids := t.SortedChildren(t.Root(), cfg.SizeMode)
|
||||
if len(ids) > cfg.Top {
|
||||
ids = ids[:cfg.Top]
|
||||
}
|
||||
export.Table(out, t, ids, cfg.SizeMode)
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if cfg.LargestFiles > 0 {
|
||||
ids := t.Top(t.Root(), cfg.LargestFiles,
|
||||
func(n *inventory.Node) bool {
|
||||
return n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0
|
||||
},
|
||||
func(n *inventory.Node) int64 { return n.Own(cfg.SizeMode) })
|
||||
export.Table(out, t, ids, cfg.SizeMode)
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
export.Summary(out, t)
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitWithProgress blocks until the scan finishes, printing a progress line
|
||||
// to w if it is a terminal.
|
||||
func waitWithProgress(s *scan.Scanner, w io.Writer) {
|
||||
tty := isTerminal(w)
|
||||
tick := time.NewTicker(250 * time.Millisecond)
|
||||
defer tick.Stop()
|
||||
start := time.Now()
|
||||
for {
|
||||
select {
|
||||
case <-s.Done():
|
||||
if tty {
|
||||
fmt.Fprint(w, "\r\033[K")
|
||||
}
|
||||
return
|
||||
case <-tick.C:
|
||||
if tty {
|
||||
p := s.Progress()
|
||||
el := time.Since(start)
|
||||
fmt.Fprintf(w, "\r\033[Kscanning… %s entries %s/s queue %d %s",
|
||||
textutil.Count(p.Entries), textutil.Count(int64(float64(p.Entries)/el.Seconds())),
|
||||
p.Pending, textutil.Duration(el))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isTerminal(w io.Writer) bool {
|
||||
f, ok := w.(*os.File)
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
fi, err := f.Stat()
|
||||
return err == nil && fi.Mode()&os.ModeCharDevice != 0
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package app
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/duplicate"
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
"github.com/andydixon/mapsize/internal/snapshot"
|
||||
"github.com/andydixon/mapsize/internal/textutil"
|
||||
)
|
||||
|
||||
func runTUI(*Config) error {
|
||||
return errors.New("the interactive UI is not built yet; use --no-ui, --top, --json or --csv")
|
||||
}
|
||||
|
||||
func loadSnapshot(path string) (*inventory.Tree, error) { return snapshot.LoadFile(path) }
|
||||
|
||||
func saveSnapshot(t *inventory.Tree, path string) error { return snapshot.SaveFile(path, t) }
|
||||
|
||||
func runCompare(cfg *Config) error {
|
||||
old, err := snapshot.LoadFile(cfg.Compare[0])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", cfg.Compare[0], err)
|
||||
}
|
||||
nw, err := snapshot.LoadFile(cfg.Compare[1])
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", cfg.Compare[1], err)
|
||||
}
|
||||
d := snapshot.Compare(old, nw)
|
||||
k := cfg.Top
|
||||
if k <= 0 {
|
||||
k = 30
|
||||
}
|
||||
out := cfg.Stdout
|
||||
ro, rn := old.Node(0), nw.Node(0)
|
||||
fmt.Fprintf(out, "%s → %s\n", textutil.Sanitize(old.Stats.FromSnapshot), textutil.Sanitize(nw.Stats.FromSnapshot))
|
||||
fmt.Fprintf(out, "Total %s → %s (%s)\n\n", textutil.Size(ro.Total(cfg.SizeMode)), textutil.Size(rn.Total(cfg.SizeMode)),
|
||||
textutil.SignedSize(rn.Total(cfg.SizeMode)-ro.Total(cfg.SizeMode)))
|
||||
for _, c := range d.Changes(k, cfg.SizeMode) {
|
||||
fmt.Fprintf(out, "%-10s %14s %s\n", c.Status, textutil.SignedSize(c.Delta), textutil.Sanitize(c.Path))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func runDuplicates(ctx context.Context, cfg *Config, t *inventory.Tree) error {
|
||||
// The caller holds RLock; Find takes its own read lock, which is fine
|
||||
// for a RWMutex only if no writer is waiting — the scan is finished.
|
||||
var f duplicate.Finder
|
||||
groups, err := f.Find(ctx, t, duplicate.Options{MinSize: 1})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
out := cfg.Stdout
|
||||
var wasted int64
|
||||
for _, g := range groups {
|
||||
wasted += g.Wasted()
|
||||
fmt.Fprintf(out, "%s × %d (sha256 %x…)\n", textutil.Size(g.Size), len(g.Files), g.Hash[:6])
|
||||
for _, id := range g.Files {
|
||||
fmt.Fprintf(out, " %s\n", textutil.Sanitize(t.Path(id)))
|
||||
}
|
||||
}
|
||||
p := f.Progress()
|
||||
fmt.Fprintf(out, "\n%d verified duplicate groups, %s reclaimable (%d candidates by size, %d unreadable/changed)\n\n",
|
||||
len(groups), textutil.Size(wasted), p.Candidates, p.Skipped)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
// Package duplicate finds files with identical content in stages:
|
||||
//
|
||||
// 1. group by exact size (inventory only, no I/O)
|
||||
// 2. hash three 16 KiB samples (start, middle, end) of each candidate
|
||||
// 3. full SHA-256 of files whose samples still collide
|
||||
//
|
||||
// Only stage 3 establishes identity. Files are never declared duplicates by
|
||||
// name or size alone. Reading files may update their access times.
|
||||
package duplicate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"io"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
)
|
||||
|
||||
// Group is a set of files verified to have identical content.
|
||||
type Group struct {
|
||||
Size int64
|
||||
Hash [32]byte
|
||||
Files []inventory.NodeID
|
||||
}
|
||||
|
||||
// Wasted returns the bytes reclaimable by keeping one copy.
|
||||
func (g *Group) Wasted() int64 { return g.Size * int64(len(g.Files)-1) }
|
||||
|
||||
// Progress reports activity.
|
||||
type Progress struct {
|
||||
Stage string
|
||||
Candidates int64 // files sharing a size with another file
|
||||
BytesToHash int64
|
||||
BytesHashed int64
|
||||
Skipped int64 // unreadable or changed during hashing
|
||||
VerifiedGroups int64
|
||||
CandidateGroups int64
|
||||
}
|
||||
|
||||
// Options tunes the search.
|
||||
type Options struct {
|
||||
MinSize int64 // ignore files smaller than this (default 1 byte)
|
||||
Workers int // concurrent readers (default 4)
|
||||
}
|
||||
|
||||
type cand struct {
|
||||
id inventory.NodeID
|
||||
path string
|
||||
size int64
|
||||
}
|
||||
|
||||
const sampleSize = 16 << 10
|
||||
|
||||
// Finder runs a search and exposes live progress.
|
||||
type Finder struct {
|
||||
hashed, toHash, skipped atomic.Int64
|
||||
mu sync.Mutex
|
||||
p Progress
|
||||
}
|
||||
|
||||
// Progress returns a copy of the current progress.
|
||||
func (f *Finder) Progress() Progress {
|
||||
f.mu.Lock()
|
||||
p := f.p
|
||||
f.mu.Unlock()
|
||||
p.BytesHashed, p.BytesToHash, p.Skipped = f.hashed.Load(), f.toHash.Load(), f.skipped.Load()
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *Finder) stage(s string, fn func(*Progress)) {
|
||||
f.mu.Lock()
|
||||
f.p.Stage = s
|
||||
if fn != nil {
|
||||
fn(&f.p)
|
||||
}
|
||||
f.mu.Unlock()
|
||||
}
|
||||
|
||||
// Find searches t (read-locked internally only while collecting candidates).
|
||||
func (f *Finder) Find(ctx context.Context, t *inventory.Tree, o Options) ([]Group, error) {
|
||||
if o.MinSize < 1 {
|
||||
o.MinSize = 1
|
||||
}
|
||||
if o.Workers < 1 {
|
||||
o.Workers = 4
|
||||
}
|
||||
f.stage("grouping by size", nil)
|
||||
bySize := map[int64][]cand{}
|
||||
t.RLock()
|
||||
t.Walk(t.Root(), func(id inventory.NodeID, n *inventory.Node) bool {
|
||||
if n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0 && n.Size >= o.MinSize {
|
||||
bySize[n.Size] = append(bySize[n.Size], cand{id: id, size: n.Size})
|
||||
}
|
||||
return true
|
||||
})
|
||||
var groups [][]cand
|
||||
var ncand int64
|
||||
for _, g := range bySize {
|
||||
if len(g) > 1 {
|
||||
for i := range g {
|
||||
g[i].path = t.Path(g[i].id)
|
||||
}
|
||||
groups = append(groups, g)
|
||||
ncand += int64(len(g))
|
||||
}
|
||||
}
|
||||
t.RUnlock()
|
||||
f.stage("sampling", func(p *Progress) { p.Candidates = ncand; p.CandidateGroups = int64(len(groups)) })
|
||||
|
||||
// Stage 2: sample hashes.
|
||||
var sampled [][]cand
|
||||
for _, g := range groups {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
if g[0].size <= 3*sampleSize {
|
||||
sampled = append(sampled, g) // a sample would be the whole file
|
||||
continue
|
||||
}
|
||||
sampled = append(sampled, f.split(ctx, g, o.Workers, sampleHash)...)
|
||||
}
|
||||
|
||||
// Stage 3: full hashes.
|
||||
var total int64
|
||||
for _, g := range sampled {
|
||||
total += g[0].size * int64(len(g))
|
||||
}
|
||||
f.toHash.Store(total)
|
||||
f.stage("hashing", nil)
|
||||
var out []Group
|
||||
for _, g := range sampled {
|
||||
if ctx.Err() != nil {
|
||||
return nil, ctx.Err()
|
||||
}
|
||||
for _, sub := range f.splitHash(ctx, g, o.Workers) {
|
||||
grp := Group{Size: sub.size, Hash: sub.hash}
|
||||
for _, c := range sub.files {
|
||||
grp.Files = append(grp.Files, c.id)
|
||||
}
|
||||
out = append(out, grp)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if a, b := out[i].Wasted(), out[j].Wasted(); a != b {
|
||||
return a > b
|
||||
}
|
||||
return out[i].Files[0] < out[j].Files[0]
|
||||
})
|
||||
f.stage("done", func(p *Progress) { p.VerifiedGroups = int64(len(out)) })
|
||||
return out, ctx.Err()
|
||||
}
|
||||
|
||||
type hashed struct {
|
||||
c cand
|
||||
sum [32]byte
|
||||
ok bool
|
||||
}
|
||||
|
||||
// hashAll hashes every candidate with a bounded worker pool.
|
||||
func (f *Finder) hashAll(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) []hashed {
|
||||
res := make([]hashed, len(g))
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, workers)
|
||||
for i, c := range g {
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
}
|
||||
sem <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() { <-sem }()
|
||||
sum, ok := fn(f, c)
|
||||
if !ok {
|
||||
f.skipped.Add(1)
|
||||
}
|
||||
res[i] = hashed{c, sum, ok}
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
return res
|
||||
}
|
||||
|
||||
func (f *Finder) split(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) [][]cand {
|
||||
by := map[[32]byte][]cand{}
|
||||
for _, h := range f.hashAll(ctx, g, workers, fn) {
|
||||
if h.ok {
|
||||
by[h.sum] = append(by[h.sum], h.c)
|
||||
}
|
||||
}
|
||||
var out [][]cand
|
||||
for _, v := range by {
|
||||
if len(v) > 1 {
|
||||
out = append(out, v)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type fullGroup struct {
|
||||
size int64
|
||||
hash [32]byte
|
||||
files []cand
|
||||
}
|
||||
|
||||
func (f *Finder) splitHash(ctx context.Context, g []cand, workers int) []fullGroup {
|
||||
by := map[[32]byte][]cand{}
|
||||
for _, h := range f.hashAll(ctx, g, workers, fullHash) {
|
||||
if h.ok {
|
||||
by[h.sum] = append(by[h.sum], h.c)
|
||||
}
|
||||
}
|
||||
var out []fullGroup
|
||||
for sum, v := range by {
|
||||
if len(v) > 1 {
|
||||
sort.Slice(v, func(i, j int) bool { return v[i].id < v[j].id })
|
||||
out = append(out, fullGroup{size: v[0].size, hash: sum, files: v})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// open opens a candidate and checks it still has the inventoried size.
|
||||
func open(c cand) (*os.File, bool) {
|
||||
fh, err := os.Open(c.path)
|
||||
if err != nil {
|
||||
return nil, false
|
||||
}
|
||||
fi, err := fh.Stat()
|
||||
if err != nil || !fi.Mode().IsRegular() || fi.Size() != c.size {
|
||||
fh.Close()
|
||||
return nil, false
|
||||
}
|
||||
return fh, true
|
||||
}
|
||||
|
||||
func sampleHash(_ *Finder, c cand) ([32]byte, bool) {
|
||||
fh, ok := open(c)
|
||||
if !ok {
|
||||
return [32]byte{}, false
|
||||
}
|
||||
defer fh.Close()
|
||||
h := sha256.New()
|
||||
buf := make([]byte, sampleSize)
|
||||
for _, off := range []int64{0, c.size/2 - sampleSize/2, c.size - sampleSize} {
|
||||
if _, err := fh.ReadAt(buf, off); err != nil && err != io.EOF {
|
||||
return [32]byte{}, false
|
||||
}
|
||||
h.Write(buf)
|
||||
}
|
||||
var s [32]byte
|
||||
h.Sum(s[:0])
|
||||
return s, true
|
||||
}
|
||||
|
||||
func fullHash(f *Finder, c cand) ([32]byte, bool) {
|
||||
fh, ok := open(c)
|
||||
if !ok {
|
||||
return [32]byte{}, false
|
||||
}
|
||||
defer fh.Close()
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(h, &countingReader{fh, &f.hashed})
|
||||
if err != nil || n != c.size {
|
||||
return [32]byte{}, false
|
||||
}
|
||||
var s [32]byte
|
||||
h.Sum(s[:0])
|
||||
return s, true
|
||||
}
|
||||
|
||||
type countingReader struct {
|
||||
r io.Reader
|
||||
n *atomic.Int64
|
||||
}
|
||||
|
||||
func (c *countingReader) Read(p []byte) (int, error) {
|
||||
n, err := c.r.Read(p)
|
||||
c.n.Add(int64(n))
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package duplicate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/scan"
|
||||
)
|
||||
|
||||
func TestFind(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
big := make([]byte, 200_000)
|
||||
for i := range big {
|
||||
big[i] = byte(i * 7)
|
||||
}
|
||||
w := func(name string, b []byte) {
|
||||
if err := os.WriteFile(filepath.Join(root, name), b, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
w("a.bin", big)
|
||||
w("b.bin", big)
|
||||
// Same size, same samples, different middle-of-nowhere byte: must not match.
|
||||
c := append([]byte(nil), big...)
|
||||
c[20_000] ^= 1
|
||||
w("c.bin", c)
|
||||
w("small1", []byte("hello"))
|
||||
w("small2", []byte("hello"))
|
||||
w("small3", []byte("world")) // same size, different content
|
||||
|
||||
s, err := scan.Start(context.Background(), root, scan.Options{Workers: 2})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
<-s.Done()
|
||||
var f Finder
|
||||
groups, err := f.Find(context.Background(), s.Tree, Options{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(groups) != 2 {
|
||||
t.Fatalf("groups=%d want 2", len(groups))
|
||||
}
|
||||
names := func(g Group) (out []string) {
|
||||
for _, id := range g.Files {
|
||||
out = append(out, s.Tree.Node(id).Name)
|
||||
}
|
||||
return
|
||||
}
|
||||
if g := names(groups[0]); len(g) != 2 || groups[0].Size != 200_000 {
|
||||
t.Fatalf("big group %v", g)
|
||||
}
|
||||
if g := names(groups[1]); len(g) != 2 {
|
||||
t.Fatalf("small group %v", g)
|
||||
}
|
||||
if p := f.Progress(); p.Candidates != 6 || p.VerifiedGroups != 2 {
|
||||
t.Fatalf("progress %+v", p)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
// Package export writes inventories as JSON, CSV or plain-text reports.
|
||||
// Callers must hold the tree's read lock (or own the tree exclusively).
|
||||
package export
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/brand"
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
"github.com/andydixon/mapsize/internal/textutil"
|
||||
)
|
||||
|
||||
type jsonNode struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Size int64 `json:"size"`
|
||||
Allocated int64 `json:"allocated"`
|
||||
Files uint32 `json:"files,omitempty"`
|
||||
Dirs uint32 `json:"dirs,omitempty"`
|
||||
Errors uint32 `json:"errors,omitempty"`
|
||||
Modified string `json:"modified,omitempty"`
|
||||
Flags []string `json:"flags,omitempty"`
|
||||
Children []*jsonNode `json:"children,omitempty"`
|
||||
}
|
||||
|
||||
var flagNames = []struct {
|
||||
f inventory.Flags
|
||||
n string
|
||||
}{
|
||||
{inventory.FlagHardlinkDup, "hardlink-duplicate"}, {inventory.FlagMountPoint, "mount-point"},
|
||||
{inventory.FlagVirtualFS, "virtual-fs-skipped"}, {inventory.FlagSkippedFS, "other-fs-skipped"},
|
||||
{inventory.FlagSparse, "sparse"}, {inventory.FlagError, "error"}, {inventory.FlagIncomplete, "incomplete"},
|
||||
{inventory.FlagLoop, "loop-skipped"}, {inventory.FlagFollowed, "followed-symlink"},
|
||||
{inventory.FlagAllocUnknown, "allocated-unknown"}, {inventory.FlagBrokenLink, "broken-symlink"},
|
||||
{inventory.FlagHardlinked, "hardlinked"},
|
||||
}
|
||||
|
||||
// FlagList returns human-readable names for a node's flags.
|
||||
func FlagList(f inventory.Flags) []string {
|
||||
var out []string
|
||||
for _, fn := range flagNames {
|
||||
if f&fn.f != 0 {
|
||||
out = append(out, fn.n)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// JSON writes the tree beneath root as nested JSON, down to maxDepth levels
|
||||
// (maxDepth < 0 means unlimited).
|
||||
func JSON(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error {
|
||||
bw := bufio.NewWriter(w)
|
||||
st := &t.Stats
|
||||
head := map[string]any{
|
||||
"generator": brand.Name + " " + brand.Version,
|
||||
"root": t.Path(root),
|
||||
"complete": !st.Incomplete(),
|
||||
"cancelled": st.Cancelled,
|
||||
"started": st.Start.Format(time.RFC3339),
|
||||
"elapsed_s": st.Elapsed().Seconds(),
|
||||
"errors": errorCounts(st),
|
||||
"excluded": st.Excluded,
|
||||
"excludes": st.Excludes,
|
||||
"hardlink_duplicates": st.HardlinkDups,
|
||||
}
|
||||
hb, err := json.Marshal(head)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
bw.Write(hb[:len(hb)-1])
|
||||
bw.WriteString(`,"tree":`)
|
||||
if err := writeNode(bw, t, root, maxDepth); err != nil {
|
||||
return err
|
||||
}
|
||||
bw.WriteString("}\n")
|
||||
return bw.Flush()
|
||||
}
|
||||
|
||||
func errorCounts(st *inventory.Stats) map[string]int64 {
|
||||
m := map[string]int64{}
|
||||
for k, c := range st.ErrCounts {
|
||||
if c > 0 {
|
||||
m[inventory.ErrKind(k).String()] = c
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// writeNode streams one node; children are written recursively so memory
|
||||
// stays proportional to depth, not tree size.
|
||||
func writeNode(bw *bufio.Writer, t *inventory.Tree, id inventory.NodeID, depth int) error {
|
||||
n := t.Node(id)
|
||||
jn := jsonNode{Name: n.Name, Type: kindName(n.Kind), Size: n.TotSize, Allocated: n.TotAlloc,
|
||||
Files: n.Files, Dirs: n.Dirs, Errors: n.Errors, Flags: FlagList(n.Flags)}
|
||||
if n.MTime != 0 {
|
||||
jn.Modified = time.Unix(0, n.MTime).UTC().Format(time.RFC3339)
|
||||
}
|
||||
b, err := json.Marshal(jn)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if n.FirstChild == inventory.NoNode || depth == 0 {
|
||||
_, err = bw.Write(b)
|
||||
return err
|
||||
}
|
||||
bw.Write(b[:len(b)-1])
|
||||
bw.WriteString(`,"children":[`)
|
||||
first := true
|
||||
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
|
||||
if !first {
|
||||
bw.WriteByte(',')
|
||||
}
|
||||
first = false
|
||||
if err := writeNode(bw, t, c, depth-1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
_, err = bw.WriteString("]}")
|
||||
return err
|
||||
}
|
||||
|
||||
func kindName(k inventory.Kind) string {
|
||||
return [...]string{"file", "dir", "symlink", "other"}[k&3]
|
||||
}
|
||||
|
||||
// CSV writes one row per node beneath root (depth-limited). Paths are
|
||||
// sanitised so the output is safe to print to a terminal.
|
||||
func CSV(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error {
|
||||
cw := csv.NewWriter(w)
|
||||
cw.Write([]string{"path", "type", "size", "allocated", "files", "dirs", "errors", "modified", "flags"})
|
||||
var rec func(id inventory.NodeID, path string, depth int) error
|
||||
rec = func(id inventory.NodeID, path string, depth int) error {
|
||||
n := t.Node(id)
|
||||
mod := ""
|
||||
if n.MTime != 0 {
|
||||
mod = time.Unix(0, n.MTime).UTC().Format(time.RFC3339)
|
||||
}
|
||||
flags := ""
|
||||
for i, f := range FlagList(n.Flags) {
|
||||
if i > 0 {
|
||||
flags += "|"
|
||||
}
|
||||
flags += f
|
||||
}
|
||||
if err := cw.Write([]string{textutil.Sanitize(path), kindName(n.Kind),
|
||||
strconv.FormatInt(n.TotSize, 10), strconv.FormatInt(n.TotAlloc, 10),
|
||||
strconv.FormatUint(uint64(n.Files), 10), strconv.FormatUint(uint64(n.Dirs), 10),
|
||||
strconv.FormatUint(uint64(n.Errors), 10), mod, flags}); err != nil {
|
||||
return err
|
||||
}
|
||||
if depth == 0 {
|
||||
return nil
|
||||
}
|
||||
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
|
||||
if err := rec(c, path+"/"+t.Node(c).Name, depth-1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := rec(root, t.Path(root), maxDepth); err != nil {
|
||||
return err
|
||||
}
|
||||
cw.Flush()
|
||||
return cw.Error()
|
||||
}
|
||||
|
||||
// Table writes "size % files path" rows for the given nodes.
|
||||
func Table(w io.Writer, t *inventory.Tree, ids []inventory.NodeID, m inventory.SizeMode) {
|
||||
total := t.Node(t.Root()).Total(m)
|
||||
fmt.Fprintf(w, "%12s %6s %12s %s\n", "SIZE", "%", "FILES", "PATH")
|
||||
for _, id := range ids {
|
||||
n := t.Node(id)
|
||||
files := ""
|
||||
if n.IsDir() {
|
||||
files = textutil.Count(int64(n.Files))
|
||||
}
|
||||
fmt.Fprintf(w, "%12s %6s %12s %s\n", textutil.Size(n.Total(m)), textutil.Percent(n.Total(m), total),
|
||||
files, textutil.Sanitize(t.Path(id)))
|
||||
}
|
||||
}
|
||||
|
||||
// Summary writes the scan summary, making incompleteness explicit.
|
||||
func Summary(w io.Writer, t *inventory.Tree) {
|
||||
st := &t.Stats
|
||||
r := t.Node(t.Root())
|
||||
fmt.Fprintf(w, "Root %s\n", textutil.Sanitize(st.Root))
|
||||
fmt.Fprintf(w, "Files %s\n", textutil.Count(st.Files+st.Symlinks+st.Others))
|
||||
fmt.Fprintf(w, "Directories %s\n", textutil.Count(st.Dirs))
|
||||
fmt.Fprintf(w, "Logical size %s (%d bytes)\n", textutil.Size(r.TotSize), r.TotSize)
|
||||
fmt.Fprintf(w, "Allocated size %s (%d bytes)\n", textutil.Size(r.TotAlloc), r.TotAlloc)
|
||||
fmt.Fprintf(w, "Elapsed %s\n", st.Elapsed().Round(time.Millisecond))
|
||||
for k, c := range st.ErrCounts {
|
||||
if c > 0 {
|
||||
fmt.Fprintf(w, "%-15s %s\n", inventory.ErrKind(k).String(), textutil.Count(c))
|
||||
}
|
||||
}
|
||||
if st.Excluded > 0 {
|
||||
fmt.Fprintf(w, "Excluded %s entries (%v)\n", textutil.Count(st.Excluded), st.Excludes)
|
||||
}
|
||||
if st.SkippedMounts > 0 {
|
||||
fmt.Fprintf(w, "Other FS skipped %s\n", textutil.Count(st.SkippedMounts))
|
||||
}
|
||||
if st.VirtualSkipped > 0 {
|
||||
fmt.Fprintf(w, "Virtual FS skipped %s\n", textutil.Count(st.VirtualSkipped))
|
||||
}
|
||||
if st.HardlinkDups > 0 {
|
||||
fmt.Fprintf(w, "Hard-link dups %s (counted once)\n", textutil.Count(st.HardlinkDups))
|
||||
}
|
||||
if st.Incomplete() {
|
||||
fmt.Fprintln(w, "WARNING: totals are incomplete — some data could not be inspected.")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,177 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"sort"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
)
|
||||
|
||||
// Diff relates two inventories by path. Both trees must not be mutated
|
||||
// while a Diff is in use (snapshots and finished scans satisfy this).
|
||||
type Diff struct {
|
||||
Old, New *inventory.Tree
|
||||
// OldOf maps a New NodeID to the matching Old NodeID, or NoNode if the
|
||||
// entry is new.
|
||||
OldOf []inventory.NodeID
|
||||
// Removed lists the topmost Old nodes with no counterpart in New.
|
||||
Removed []inventory.NodeID
|
||||
// RemovedUnder sums, per New directory, the sizes of removed children
|
||||
// (in both size modes) so growth colouring can account for them.
|
||||
removedUnder map[inventory.NodeID][2]int64
|
||||
}
|
||||
|
||||
// Compare matches nodes by name, directory by directory, from the roots.
|
||||
func Compare(old, new *inventory.Tree) *Diff {
|
||||
d := &Diff{Old: old, New: new, OldOf: make([]inventory.NodeID, new.Len()),
|
||||
removedUnder: map[inventory.NodeID][2]int64{}}
|
||||
for i := range d.OldOf {
|
||||
d.OldOf[i] = inventory.NoNode
|
||||
}
|
||||
type pair struct{ o, n inventory.NodeID }
|
||||
stack := []pair{{old.Root(), new.Root()}}
|
||||
for len(stack) > 0 {
|
||||
p := stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
d.OldOf[p.n] = p.o
|
||||
on, nn := old.Node(p.o), new.Node(p.n)
|
||||
if !on.IsDir() || !nn.IsDir() {
|
||||
continue
|
||||
}
|
||||
byName := map[string]inventory.NodeID{}
|
||||
old.Children(p.o, func(id inventory.NodeID, n *inventory.Node) { byName[n.Name] = id })
|
||||
new.Children(p.n, func(id inventory.NodeID, n *inventory.Node) {
|
||||
if oid, ok := byName[n.Name]; ok {
|
||||
delete(byName, n.Name)
|
||||
stack = append(stack, pair{oid, id})
|
||||
}
|
||||
})
|
||||
for _, oid := range byName {
|
||||
d.Removed = append(d.Removed, oid)
|
||||
o := old.Node(oid)
|
||||
r := d.removedUnder[p.n]
|
||||
r[0] += o.TotAlloc
|
||||
r[1] += o.TotSize
|
||||
d.removedUnder[p.n] = r
|
||||
}
|
||||
}
|
||||
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i] < d.Removed[j] })
|
||||
return d
|
||||
}
|
||||
|
||||
// OldSize returns the matching old size of a New node and whether it existed.
|
||||
func (d *Diff) OldSize(id inventory.NodeID, m inventory.SizeMode) (int64, bool) {
|
||||
o := d.OldOf[id]
|
||||
if o == inventory.NoNode {
|
||||
return 0, false
|
||||
}
|
||||
return d.Old.Node(o).Total(m), true
|
||||
}
|
||||
|
||||
// Delta returns new minus old size for a New node (full size if new).
|
||||
func (d *Diff) Delta(id inventory.NodeID, m inventory.SizeMode) int64 {
|
||||
old, _ := d.OldSize(id, m)
|
||||
return d.New.Node(id).Total(m) - old
|
||||
}
|
||||
|
||||
// Status classifies a node.
|
||||
type Status uint8
|
||||
|
||||
const (
|
||||
Unchanged Status = iota
|
||||
Added
|
||||
Removed
|
||||
Grew
|
||||
Shrank
|
||||
)
|
||||
|
||||
func (s Status) String() string {
|
||||
return [...]string{"unchanged", "new", "removed", "increased", "decreased"}[s]
|
||||
}
|
||||
|
||||
// StatusOf classifies a New node.
|
||||
func (d *Diff) StatusOf(id inventory.NodeID, m inventory.SizeMode) Status {
|
||||
if d.OldOf[id] == inventory.NoNode {
|
||||
return Added
|
||||
}
|
||||
switch delta := d.Delta(id, m); {
|
||||
case delta > 0:
|
||||
return Grew
|
||||
case delta < 0:
|
||||
return Shrank
|
||||
}
|
||||
return Unchanged
|
||||
}
|
||||
|
||||
// Change is one row in a change report.
|
||||
type Change struct {
|
||||
Path string
|
||||
Status Status
|
||||
Delta int64
|
||||
Old int64
|
||||
New int64
|
||||
Node inventory.NodeID // in New (NoNode for removed)
|
||||
}
|
||||
|
||||
// Changes returns the k most significant changes. A directory is listed
|
||||
// only if its change is not almost entirely explained by a single child, so
|
||||
// the report points at /var/lib/postgresql rather than /, /var and /var/lib.
|
||||
func (d *Diff) Changes(k int, m inventory.SizeMode) []Change {
|
||||
var out []Change
|
||||
for i := range d.New.Len() {
|
||||
id := inventory.NodeID(i)
|
||||
delta := d.Delta(id, m)
|
||||
if delta == 0 {
|
||||
continue
|
||||
}
|
||||
n := d.New.Node(id)
|
||||
if d.OldOf[id] != inventory.NoNode && n.IsDir() && explainedByChild(d, id, delta, m) {
|
||||
continue
|
||||
}
|
||||
if p := n.Parent; p != inventory.NoNode && d.OldOf[p] == inventory.NoNode {
|
||||
continue // inside a wholly new directory; the directory is listed
|
||||
}
|
||||
old, _ := d.OldSize(id, m)
|
||||
out = append(out, Change{Path: d.New.Path(id), Status: d.StatusOf(id, m), Delta: delta, Old: old, New: n.Total(m), Node: id})
|
||||
}
|
||||
for _, oid := range d.Removed {
|
||||
o := d.Old.Node(oid)
|
||||
out = append(out, Change{Path: d.Old.Path(oid), Status: Removed, Delta: -o.Total(m), Old: o.Total(m), Node: inventory.NoNode})
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := abs(out[i].Delta), abs(out[j].Delta)
|
||||
if a != b {
|
||||
return a > b
|
||||
}
|
||||
return out[i].Path < out[j].Path
|
||||
})
|
||||
if k > 0 && len(out) > k {
|
||||
out = out[:k]
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func explainedByChild(d *Diff, id inventory.NodeID, delta int64, m inventory.SizeMode) bool {
|
||||
explained := false
|
||||
d.New.Children(id, func(c inventory.NodeID, _ *inventory.Node) {
|
||||
if cd := d.Delta(c, m); (cd > 0) == (delta > 0) && abs(cd)*10 >= abs(delta)*8 {
|
||||
explained = true
|
||||
}
|
||||
})
|
||||
if r, ok := d.removedUnder[id]; ok && !explained {
|
||||
v := r[0]
|
||||
if m == inventory.SizeLogical {
|
||||
v = r[1]
|
||||
}
|
||||
if delta < 0 && v*10 >= abs(delta)*8 {
|
||||
explained = true
|
||||
}
|
||||
}
|
||||
return explained
|
||||
}
|
||||
|
||||
func abs(v int64) int64 {
|
||||
if v < 0 {
|
||||
return -v
|
||||
}
|
||||
return v
|
||||
}
|
||||
@@ -0,0 +1,433 @@
|
||||
// Package snapshot implements the versioned, checksummed binary inventory
|
||||
// format and snapshot comparison.
|
||||
//
|
||||
// Layout (all integers little-endian / varint):
|
||||
//
|
||||
// magic 8 bytes brand.SnapshotMagic
|
||||
// version uint16 currently 1
|
||||
// flags uint16 bit 0: body is gzip-compressed (always set by Save)
|
||||
// body gzip stream of:
|
||||
// meta uvarint length + JSON (root, timestamps, counters, options)
|
||||
// exts uvarint count, then count × (uvarint length + bytes)
|
||||
// nodes uvarint count, then per node:
|
||||
// uvarint parentDelta (id-parent; 0 only for the root)
|
||||
// byte kind, byte category, uvarint flags
|
||||
// uvarint nameLen + name
|
||||
// varint size, varint alloc, varint mtime
|
||||
// uvarint mode, uid, gid, nlink, ext
|
||||
// errors uvarint count, then per record: uvarint node, byte kind,
|
||||
// uvarint len + name, uvarint len + message
|
||||
// sha256 32 bytes over the uncompressed body
|
||||
//
|
||||
// Nodes are written in ID order; a parent always precedes its children, so a
|
||||
// reader can rebuild the tree in one pass and malformed input cannot create
|
||||
// cycles. Aggregates are recomputed on load, never trusted.
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/brand"
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
)
|
||||
|
||||
// Version is the current format version.
|
||||
const Version = 1
|
||||
|
||||
// Limits applied when reading untrusted snapshots.
|
||||
const (
|
||||
MaxNodes = 1<<31 - 1
|
||||
MaxNameLen = 1 << 16
|
||||
MaxMetaLen = 16 << 20
|
||||
MaxExtLen = 64
|
||||
MaxMsgLen = 4096
|
||||
maxExts = 1<<16 - 1
|
||||
flagGzipped = 1
|
||||
)
|
||||
|
||||
type meta struct {
|
||||
Root string `json:"root"`
|
||||
Created time.Time `json:"created"`
|
||||
Start time.Time `json:"start"`
|
||||
End time.Time `json:"end"`
|
||||
Complete bool `json:"complete"`
|
||||
Cancelled bool `json:"cancelled"`
|
||||
Files int64 `json:"files"`
|
||||
Dirs int64 `json:"dirs"`
|
||||
Symlinks int64 `json:"symlinks"`
|
||||
Others int64 `json:"others"`
|
||||
ErrCounts []int64 `json:"err_counts"`
|
||||
Excluded int64 `json:"excluded"`
|
||||
SkippedMounts int64 `json:"skipped_mounts"`
|
||||
VirtualSkipped int64 `json:"virtual_skipped"`
|
||||
LoopsSkipped int64 `json:"loops_skipped"`
|
||||
BrokenLinks int64 `json:"broken_links"`
|
||||
HardlinkDups int64 `json:"hardlink_dups"`
|
||||
Unscanned int64 `json:"unscanned"`
|
||||
Excludes []string `json:"excludes"`
|
||||
OneFileSystem bool `json:"one_file_system"`
|
||||
Follow string `json:"follow"`
|
||||
Workers int `json:"workers"`
|
||||
Host string `json:"host"`
|
||||
Generator string `json:"generator"`
|
||||
}
|
||||
|
||||
// IsSnapshot reports whether path starts with the snapshot magic.
|
||||
func IsSnapshot(path string) bool {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer f.Close()
|
||||
b := make([]byte, len(brand.SnapshotMagic))
|
||||
_, err = io.ReadFull(f, b)
|
||||
return err == nil && string(b) == brand.SnapshotMagic
|
||||
}
|
||||
|
||||
type writer struct {
|
||||
w io.Writer
|
||||
buf [binary.MaxVarintLen64]byte
|
||||
err error
|
||||
}
|
||||
|
||||
func (w *writer) bytes(b []byte) {
|
||||
if w.err == nil {
|
||||
_, w.err = w.w.Write(b)
|
||||
}
|
||||
}
|
||||
func (w *writer) uvarint(v uint64) { w.bytes(w.buf[:binary.PutUvarint(w.buf[:], v)]) }
|
||||
func (w *writer) varint(v int64) { w.bytes(w.buf[:binary.PutVarint(w.buf[:], v)]) }
|
||||
func (w *writer) str(s string) { w.uvarint(uint64(len(s))); w.bytes([]byte(s)) }
|
||||
|
||||
// Save writes t to w. The caller must hold t's read lock.
|
||||
func Save(out io.Writer, t *inventory.Tree) error {
|
||||
bw := bufio.NewWriterSize(out, 1<<20)
|
||||
hdr := make([]byte, 0, 12)
|
||||
hdr = append(hdr, brand.SnapshotMagic...)
|
||||
hdr = binary.LittleEndian.AppendUint16(hdr, Version)
|
||||
hdr = binary.LittleEndian.AppendUint16(hdr, flagGzipped)
|
||||
bw.Write(hdr)
|
||||
|
||||
gz, _ := gzip.NewWriterLevel(bw, gzip.BestSpeed)
|
||||
h := sha256.New()
|
||||
w := &writer{w: io.MultiWriter(gz, h)}
|
||||
|
||||
st := &t.Stats
|
||||
host, _ := os.Hostname()
|
||||
m := meta{Root: st.Root, Created: time.Now().UTC(), Start: st.Start, End: st.End, Complete: st.Complete,
|
||||
Cancelled: st.Cancelled, Files: st.Files, Dirs: st.Dirs, Symlinks: st.Symlinks, Others: st.Others,
|
||||
ErrCounts: st.ErrCounts[:], Excluded: st.Excluded, SkippedMounts: st.SkippedMounts,
|
||||
VirtualSkipped: st.VirtualSkipped, LoopsSkipped: st.LoopsSkipped, BrokenLinks: st.BrokenLinks,
|
||||
HardlinkDups: st.HardlinkDups, Unscanned: st.Unscanned, Excludes: st.Excludes,
|
||||
OneFileSystem: st.OneFileSystem, Follow: st.Follow, Workers: st.Workers, Host: host,
|
||||
Generator: brand.Name + " " + brand.Version}
|
||||
mb, err := json.Marshal(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
w.uvarint(uint64(len(mb)))
|
||||
w.bytes(mb)
|
||||
|
||||
w.uvarint(uint64(t.ExtCount()))
|
||||
for i := range t.ExtCount() {
|
||||
w.str(t.ExtByIndex(i))
|
||||
}
|
||||
w.uvarint(uint64(t.Len()))
|
||||
for i := range t.Len() {
|
||||
id := inventory.NodeID(i)
|
||||
n := t.Node(id)
|
||||
if i == 0 {
|
||||
w.uvarint(0)
|
||||
} else {
|
||||
w.uvarint(uint64(id - n.Parent))
|
||||
}
|
||||
w.bytes([]byte{byte(n.Kind), byte(n.Cat)})
|
||||
w.uvarint(uint64(n.Flags))
|
||||
w.str(n.Name)
|
||||
w.varint(n.Size)
|
||||
w.varint(n.Alloc)
|
||||
w.varint(n.MTime)
|
||||
w.uvarint(uint64(n.Mode))
|
||||
w.uvarint(uint64(n.UID))
|
||||
w.uvarint(uint64(n.GID))
|
||||
w.uvarint(uint64(n.Nlink))
|
||||
w.uvarint(uint64(n.Ext))
|
||||
}
|
||||
w.uvarint(uint64(len(st.Errors)))
|
||||
for _, e := range st.Errors {
|
||||
w.uvarint(uint64(e.Node))
|
||||
w.bytes([]byte{byte(e.Kind)})
|
||||
w.str(e.Name)
|
||||
w.str(e.Msg)
|
||||
}
|
||||
if w.err != nil {
|
||||
return w.err
|
||||
}
|
||||
if err := gz.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
bw.Write(h.Sum(nil))
|
||||
return bw.Flush()
|
||||
}
|
||||
|
||||
// SaveFile writes a snapshot atomically (temp file + rename).
|
||||
func SaveFile(path string, t *inventory.Tree) error {
|
||||
tmp, err := os.CreateTemp(dirOf(path), ".mapsize-*.tmp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
if err := Save(tmp, t); err != nil {
|
||||
tmp.Close()
|
||||
return err
|
||||
}
|
||||
if err := tmp.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(tmp.Name(), path)
|
||||
}
|
||||
|
||||
func dirOf(p string) string {
|
||||
i := strings.LastIndexAny(p, `/\`)
|
||||
if i < 0 {
|
||||
return "."
|
||||
}
|
||||
if i == 0 {
|
||||
return p[:1]
|
||||
}
|
||||
return p[:i]
|
||||
}
|
||||
|
||||
type reader struct {
|
||||
r *bufio.Reader
|
||||
err error
|
||||
}
|
||||
|
||||
func (r *reader) fail(format string, a ...any) {
|
||||
if r.err == nil {
|
||||
r.err = fmt.Errorf("snapshot: "+format, a...)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *reader) byte1() byte {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
b, err := r.r.ReadByte()
|
||||
if err != nil {
|
||||
r.fail("truncated: %v", err)
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (r *reader) uvarint(max uint64, what string) uint64 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
v, err := binary.ReadUvarint(r.r)
|
||||
if err != nil {
|
||||
r.fail("bad %s: %v", what, err)
|
||||
return 0
|
||||
}
|
||||
if v > max {
|
||||
r.fail("%s %d exceeds limit %d", what, v, max)
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (r *reader) varint(what string) int64 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
v, err := binary.ReadVarint(r.r)
|
||||
if err != nil {
|
||||
r.fail("bad %s: %v", what, err)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (r *reader) bytesN(n uint64) []byte {
|
||||
if r.err != nil {
|
||||
return nil
|
||||
}
|
||||
b := make([]byte, n)
|
||||
if _, err := io.ReadFull(r.r, b); err != nil {
|
||||
r.fail("truncated: %v", err)
|
||||
return nil
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (r *reader) str(max uint64, what string) string {
|
||||
return string(r.bytesN(r.uvarint(max, what+" length")))
|
||||
}
|
||||
|
||||
// Load reads and validates a snapshot.
|
||||
func Load(in io.Reader) (*inventory.Tree, error) {
|
||||
br := bufio.NewReaderSize(in, 1<<20)
|
||||
hdr := make([]byte, len(brand.SnapshotMagic)+4)
|
||||
if _, err := io.ReadFull(br, hdr); err != nil {
|
||||
return nil, errors.New("snapshot: file too short")
|
||||
}
|
||||
if string(hdr[:8]) != brand.SnapshotMagic {
|
||||
return nil, errors.New("snapshot: not a " + brand.Name + " snapshot")
|
||||
}
|
||||
if v := binary.LittleEndian.Uint16(hdr[8:]); v != Version {
|
||||
return nil, fmt.Errorf("snapshot: unsupported version %d (this build reads %d)", v, Version)
|
||||
}
|
||||
if binary.LittleEndian.Uint16(hdr[10:])&flagGzipped == 0 {
|
||||
return nil, errors.New("snapshot: uncompressed bodies are not supported")
|
||||
}
|
||||
gz, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("snapshot: %w", err)
|
||||
}
|
||||
gz.Multistream(false)
|
||||
h := sha256.New()
|
||||
r := &reader{r: bufio.NewReaderSize(io.TeeReader(gz, h), 1<<16)}
|
||||
|
||||
var m meta
|
||||
if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil {
|
||||
if err := json.Unmarshal(mb, &m); err != nil {
|
||||
return nil, fmt.Errorf("snapshot: metadata: %w", err)
|
||||
}
|
||||
}
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
t := inventory.New(m.Root, inventory.KindDir)
|
||||
nExt := r.uvarint(maxExts, "extension count")
|
||||
exts := make([]string, 0, min(nExt, 1024))
|
||||
for range nExt {
|
||||
exts = append(exts, r.str(MaxExtLen, "extension"))
|
||||
}
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
extIndex := t.ExtIndexer()
|
||||
nNodes := r.uvarint(MaxNodes, "node count")
|
||||
if nNodes == 0 {
|
||||
return nil, errors.New("snapshot: no root node")
|
||||
}
|
||||
for i := uint64(0); i < nNodes && r.err == nil; i++ {
|
||||
pd := r.uvarint(MaxNodes, "parent")
|
||||
kind := inventory.Kind(r.byte1())
|
||||
cat := inventory.Category(r.byte1())
|
||||
flags := inventory.Flags(r.uvarint(1<<16-1, "flags"))
|
||||
name := r.str(MaxNameLen, "name")
|
||||
size, alloc, mtime := r.varint("size"), r.varint("alloc"), r.varint("mtime")
|
||||
mode := r.uvarint(1<<32-1, "mode")
|
||||
uid := r.uvarint(1<<32-1, "uid")
|
||||
gid := r.uvarint(1<<32-1, "gid")
|
||||
nlink := r.uvarint(1<<32-1, "nlink")
|
||||
ext := r.uvarint(nExt, "ext")
|
||||
if r.err != nil {
|
||||
break
|
||||
}
|
||||
if kind > inventory.KindOther || cat >= inventory.NumCategories {
|
||||
r.fail("node %d: invalid kind/category", i)
|
||||
break
|
||||
}
|
||||
if size < 0 || alloc < 0 {
|
||||
r.fail("node %d: negative size", i)
|
||||
break
|
||||
}
|
||||
var id inventory.NodeID
|
||||
if i == 0 {
|
||||
if pd != 0 || kind != inventory.KindDir && nNodes > 1 {
|
||||
r.fail("invalid root")
|
||||
break
|
||||
}
|
||||
id = 0
|
||||
rn := t.Node(0)
|
||||
rn.Kind = kind
|
||||
} else {
|
||||
if pd == 0 || pd > i {
|
||||
r.fail("node %d: invalid parent reference", i)
|
||||
break
|
||||
}
|
||||
parent := inventory.NodeID(i - pd)
|
||||
if t.Node(parent).Kind != inventory.KindDir {
|
||||
r.fail("node %d: parent is not a directory", i)
|
||||
break
|
||||
}
|
||||
if !validName(name) {
|
||||
r.fail("node %d: invalid name %q", i, name)
|
||||
break
|
||||
}
|
||||
id = t.Add(parent, name, kind)
|
||||
}
|
||||
n := t.Node(id)
|
||||
n.Cat, n.Flags = cat, flags
|
||||
n.Size, n.Alloc, n.MTime = size, alloc, mtime
|
||||
n.Mode, n.UID, n.GID, n.Nlink = uint32(mode), uint32(uid), uint32(gid), uint32(nlink)
|
||||
if kind != inventory.KindDir && ext < uint64(len(exts)) {
|
||||
n.Ext = extIndex(exts[ext])
|
||||
}
|
||||
}
|
||||
nErr := r.uvarint(inventory.MaxErrorRecords, "error count")
|
||||
for range nErr {
|
||||
node := r.uvarint(nNodes-1, "error node")
|
||||
kind := inventory.ErrKind(r.byte1())
|
||||
name := r.str(MaxNameLen, "error name")
|
||||
msg := r.str(MaxMsgLen, "error message")
|
||||
if r.err != nil {
|
||||
break
|
||||
}
|
||||
if kind >= inventory.NumErrKinds {
|
||||
kind = inventory.ErrOther
|
||||
}
|
||||
t.Stats.Errors = append(t.Stats.Errors, inventory.ErrorRecord{Node: inventory.NodeID(node), Kind: kind, Name: name, Msg: msg})
|
||||
}
|
||||
if r.err != nil {
|
||||
return nil, r.err
|
||||
}
|
||||
// The body must end here, then the checksum follows the gzip stream.
|
||||
if _, err := r.r.ReadByte(); err != io.EOF {
|
||||
return nil, errors.New("snapshot: trailing data in body")
|
||||
}
|
||||
sum := make([]byte, sha256.Size)
|
||||
if _, err := io.ReadFull(br, sum); err != nil {
|
||||
return nil, errors.New("snapshot: missing checksum")
|
||||
}
|
||||
if !bytes.Equal(sum, h.Sum(nil)) {
|
||||
return nil, errors.New("snapshot: checksum mismatch (file corrupt)")
|
||||
}
|
||||
|
||||
t.Recompute()
|
||||
st := &t.Stats
|
||||
st.Start, st.End, st.Complete, st.Cancelled = m.Start, m.End, true, m.Cancelled
|
||||
st.Files, st.Dirs, st.Symlinks, st.Others = m.Files, m.Dirs, m.Symlinks, m.Others
|
||||
copy(st.ErrCounts[:], m.ErrCounts)
|
||||
st.Excluded, st.SkippedMounts, st.VirtualSkipped = m.Excluded, m.SkippedMounts, m.VirtualSkipped
|
||||
st.LoopsSkipped, st.BrokenLinks, st.HardlinkDups, st.Unscanned = m.LoopsSkipped, m.BrokenLinks, m.HardlinkDups, m.Unscanned
|
||||
st.Excludes, st.OneFileSystem, st.Follow, st.Workers = m.Excludes, m.OneFileSystem, m.Follow, m.Workers
|
||||
st.FromSnapshot = fmt.Sprintf("%s (%s, %s)", m.Created.Local().Format("2006-01-02 15:04"), m.Host, m.Generator)
|
||||
return t, nil
|
||||
}
|
||||
|
||||
func validName(s string) bool {
|
||||
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\x00")
|
||||
}
|
||||
|
||||
// LoadFile opens and loads a snapshot file.
|
||||
func LoadFile(path string) (*inventory.Tree, error) {
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer f.Close()
|
||||
return Load(f)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
)
|
||||
|
||||
func sample() *inventory.Tree {
|
||||
t := inventory.New("/srv", inventory.KindDir)
|
||||
a := t.Add(0, "a", inventory.KindDir)
|
||||
b := t.Add(0, "b", inventory.KindDir)
|
||||
f1 := t.Add(a, "x.iso", inventory.KindFile)
|
||||
t.Node(f1).Size, t.Node(f1).Alloc = 1000, 1024
|
||||
f2 := t.Add(b, "y.log", inventory.KindFile)
|
||||
t.Node(f2).Size, t.Node(f2).Alloc = 50, 4096
|
||||
t.Node(f2).Flags = inventory.FlagSparse
|
||||
t.Recompute()
|
||||
t.Stats.Files, t.Stats.Dirs = 2, 3
|
||||
t.Stats.AddError(inventory.ErrorRecord{Node: b, Kind: inventory.ErrPermission, Msg: "denied"})
|
||||
return t
|
||||
}
|
||||
|
||||
func TestRoundTrip(t *testing.T) {
|
||||
src := sample()
|
||||
var buf bytes.Buffer
|
||||
if err := Save(&buf, src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got, err := Load(bytes.NewReader(buf.Bytes()))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.Len() != src.Len() {
|
||||
t.Fatalf("len %d want %d", got.Len(), src.Len())
|
||||
}
|
||||
for i := range src.Len() {
|
||||
a, b := src.Node(inventory.NodeID(i)), got.Node(inventory.NodeID(i))
|
||||
if a.Name != b.Name || a.TotSize != b.TotSize || a.TotAlloc != b.TotAlloc || a.Flags != b.Flags ||
|
||||
src.Path(inventory.NodeID(i)) != got.Path(inventory.NodeID(i)) || src.ExtName(a) != got.ExtName(b) {
|
||||
t.Fatalf("node %d differs: %+v vs %+v", i, a, b)
|
||||
}
|
||||
}
|
||||
if got.Stats.ErrCounts[inventory.ErrPermission] != 1 || len(got.Stats.Errors) != 1 {
|
||||
t.Fatalf("errors not restored: %+v", got.Stats)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorruptionDetected(t *testing.T) {
|
||||
var buf bytes.Buffer
|
||||
Save(&buf, sample())
|
||||
data := buf.Bytes()
|
||||
// Flip a byte in the checksum trailer.
|
||||
bad := append([]byte(nil), data...)
|
||||
bad[len(bad)-1] ^= 0xff
|
||||
if _, err := Load(bytes.NewReader(bad)); err == nil {
|
||||
t.Fatal("checksum corruption not detected")
|
||||
}
|
||||
// Every truncation must fail cleanly.
|
||||
for i := 0; i < len(data)-1; i += 7 {
|
||||
if _, err := Load(bytes.NewReader(data[:i])); err == nil {
|
||||
t.Fatalf("truncation at %d accepted", i)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func FuzzLoad(f *testing.F) {
|
||||
var buf bytes.Buffer
|
||||
Save(&buf, sample())
|
||||
f.Add(buf.Bytes())
|
||||
f.Add([]byte("MAPSIZE\x00\x01\x00\x01\x00"))
|
||||
f.Fuzz(func(t *testing.T, data []byte) {
|
||||
tr, err := Load(bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// A successfully loaded tree must be internally consistent.
|
||||
for i := 1; i < tr.Len(); i++ {
|
||||
if p := tr.Node(inventory.NodeID(i)).Parent; p >= inventory.NodeID(i) {
|
||||
t.Fatalf("parent %d >= child %d", p, i)
|
||||
}
|
||||
}
|
||||
_ = tr.Path(inventory.NodeID(tr.Len() - 1))
|
||||
})
|
||||
}
|
||||
|
||||
func TestCompare(t *testing.T) {
|
||||
old := sample()
|
||||
nw := inventory.New("/srv", inventory.KindDir)
|
||||
a := nw.Add(0, "a", inventory.KindDir)
|
||||
f1 := nw.Add(a, "x.iso", inventory.KindFile)
|
||||
nw.Node(f1).Size, nw.Node(f1).Alloc = 5000, 5120
|
||||
c := nw.Add(0, "c", inventory.KindDir)
|
||||
f3 := nw.Add(c, "z", inventory.KindFile)
|
||||
nw.Node(f3).Size, nw.Node(f3).Alloc = 10, 4096
|
||||
nw.Recompute()
|
||||
|
||||
d := Compare(old, nw)
|
||||
if d.StatusOf(f1, inventory.SizeAllocated) != Grew || d.Delta(f1, inventory.SizeAllocated) != 4096 {
|
||||
t.Fatalf("x.iso: %v %d", d.StatusOf(f1, 0), d.Delta(f1, 0))
|
||||
}
|
||||
if d.StatusOf(c, 0) != Added {
|
||||
t.Fatal("c should be new")
|
||||
}
|
||||
if len(d.Removed) != 1 || old.Node(d.Removed[0]).Name != "b" {
|
||||
t.Fatalf("removed %v", d.Removed)
|
||||
}
|
||||
ch := d.Changes(10, inventory.SizeAllocated)
|
||||
paths := map[string]Status{}
|
||||
for _, c := range ch {
|
||||
paths[c.Path] = c.Status
|
||||
}
|
||||
if paths["/srv/a/x.iso"] != Grew || paths["/srv/b"] != Removed || paths["/srv/c"] != Added {
|
||||
t.Fatalf("changes %+v", ch)
|
||||
}
|
||||
if _, ok := paths["/srv/a"]; ok {
|
||||
t.Fatal("/srv/a should be explained by its child")
|
||||
}
|
||||
if _, ok := paths["/srv/c/z"]; ok {
|
||||
t.Fatal("contents of new dirs should not be listed separately")
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user