andy/mapsize
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Add CLI reports, snapshot format and duplicate detection

Browse Source
This commit is contained in:
Andy Dixon
2026-10-02 08:02:32 +01:00
parent 8233ddcfba
commit 900931ae26
9 changed files with 1734 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mapsize/main.go
+199
View File
@@ -0,0 +1,199 @@
// Command mapsize is an interactive terminal disk usage analyser.
package main
import (
"flag"
"fmt"
"io"
"log/slog"
"os"
"runtime"
"runtime/pprof"
"strings"
"github.com/andydixon/mapsize/internal/app"
"github.com/andydixon/mapsize/internal/brand"
"github.com/andydixon/mapsize/internal/config"
"github.com/andydixon/mapsize/internal/inventory"
"github.com/andydixon/mapsize/internal/scan"
"github.com/andydixon/mapsize/internal/snapshot"
"github.com/andydixon/mapsize/internal/textutil"
)
type multiFlag []string
func (m *multiFlag) String() string { return strings.Join(*m, ",") }
func (m *multiFlag) Set(s string) error { *m = append(*m, s); return nil }
func main() {
os.Exit(run(os.Args[1:]))
}
func run(args []string) int {
settings, cfgErr := config.Load()
fs := flag.NewFlagSet(brand.Name, flag.ContinueOnError)
fs.Usage = func() { usage(fs) }
var (
excludes multiFlag
workers = fs.String("workers", settings.Workers, "concurrency: conservative, balanced, aggressive or a number")
follow = fs.String("follow-symlinks", settings.FollowSymlinks, "none, same-filesystem or all")
oneFS = fs.Bool("one-file-system", settings.OneFileSystem, "do not cross filesystem boundaries")
oneFSx = fs.Bool("x", false, "short for --one-file-system")
readOnly = fs.Bool("read-only", false, "disable all modifying actions")
noUI = fs.Bool("no-ui", false, "scan without the interactive interface and print a summary")
jsonOut = fs.Bool("json", false, "print the inventory as JSON")
csvOut = fs.Bool("csv", false, "print the inventory as CSV")
depth = fs.Int("depth", -1, "maximum depth for --json/--csv (-1 = unlimited)")
top = fs.Int("top", 0, "print the N largest entries of the root")
largest = fs.Int("largest-files", 0, "print the N largest files")
dups = fs.Bool("duplicates", false, "find duplicate files (non-interactive)")
save = fs.String("save", "", "save a snapshot to `FILE` after scanning")
snap = fs.String("snapshot", "", "alias for --save")
load = fs.String("load", "", "open a saved snapshot `FILE` instead of scanning")
scanPath = fs.String("scan", "", "path to scan (alternative to a positional argument)")
compare = fs.Bool("compare", false, "compare two snapshots: --compare OLD NEW")
si = fs.Bool("si", settings.Units == "si", "use SI units (kB, MB) instead of IEC (KiB, MiB)")
logical = fs.Bool("apparent", settings.SizeMode == "logical", "size by logical (apparent) size instead of disk usage")
theme = fs.String("theme", settings.Theme, "colour theme: default, dark, high-contrast, mono")
noMouse = fs.Bool("no-mouse", !settings.Mouse, "disable mouse support")
logFile = fs.String("log", "", "write a debug log to `FILE`")
logLevel = fs.String("log-level", "info", "log level: error, warn, info, debug, trace")
cpuProfile = fs.String("cpuprofile", "", "write a CPU profile to `FILE`")
memProfile = fs.String("memprofile", "", "write a heap profile to `FILE` on exit")
version = fs.Bool("version", false, "print version and exit")
)
fs.Var(&excludes, "exclude", "exclude `PATTERN` (glob on name, or path prefix if it contains a separator); repeatable")
var positional []string
for {
if err := fs.Parse(args); err != nil {
if err == flag.ErrHelp {
return 0
}
return 2
}
args = fs.Args()
if len(args) == 0 {
break
}
positional = append(positional, args[0])
args = args[1:]
}
if *version {
fmt.Printf("%s %s (%s/%s)\n", brand.Name, brand.Version, runtime.GOOS, runtime.GOARCH)
return 0
}
if cfgErr != nil {
fmt.Fprintf(os.Stderr, "%s: warning: config: %v (using defaults)\n", brand.Name, cfgErr)
}
textutil.SI = *si
if *logFile != "" {
closeLog, err := setupLog(*logFile, *logLevel)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err)
return 2
}
defer closeLog()
} else {
slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil)))
}
if *cpuProfile != "" {
f, err := os.Create(*cpuProfile)
if err == nil {
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
}
if *memProfile != "" {
defer func() {
if f, err := os.Create(*memProfile); err == nil {
runtime.GC()
pprof.WriteHeapProfile(f)
f.Close()
}
}()
}
cfg := &app.Config{
NoUI: *noUI, JSON: *jsonOut, CSV: *csvOut, Depth: *depth, Top: *top, LargestFiles: *largest,
Duplicates: *dups, ReadOnly: *readOnly, Theme: *theme, Mouse: !*noMouse, Load: *load,
Settings: settings,
}
if *logical {
cfg.SizeMode = inventory.SizeLogical
}
cfg.Save = *save
if cfg.Save == "" {
cfg.Save = *snap
}
if *scanPath != "" {
positional = append([]string{*scanPath}, positional...)
}
if *compare {
if len(positional) != 2 {
fmt.Fprintf(os.Stderr, "%s: --compare needs exactly two snapshot files\n", brand.Name)
return 2
}
cfg.Compare = [2]string{positional[0], positional[1]}
} else {
if len(positional) > 1 {
fmt.Fprintf(os.Stderr, "%s: only one path may be scanned at a time\n", brand.Name)
return 2
}
if len(positional) == 1 && cfg.Load == "" && snapshot.IsSnapshot(positional[0]) {
cfg.Load = positional[0]
} else {
cfg.Paths = positional
}
}
w, err := scan.Workers(*workers)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err)
return 2
}
fm, err := scan.ParseFollow(*follow)
if err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err)
return 2
}
cfg.ScanOpts = scan.Options{Workers: w, Follow: fm, OneFileSystem: *oneFS || *oneFSx,
Excludes: append(append([]string(nil), settings.Excludes...), excludes...)}
if err := app.Run(cfg); err != nil {
fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, textutil.Sanitize(err.Error()))
return 1
}
return 0
}
func setupLog(path, level string) (func(), error) {
lv := map[string]slog.Level{"error": slog.LevelError, "warn": slog.LevelWarn, "info": slog.LevelInfo,
"debug": slog.LevelDebug, "trace": slog.LevelDebug - 4}
l, ok := lv[level]
if !ok {
return nil, fmt.Errorf("invalid --log-level %q", level)
}
f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600)
if err != nil {
return nil, err
}
slog.SetDefault(slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{Level: l})))
return func() { f.Close() }, nil
}
func usage(fs *flag.FlagSet) {
w := fs.Output()
fmt.Fprintf(w, `%[1]s — interactive terminal disk usage analyser
Usage:
%[1]s [flags] [PATH] scan PATH (default .) interactively
%[1]s [flags] SNAPSHOT%[2]s open a saved snapshot
%[1]s --compare OLD NEW compare two snapshots
%[1]s PATH --top 50 | --json | --csv | --no-ui --save FILE
Flags:
`, brand.Name, brand.SnapshotExt)
fs.PrintDefaults()
}
internal/app/app.go
+169
View File
@@ -0,0 +1,169 @@
// Package app wires the command-line modes together: interactive TUI,
// non-interactive reports, snapshots and comparisons. TUI and CLI share the
// same scanner and inventory packages.
package app
import (
"context"
"fmt"
"io"
"log/slog"
"os"
"os/signal"
"time"
"github.com/andydixon/mapsize/internal/config"
"github.com/andydixon/mapsize/internal/export"
"github.com/andydixon/mapsize/internal/inventory"
"github.com/andydixon/mapsize/internal/scan"
"github.com/andydixon/mapsize/internal/textutil"
)
// Config is the fully-resolved command line.
type Config struct {
Paths []string
ScanOpts scan.Options
NoUI bool
JSON, CSV bool
Depth int
Top int
LargestFiles int
Save string
Load string
Compare [2]string
Duplicates bool
ReadOnly bool
Theme string
SizeMode inventory.SizeMode
Mouse bool
Settings config.Settings
Stdout io.Writer
Stderr io.Writer
}
// Report reports whether any non-interactive output was requested.
func (c *Config) Report() bool {
return c.NoUI || c.JSON || c.CSV || c.Top > 0 || c.LargestFiles > 0 || c.Duplicates
}
// Run executes the configured mode.
func Run(cfg *Config) error {
if cfg.Stdout == nil {
cfg.Stdout = os.Stdout
}
if cfg.Stderr == nil {
cfg.Stderr = os.Stderr
}
if cfg.Compare[0] != "" {
return runCompare(cfg)
}
if !cfg.Report() {
return runTUI(cfg)
}
return runReport(cfg)
}
// startSource begins a scan (or loads a snapshot). The scanner is nil when
// the tree came from a snapshot.
func startSource(ctx context.Context, cfg *Config) (*scan.Scanner, *inventory.Tree, error) {
if cfg.Load != "" {
t, err := loadSnapshot(cfg.Load)
return nil, t, err
}
root := "."
if len(cfg.Paths) > 0 {
root = cfg.Paths[0]
}
slog.Info("scan start", "workers", cfg.ScanOpts.Workers, "follow", cfg.ScanOpts.Follow.String())
s, err := scan.Start(ctx, root, cfg.ScanOpts)
if err != nil {
return nil, nil, err
}
return s, s.Tree, nil
}
func runReport(cfg *Config) error {
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
defer stop()
s, t, err := startSource(ctx, cfg)
if err != nil {
return err
}
if s != nil {
waitWithProgress(s, cfg.Stderr)
}
t.RLock()
defer t.RUnlock()
if s != nil && cfg.Save != "" {
if err := saveSnapshot(t, cfg.Save); err != nil {
return err
}
fmt.Fprintf(cfg.Stderr, "Saved snapshot %s\n", cfg.Save)
}
out := cfg.Stdout
switch {
case cfg.JSON:
return export.JSON(out, t, t.Root(), cfg.Depth)
case cfg.CSV:
return export.CSV(out, t, t.Root(), cfg.Depth)
}
if cfg.Duplicates {
if err := runDuplicates(ctx, cfg, t); err != nil {
return err
}
}
if cfg.Top > 0 {
ids := t.SortedChildren(t.Root(), cfg.SizeMode)
if len(ids) > cfg.Top {
ids = ids[:cfg.Top]
}
export.Table(out, t, ids, cfg.SizeMode)
fmt.Fprintln(out)
}
if cfg.LargestFiles > 0 {
ids := t.Top(t.Root(), cfg.LargestFiles,
func(n *inventory.Node) bool {
return n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0
},
func(n *inventory.Node) int64 { return n.Own(cfg.SizeMode) })
export.Table(out, t, ids, cfg.SizeMode)
fmt.Fprintln(out)
}
export.Summary(out, t)
return nil
}
// waitWithProgress blocks until the scan finishes, printing a progress line
// to w if it is a terminal.
func waitWithProgress(s *scan.Scanner, w io.Writer) {
tty := isTerminal(w)
tick := time.NewTicker(250 * time.Millisecond)
defer tick.Stop()
start := time.Now()
for {
select {
case <-s.Done():
if tty {
fmt.Fprint(w, "\r\033[K")
}
return
case <-tick.C:
if tty {
p := s.Progress()
el := time.Since(start)
fmt.Fprintf(w, "\r\033[Kscanning… %s entries %s/s queue %d %s",
textutil.Count(p.Entries), textutil.Count(int64(float64(p.Entries)/el.Seconds())),
p.Pending, textutil.Duration(el))
}
}
}
}
func isTerminal(w io.Writer) bool {
f, ok := w.(*os.File)
if !ok {
return false
}
fi, err := f.Stat()
return err == nil && fi.Mode()&os.ModeCharDevice != 0
}
internal/app/modes.go
+68
View File
@@ -0,0 +1,68 @@
package app
import (
"context"
"errors"
"fmt"
"github.com/andydixon/mapsize/internal/duplicate"
"github.com/andydixon/mapsize/internal/inventory"
"github.com/andydixon/mapsize/internal/snapshot"
"github.com/andydixon/mapsize/internal/textutil"
)
func runTUI(*Config) error {
return errors.New("the interactive UI is not built yet; use --no-ui, --top, --json or --csv")
}
func loadSnapshot(path string) (*inventory.Tree, error) { return snapshot.LoadFile(path) }
func saveSnapshot(t *inventory.Tree, path string) error { return snapshot.SaveFile(path, t) }
func runCompare(cfg *Config) error {
old, err := snapshot.LoadFile(cfg.Compare[0])
if err != nil {
return fmt.Errorf("%s: %w", cfg.Compare[0], err)
}
nw, err := snapshot.LoadFile(cfg.Compare[1])
if err != nil {
return fmt.Errorf("%s: %w", cfg.Compare[1], err)
}
d := snapshot.Compare(old, nw)
k := cfg.Top
if k <= 0 {
k = 30
}
out := cfg.Stdout
ro, rn := old.Node(0), nw.Node(0)
fmt.Fprintf(out, "%s → %s\n", textutil.Sanitize(old.Stats.FromSnapshot), textutil.Sanitize(nw.Stats.FromSnapshot))
fmt.Fprintf(out, "Total %s → %s (%s)\n\n", textutil.Size(ro.Total(cfg.SizeMode)), textutil.Size(rn.Total(cfg.SizeMode)),
textutil.SignedSize(rn.Total(cfg.SizeMode)-ro.Total(cfg.SizeMode)))
for _, c := range d.Changes(k, cfg.SizeMode) {
fmt.Fprintf(out, "%-10s %14s %s\n", c.Status, textutil.SignedSize(c.Delta), textutil.Sanitize(c.Path))
}
return nil
}
func runDuplicates(ctx context.Context, cfg *Config, t *inventory.Tree) error {
// The caller holds RLock; Find takes its own read lock, which is fine
// for a RWMutex only if no writer is waiting — the scan is finished.
var f duplicate.Finder
groups, err := f.Find(ctx, t, duplicate.Options{MinSize: 1})
if err != nil {
return err
}
out := cfg.Stdout
var wasted int64
for _, g := range groups {
wasted += g.Wasted()
fmt.Fprintf(out, "%s × %d (sha256 %x…)\n", textutil.Size(g.Size), len(g.Files), g.Hash[:6])
for _, id := range g.Files {
fmt.Fprintf(out, " %s\n", textutil.Sanitize(t.Path(id)))
}
}
p := f.Progress()
fmt.Fprintf(out, "\n%d verified duplicate groups, %s reclaimable (%d candidates by size, %d unreadable/changed)\n\n",
len(groups), textutil.Size(wasted), p.Candidates, p.Skipped)
return nil
}
internal/duplicate/duplicate.go
+285
View File
@@ -0,0 +1,285 @@
// Package duplicate finds files with identical content in stages:
//
// 1. group by exact size (inventory only, no I/O)
// 2. hash three 16 KiB samples (start, middle, end) of each candidate
// 3. full SHA-256 of files whose samples still collide
//
// Only stage 3 establishes identity. Files are never declared duplicates by
// name or size alone. Reading files may update their access times.
package duplicate
import (
"context"
"crypto/sha256"
"io"
"os"
"sort"
"sync"
"sync/atomic"
"github.com/andydixon/mapsize/internal/inventory"
)
// Group is a set of files verified to have identical content.
type Group struct {
Size int64
Hash [32]byte
Files []inventory.NodeID
}
// Wasted returns the bytes reclaimable by keeping one copy.
func (g *Group) Wasted() int64 { return g.Size * int64(len(g.Files)-1) }
// Progress reports activity.
type Progress struct {
Stage string
Candidates int64 // files sharing a size with another file
BytesToHash int64
BytesHashed int64
Skipped int64 // unreadable or changed during hashing
VerifiedGroups int64
CandidateGroups int64
}
// Options tunes the search.
type Options struct {
MinSize int64 // ignore files smaller than this (default 1 byte)
Workers int // concurrent readers (default 4)
}
type cand struct {
id inventory.NodeID
path string
size int64
}
const sampleSize = 16 << 10
// Finder runs a search and exposes live progress.
type Finder struct {
hashed, toHash, skipped atomic.Int64
mu sync.Mutex
p Progress
}
// Progress returns a copy of the current progress.
func (f *Finder) Progress() Progress {
f.mu.Lock()
p := f.p
f.mu.Unlock()
p.BytesHashed, p.BytesToHash, p.Skipped = f.hashed.Load(), f.toHash.Load(), f.skipped.Load()
return p
}
func (f *Finder) stage(s string, fn func(*Progress)) {
f.mu.Lock()
f.p.Stage = s
if fn != nil {
fn(&f.p)
}
f.mu.Unlock()
}
// Find searches t (read-locked internally only while collecting candidates).
func (f *Finder) Find(ctx context.Context, t *inventory.Tree, o Options) ([]Group, error) {
if o.MinSize < 1 {
o.MinSize = 1
}
if o.Workers < 1 {
o.Workers = 4
}
f.stage("grouping by size", nil)
bySize := map[int64][]cand{}
t.RLock()
t.Walk(t.Root(), func(id inventory.NodeID, n *inventory.Node) bool {
if n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0 && n.Size >= o.MinSize {
bySize[n.Size] = append(bySize[n.Size], cand{id: id, size: n.Size})
}
return true
})
var groups [][]cand
var ncand int64
for _, g := range bySize {
if len(g) > 1 {
for i := range g {
g[i].path = t.Path(g[i].id)
}
groups = append(groups, g)
ncand += int64(len(g))
}
}
t.RUnlock()
f.stage("sampling", func(p *Progress) { p.Candidates = ncand; p.CandidateGroups = int64(len(groups)) })
// Stage 2: sample hashes.
var sampled [][]cand
for _, g := range groups {
if ctx.Err() != nil {
return nil, ctx.Err()
}
if g[0].size <= 3*sampleSize {
sampled = append(sampled, g) // a sample would be the whole file
continue
}
sampled = append(sampled, f.split(ctx, g, o.Workers, sampleHash)...)
}
// Stage 3: full hashes.
var total int64
for _, g := range sampled {
total += g[0].size * int64(len(g))
}
f.toHash.Store(total)
f.stage("hashing", nil)
var out []Group
for _, g := range sampled {
if ctx.Err() != nil {
return nil, ctx.Err()
}
for _, sub := range f.splitHash(ctx, g, o.Workers) {
grp := Group{Size: sub.size, Hash: sub.hash}
for _, c := range sub.files {
grp.Files = append(grp.Files, c.id)
}
out = append(out, grp)
}
}
sort.Slice(out, func(i, j int) bool {
if a, b := out[i].Wasted(), out[j].Wasted(); a != b {
return a > b
}
return out[i].Files[0] < out[j].Files[0]
})
f.stage("done", func(p *Progress) { p.VerifiedGroups = int64(len(out)) })
return out, ctx.Err()
}
type hashed struct {
c cand
sum [32]byte
ok bool
}
// hashAll hashes every candidate with a bounded worker pool.
func (f *Finder) hashAll(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) []hashed {
res := make([]hashed, len(g))
var wg sync.WaitGroup
sem := make(chan struct{}, workers)
for i, c := range g {
if ctx.Err() != nil {
break
}
sem <- struct{}{}
wg.Add(1)
go func() {
defer wg.Done()
defer func() { <-sem }()
sum, ok := fn(f, c)
if !ok {
f.skipped.Add(1)
}
res[i] = hashed{c, sum, ok}
}()
}
wg.Wait()
return res
}
func (f *Finder) split(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) [][]cand {
by := map[[32]byte][]cand{}
for _, h := range f.hashAll(ctx, g, workers, fn) {
if h.ok {
by[h.sum] = append(by[h.sum], h.c)
}
}
var out [][]cand
for _, v := range by {
if len(v) > 1 {
out = append(out, v)
}
}
return out
}
type fullGroup struct {
size int64
hash [32]byte
files []cand
}
func (f *Finder) splitHash(ctx context.Context, g []cand, workers int) []fullGroup {
by := map[[32]byte][]cand{}
for _, h := range f.hashAll(ctx, g, workers, fullHash) {
if h.ok {
by[h.sum] = append(by[h.sum], h.c)
}
}
var out []fullGroup
for sum, v := range by {
if len(v) > 1 {
sort.Slice(v, func(i, j int) bool { return v[i].id < v[j].id })
out = append(out, fullGroup{size: v[0].size, hash: sum, files: v})
}
}
return out
}
// open opens a candidate and checks it still has the inventoried size.
func open(c cand) (*os.File, bool) {
fh, err := os.Open(c.path)
if err != nil {
return nil, false
}
fi, err := fh.Stat()
if err != nil || !fi.Mode().IsRegular() || fi.Size() != c.size {
fh.Close()
return nil, false
}
return fh, true
}
func sampleHash(_ *Finder, c cand) ([32]byte, bool) {
fh, ok := open(c)
if !ok {
return [32]byte{}, false
}
defer fh.Close()
h := sha256.New()
buf := make([]byte, sampleSize)
for _, off := range []int64{0, c.size/2 - sampleSize/2, c.size - sampleSize} {
if _, err := fh.ReadAt(buf, off); err != nil && err != io.EOF {
return [32]byte{}, false
}
h.Write(buf)
}
var s [32]byte
h.Sum(s[:0])
return s, true
}
func fullHash(f *Finder, c cand) ([32]byte, bool) {
fh, ok := open(c)
if !ok {
return [32]byte{}, false
}
defer fh.Close()
h := sha256.New()
n, err := io.Copy(h, &countingReader{fh, &f.hashed})
if err != nil || n != c.size {
return [32]byte{}, false
}
var s [32]byte
h.Sum(s[:0])
return s, true
}
type countingReader struct {
r io.Reader
n *atomic.Int64
}
func (c *countingReader) Read(p []byte) (int, error) {
n, err := c.r.Read(p)
c.n.Add(int64(n))
return n, err
}
internal/duplicate/duplicate_test.go
+61
View File
@@ -0,0 +1,61 @@
package duplicate
import (
"context"
"os"
"path/filepath"
"testing"
"github.com/andydixon/mapsize/internal/scan"
)
func TestFind(t *testing.T) {
root := t.TempDir()
big := make([]byte, 200_000)
for i := range big {
big[i] = byte(i * 7)
}
w := func(name string, b []byte) {
if err := os.WriteFile(filepath.Join(root, name), b, 0o644); err != nil {
t.Fatal(err)
}
}
w("a.bin", big)
w("b.bin", big)
// Same size, same samples, different middle-of-nowhere byte: must not match.
c := append([]byte(nil), big...)
c[20_000] ^= 1
w("c.bin", c)
w("small1", []byte("hello"))
w("small2", []byte("hello"))
w("small3", []byte("world")) // same size, different content
s, err := scan.Start(context.Background(), root, scan.Options{Workers: 2})
if err != nil {
t.Fatal(err)
}
<-s.Done()
var f Finder
groups, err := f.Find(context.Background(), s.Tree, Options{})
if err != nil {
t.Fatal(err)
}
if len(groups) != 2 {
t.Fatalf("groups=%d want 2", len(groups))
}
names := func(g Group) (out []string) {
for _, id := range g.Files {
out = append(out, s.Tree.Node(id).Name)
}
return
}
if g := names(groups[0]); len(g) != 2 || groups[0].Size != 200_000 {
t.Fatalf("big group %v", g)
}
if g := names(groups[1]); len(g) != 2 {
t.Fatalf("small group %v", g)
}
if p := f.Progress(); p.Candidates != 6 || p.VerifiedGroups != 2 {
t.Fatalf("progress %+v", p)
}
}
internal/export/export.go
+219
View File
@@ -0,0 +1,219 @@
// Package export writes inventories as JSON, CSV or plain-text reports.
// Callers must hold the tree's read lock (or own the tree exclusively).
package export
import (
"bufio"
"encoding/csv"
"encoding/json"
"fmt"
"io"
"strconv"
"time"
"github.com/andydixon/mapsize/internal/brand"
"github.com/andydixon/mapsize/internal/inventory"
"github.com/andydixon/mapsize/internal/textutil"
)
type jsonNode struct {
Name string `json:"name"`
Type string `json:"type"`
Size int64 `json:"size"`
Allocated int64 `json:"allocated"`
Files uint32 `json:"files,omitempty"`
Dirs uint32 `json:"dirs,omitempty"`
Errors uint32 `json:"errors,omitempty"`
Modified string `json:"modified,omitempty"`
Flags []string `json:"flags,omitempty"`
Children []*jsonNode `json:"children,omitempty"`
}
var flagNames = []struct {
f inventory.Flags
n string
}{
{inventory.FlagHardlinkDup, "hardlink-duplicate"}, {inventory.FlagMountPoint, "mount-point"},
{inventory.FlagVirtualFS, "virtual-fs-skipped"}, {inventory.FlagSkippedFS, "other-fs-skipped"},
{inventory.FlagSparse, "sparse"}, {inventory.FlagError, "error"}, {inventory.FlagIncomplete, "incomplete"},
{inventory.FlagLoop, "loop-skipped"}, {inventory.FlagFollowed, "followed-symlink"},
{inventory.FlagAllocUnknown, "allocated-unknown"}, {inventory.FlagBrokenLink, "broken-symlink"},
{inventory.FlagHardlinked, "hardlinked"},
}
// FlagList returns human-readable names for a node's flags.
func FlagList(f inventory.Flags) []string {
var out []string
for _, fn := range flagNames {
if f&fn.f != 0 {
out = append(out, fn.n)
}
}
return out
}
// JSON writes the tree beneath root as nested JSON, down to maxDepth levels
// (maxDepth < 0 means unlimited).
func JSON(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error {
bw := bufio.NewWriter(w)
st := &t.Stats
head := map[string]any{
"generator": brand.Name + " " + brand.Version,
"root": t.Path(root),
"complete": !st.Incomplete(),
"cancelled": st.Cancelled,
"started": st.Start.Format(time.RFC3339),
"elapsed_s": st.Elapsed().Seconds(),
"errors": errorCounts(st),
"excluded": st.Excluded,
"excludes": st.Excludes,
"hardlink_duplicates": st.HardlinkDups,
}
hb, err := json.Marshal(head)
if err != nil {
return err
}
bw.Write(hb[:len(hb)-1])
bw.WriteString(`,"tree":`)
if err := writeNode(bw, t, root, maxDepth); err != nil {
return err
}
bw.WriteString("}\n")
return bw.Flush()
}
func errorCounts(st *inventory.Stats) map[string]int64 {
m := map[string]int64{}
for k, c := range st.ErrCounts {
if c > 0 {
m[inventory.ErrKind(k).String()] = c
}
}
return m
}
// writeNode streams one node; children are written recursively so memory
// stays proportional to depth, not tree size.
func writeNode(bw *bufio.Writer, t *inventory.Tree, id inventory.NodeID, depth int) error {
n := t.Node(id)
jn := jsonNode{Name: n.Name, Type: kindName(n.Kind), Size: n.TotSize, Allocated: n.TotAlloc,
Files: n.Files, Dirs: n.Dirs, Errors: n.Errors, Flags: FlagList(n.Flags)}
if n.MTime != 0 {
jn.Modified = time.Unix(0, n.MTime).UTC().Format(time.RFC3339)
}
b, err := json.Marshal(jn)
if err != nil {
return err
}
if n.FirstChild == inventory.NoNode || depth == 0 {
_, err = bw.Write(b)
return err
}
bw.Write(b[:len(b)-1])
bw.WriteString(`,"children":[`)
first := true
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
if !first {
bw.WriteByte(',')
}
first = false
if err := writeNode(bw, t, c, depth-1); err != nil {
return err
}
}
_, err = bw.WriteString("]}")
return err
}
func kindName(k inventory.Kind) string {
return [...]string{"file", "dir", "symlink", "other"}[k&3]
}
// CSV writes one row per node beneath root (depth-limited). Paths are
// sanitised so the output is safe to print to a terminal.
func CSV(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error {
cw := csv.NewWriter(w)
cw.Write([]string{"path", "type", "size", "allocated", "files", "dirs", "errors", "modified", "flags"})
var rec func(id inventory.NodeID, path string, depth int) error
rec = func(id inventory.NodeID, path string, depth int) error {
n := t.Node(id)
mod := ""
if n.MTime != 0 {
mod = time.Unix(0, n.MTime).UTC().Format(time.RFC3339)
}
flags := ""
for i, f := range FlagList(n.Flags) {
if i > 0 {
flags += "|"
}
flags += f
}
if err := cw.Write([]string{textutil.Sanitize(path), kindName(n.Kind),
strconv.FormatInt(n.TotSize, 10), strconv.FormatInt(n.TotAlloc, 10),
strconv.FormatUint(uint64(n.Files), 10), strconv.FormatUint(uint64(n.Dirs), 10),
strconv.FormatUint(uint64(n.Errors), 10), mod, flags}); err != nil {
return err
}
if depth == 0 {
return nil
}
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
if err := rec(c, path+"/"+t.Node(c).Name, depth-1); err != nil {
return err
}
}
return nil
}
if err := rec(root, t.Path(root), maxDepth); err != nil {
return err
}
cw.Flush()
return cw.Error()
}
// Table writes "size % files path" rows for the given nodes.
func Table(w io.Writer, t *inventory.Tree, ids []inventory.NodeID, m inventory.SizeMode) {
total := t.Node(t.Root()).Total(m)
fmt.Fprintf(w, "%12s %6s %12s %s\n", "SIZE", "%", "FILES", "PATH")
for _, id := range ids {
n := t.Node(id)
files := ""
if n.IsDir() {
files = textutil.Count(int64(n.Files))
}
fmt.Fprintf(w, "%12s %6s %12s %s\n", textutil.Size(n.Total(m)), textutil.Percent(n.Total(m), total),
files, textutil.Sanitize(t.Path(id)))
}
}
// Summary writes the scan summary, making incompleteness explicit.
func Summary(w io.Writer, t *inventory.Tree) {
st := &t.Stats
r := t.Node(t.Root())
fmt.Fprintf(w, "Root %s\n", textutil.Sanitize(st.Root))
fmt.Fprintf(w, "Files %s\n", textutil.Count(st.Files+st.Symlinks+st.Others))
fmt.Fprintf(w, "Directories %s\n", textutil.Count(st.Dirs))
fmt.Fprintf(w, "Logical size %s (%d bytes)\n", textutil.Size(r.TotSize), r.TotSize)
fmt.Fprintf(w, "Allocated size %s (%d bytes)\n", textutil.Size(r.TotAlloc), r.TotAlloc)
fmt.Fprintf(w, "Elapsed %s\n", st.Elapsed().Round(time.Millisecond))
for k, c := range st.ErrCounts {
if c > 0 {
fmt.Fprintf(w, "%-15s %s\n", inventory.ErrKind(k).String(), textutil.Count(c))
}
}
if st.Excluded > 0 {
fmt.Fprintf(w, "Excluded %s entries (%v)\n", textutil.Count(st.Excluded), st.Excludes)
}
if st.SkippedMounts > 0 {
fmt.Fprintf(w, "Other FS skipped %s\n", textutil.Count(st.SkippedMounts))
}
if st.VirtualSkipped > 0 {
fmt.Fprintf(w, "Virtual FS skipped %s\n", textutil.Count(st.VirtualSkipped))
}
if st.HardlinkDups > 0 {
fmt.Fprintf(w, "Hard-link dups %s (counted once)\n", textutil.Count(st.HardlinkDups))
}
if st.Incomplete() {
fmt.Fprintln(w, "WARNING: totals are incomplete — some data could not be inspected.")
}
}
internal/snapshot/compare.go
+177
View File
@@ -0,0 +1,177 @@
package snapshot
import (
"sort"
"github.com/andydixon/mapsize/internal/inventory"
)
// Diff relates two inventories by path. Both trees must not be mutated
// while a Diff is in use (snapshots and finished scans satisfy this).
type Diff struct {
Old, New *inventory.Tree
// OldOf maps a New NodeID to the matching Old NodeID, or NoNode if the
// entry is new.
OldOf []inventory.NodeID
// Removed lists the topmost Old nodes with no counterpart in New.
Removed []inventory.NodeID
// RemovedUnder sums, per New directory, the sizes of removed children
// (in both size modes) so growth colouring can account for them.
removedUnder map[inventory.NodeID][2]int64
}
// Compare matches nodes by name, directory by directory, from the roots.
func Compare(old, new *inventory.Tree) *Diff {
d := &Diff{Old: old, New: new, OldOf: make([]inventory.NodeID, new.Len()),
removedUnder: map[inventory.NodeID][2]int64{}}
for i := range d.OldOf {
d.OldOf[i] = inventory.NoNode
}
type pair struct{ o, n inventory.NodeID }
stack := []pair{{old.Root(), new.Root()}}
for len(stack) > 0 {
p := stack[len(stack)-1]
stack = stack[:len(stack)-1]
d.OldOf[p.n] = p.o
on, nn := old.Node(p.o), new.Node(p.n)
if !on.IsDir() || !nn.IsDir() {
continue
}
byName := map[string]inventory.NodeID{}
old.Children(p.o, func(id inventory.NodeID, n *inventory.Node) { byName[n.Name] = id })
new.Children(p.n, func(id inventory.NodeID, n *inventory.Node) {
if oid, ok := byName[n.Name]; ok {
delete(byName, n.Name)
stack = append(stack, pair{oid, id})
}
})
for _, oid := range byName {
d.Removed = append(d.Removed, oid)
o := old.Node(oid)
r := d.removedUnder[p.n]
r[0] += o.TotAlloc
r[1] += o.TotSize
d.removedUnder[p.n] = r
}
}
sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i] < d.Removed[j] })
return d
}
// OldSize returns the matching old size of a New node and whether it existed.
func (d *Diff) OldSize(id inventory.NodeID, m inventory.SizeMode) (int64, bool) {
o := d.OldOf[id]
if o == inventory.NoNode {
return 0, false
}
return d.Old.Node(o).Total(m), true
}
// Delta returns new minus old size for a New node (full size if new).
func (d *Diff) Delta(id inventory.NodeID, m inventory.SizeMode) int64 {
old, _ := d.OldSize(id, m)
return d.New.Node(id).Total(m) - old
}
// Status classifies a node.
type Status uint8
const (
Unchanged Status = iota
Added
Removed
Grew
Shrank
)
func (s Status) String() string {
return [...]string{"unchanged", "new", "removed", "increased", "decreased"}[s]
}
// StatusOf classifies a New node.
func (d *Diff) StatusOf(id inventory.NodeID, m inventory.SizeMode) Status {
if d.OldOf[id] == inventory.NoNode {
return Added
}
switch delta := d.Delta(id, m); {
case delta > 0:
return Grew
case delta < 0:
return Shrank
}
return Unchanged
}
// Change is one row in a change report.
type Change struct {
Path string
Status Status
Delta int64
Old int64
New int64
Node inventory.NodeID // in New (NoNode for removed)
}
// Changes returns the k most significant changes. A directory is listed
// only if its change is not almost entirely explained by a single child, so
// the report points at /var/lib/postgresql rather than /, /var and /var/lib.
func (d *Diff) Changes(k int, m inventory.SizeMode) []Change {
var out []Change
for i := range d.New.Len() {
id := inventory.NodeID(i)
delta := d.Delta(id, m)
if delta == 0 {
continue
}
n := d.New.Node(id)
if d.OldOf[id] != inventory.NoNode && n.IsDir() && explainedByChild(d, id, delta, m) {
continue
}
if p := n.Parent; p != inventory.NoNode && d.OldOf[p] == inventory.NoNode {
continue // inside a wholly new directory; the directory is listed
}
old, _ := d.OldSize(id, m)
out = append(out, Change{Path: d.New.Path(id), Status: d.StatusOf(id, m), Delta: delta, Old: old, New: n.Total(m), Node: id})
}
for _, oid := range d.Removed {
o := d.Old.Node(oid)
out = append(out, Change{Path: d.Old.Path(oid), Status: Removed, Delta: -o.Total(m), Old: o.Total(m), Node: inventory.NoNode})
}
sort.Slice(out, func(i, j int) bool {
a, b := abs(out[i].Delta), abs(out[j].Delta)
if a != b {
return a > b
}
return out[i].Path < out[j].Path
})
if k > 0 && len(out) > k {
out = out[:k]
}
return out
}
func explainedByChild(d *Diff, id inventory.NodeID, delta int64, m inventory.SizeMode) bool {
explained := false
d.New.Children(id, func(c inventory.NodeID, _ *inventory.Node) {
if cd := d.Delta(c, m); (cd > 0) == (delta > 0) && abs(cd)*10 >= abs(delta)*8 {
explained = true
}
})
if r, ok := d.removedUnder[id]; ok && !explained {
v := r[0]
if m == inventory.SizeLogical {
v = r[1]
}
if delta < 0 && v*10 >= abs(delta)*8 {
explained = true
}
}
return explained
}
func abs(v int64) int64 {
if v < 0 {
return -v
}
return v
}
internal/snapshot/snapshot.go
+433
View File
@@ -0,0 +1,433 @@
// Package snapshot implements the versioned, checksummed binary inventory
// format and snapshot comparison.
//
// Layout (all integers little-endian / varint):
//
// magic 8 bytes brand.SnapshotMagic
// version uint16 currently 1
// flags uint16 bit 0: body is gzip-compressed (always set by Save)
// body gzip stream of:
// meta uvarint length + JSON (root, timestamps, counters, options)
// exts uvarint count, then count × (uvarint length + bytes)
// nodes uvarint count, then per node:
// uvarint parentDelta (id-parent; 0 only for the root)
// byte kind, byte category, uvarint flags
// uvarint nameLen + name
// varint size, varint alloc, varint mtime
// uvarint mode, uid, gid, nlink, ext
// errors uvarint count, then per record: uvarint node, byte kind,
// uvarint len + name, uvarint len + message
// sha256 32 bytes over the uncompressed body
//
// Nodes are written in ID order; a parent always precedes its children, so a
// reader can rebuild the tree in one pass and malformed input cannot create
// cycles. Aggregates are recomputed on load, never trusted.
package snapshot
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/andydixon/mapsize/internal/brand"
"github.com/andydixon/mapsize/internal/inventory"
)
// Version is the current format version.
const Version = 1
// Limits applied when reading untrusted snapshots.
const (
MaxNodes = 1<<31 - 1
MaxNameLen = 1 << 16
MaxMetaLen = 16 << 20
MaxExtLen = 64
MaxMsgLen = 4096
maxExts = 1<<16 - 1
flagGzipped = 1
)
type meta struct {
Root string `json:"root"`
Created time.Time `json:"created"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
Complete bool `json:"complete"`
Cancelled bool `json:"cancelled"`
Files int64 `json:"files"`
Dirs int64 `json:"dirs"`
Symlinks int64 `json:"symlinks"`
Others int64 `json:"others"`
ErrCounts []int64 `json:"err_counts"`
Excluded int64 `json:"excluded"`
SkippedMounts int64 `json:"skipped_mounts"`
VirtualSkipped int64 `json:"virtual_skipped"`
LoopsSkipped int64 `json:"loops_skipped"`
BrokenLinks int64 `json:"broken_links"`
HardlinkDups int64 `json:"hardlink_dups"`
Unscanned int64 `json:"unscanned"`
Excludes []string `json:"excludes"`
OneFileSystem bool `json:"one_file_system"`
Follow string `json:"follow"`
Workers int `json:"workers"`
Host string `json:"host"`
Generator string `json:"generator"`
}
// IsSnapshot reports whether path starts with the snapshot magic.
func IsSnapshot(path string) bool {
f, err := os.Open(path)
if err != nil {
return false
}
defer f.Close()
b := make([]byte, len(brand.SnapshotMagic))
_, err = io.ReadFull(f, b)
return err == nil && string(b) == brand.SnapshotMagic
}
type writer struct {
w io.Writer
buf [binary.MaxVarintLen64]byte
err error
}
func (w *writer) bytes(b []byte) {
if w.err == nil {
_, w.err = w.w.Write(b)
}
}
func (w *writer) uvarint(v uint64) { w.bytes(w.buf[:binary.PutUvarint(w.buf[:], v)]) }
func (w *writer) varint(v int64) { w.bytes(w.buf[:binary.PutVarint(w.buf[:], v)]) }
func (w *writer) str(s string) { w.uvarint(uint64(len(s))); w.bytes([]byte(s)) }
// Save writes t to w. The caller must hold t's read lock.
func Save(out io.Writer, t *inventory.Tree) error {
bw := bufio.NewWriterSize(out, 1<<20)
hdr := make([]byte, 0, 12)
hdr = append(hdr, brand.SnapshotMagic...)
hdr = binary.LittleEndian.AppendUint16(hdr, Version)
hdr = binary.LittleEndian.AppendUint16(hdr, flagGzipped)
bw.Write(hdr)
gz, _ := gzip.NewWriterLevel(bw, gzip.BestSpeed)
h := sha256.New()
w := &writer{w: io.MultiWriter(gz, h)}
st := &t.Stats
host, _ := os.Hostname()
m := meta{Root: st.Root, Created: time.Now().UTC(), Start: st.Start, End: st.End, Complete: st.Complete,
Cancelled: st.Cancelled, Files: st.Files, Dirs: st.Dirs, Symlinks: st.Symlinks, Others: st.Others,
ErrCounts: st.ErrCounts[:], Excluded: st.Excluded, SkippedMounts: st.SkippedMounts,
VirtualSkipped: st.VirtualSkipped, LoopsSkipped: st.LoopsSkipped, BrokenLinks: st.BrokenLinks,
HardlinkDups: st.HardlinkDups, Unscanned: st.Unscanned, Excludes: st.Excludes,
OneFileSystem: st.OneFileSystem, Follow: st.Follow, Workers: st.Workers, Host: host,
Generator: brand.Name + " " + brand.Version}
mb, err := json.Marshal(m)
if err != nil {
return err
}
w.uvarint(uint64(len(mb)))
w.bytes(mb)
w.uvarint(uint64(t.ExtCount()))
for i := range t.ExtCount() {
w.str(t.ExtByIndex(i))
}
w.uvarint(uint64(t.Len()))
for i := range t.Len() {
id := inventory.NodeID(i)
n := t.Node(id)
if i == 0 {
w.uvarint(0)
} else {
w.uvarint(uint64(id - n.Parent))
}
w.bytes([]byte{byte(n.Kind), byte(n.Cat)})
w.uvarint(uint64(n.Flags))
w.str(n.Name)
w.varint(n.Size)
w.varint(n.Alloc)
w.varint(n.MTime)
w.uvarint(uint64(n.Mode))
w.uvarint(uint64(n.UID))
w.uvarint(uint64(n.GID))
w.uvarint(uint64(n.Nlink))
w.uvarint(uint64(n.Ext))
}
w.uvarint(uint64(len(st.Errors)))
for _, e := range st.Errors {
w.uvarint(uint64(e.Node))
w.bytes([]byte{byte(e.Kind)})
w.str(e.Name)
w.str(e.Msg)
}
if w.err != nil {
return w.err
}
if err := gz.Close(); err != nil {
return err
}
bw.Write(h.Sum(nil))
return bw.Flush()
}
// SaveFile writes a snapshot atomically (temp file + rename).
func SaveFile(path string, t *inventory.Tree) error {
tmp, err := os.CreateTemp(dirOf(path), ".mapsize-*.tmp")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
if err := Save(tmp, t); err != nil {
tmp.Close()
return err
}
if err := tmp.Close(); err != nil {
return err
}
return os.Rename(tmp.Name(), path)
}
func dirOf(p string) string {
i := strings.LastIndexAny(p, `/\`)
if i < 0 {
return "."
}
if i == 0 {
return p[:1]
}
return p[:i]
}
type reader struct {
r *bufio.Reader
err error
}
func (r *reader) fail(format string, a ...any) {
if r.err == nil {
r.err = fmt.Errorf("snapshot: "+format, a...)
}
}
func (r *reader) byte1() byte {
if r.err != nil {
return 0
}
b, err := r.r.ReadByte()
if err != nil {
r.fail("truncated: %v", err)
}
return b
}
func (r *reader) uvarint(max uint64, what string) uint64 {
if r.err != nil {
return 0
}
v, err := binary.ReadUvarint(r.r)
if err != nil {
r.fail("bad %s: %v", what, err)
return 0
}
if v > max {
r.fail("%s %d exceeds limit %d", what, v, max)
return 0
}
return v
}
func (r *reader) varint(what string) int64 {
if r.err != nil {
return 0
}
v, err := binary.ReadVarint(r.r)
if err != nil {
r.fail("bad %s: %v", what, err)
}
return v
}
func (r *reader) bytesN(n uint64) []byte {
if r.err != nil {
return nil
}
b := make([]byte, n)
if _, err := io.ReadFull(r.r, b); err != nil {
r.fail("truncated: %v", err)
return nil
}
return b
}
func (r *reader) str(max uint64, what string) string {
return string(r.bytesN(r.uvarint(max, what+" length")))
}
// Load reads and validates a snapshot.
func Load(in io.Reader) (*inventory.Tree, error) {
br := bufio.NewReaderSize(in, 1<<20)
hdr := make([]byte, len(brand.SnapshotMagic)+4)
if _, err := io.ReadFull(br, hdr); err != nil {
return nil, errors.New("snapshot: file too short")
}
if string(hdr[:8]) != brand.SnapshotMagic {
return nil, errors.New("snapshot: not a " + brand.Name + " snapshot")
}
if v := binary.LittleEndian.Uint16(hdr[8:]); v != Version {
return nil, fmt.Errorf("snapshot: unsupported version %d (this build reads %d)", v, Version)
}
if binary.LittleEndian.Uint16(hdr[10:])&flagGzipped == 0 {
return nil, errors.New("snapshot: uncompressed bodies are not supported")
}
gz, err := gzip.NewReader(br)
if err != nil {
return nil, fmt.Errorf("snapshot: %w", err)
}
gz.Multistream(false)
h := sha256.New()
r := &reader{r: bufio.NewReaderSize(io.TeeReader(gz, h), 1<<16)}
var m meta
if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil {
if err := json.Unmarshal(mb, &m); err != nil {
return nil, fmt.Errorf("snapshot: metadata: %w", err)
}
}
if r.err != nil {
return nil, r.err
}
t := inventory.New(m.Root, inventory.KindDir)
nExt := r.uvarint(maxExts, "extension count")
exts := make([]string, 0, min(nExt, 1024))
for range nExt {
exts = append(exts, r.str(MaxExtLen, "extension"))
}
if r.err != nil {
return nil, r.err
}
extIndex := t.ExtIndexer()
nNodes := r.uvarint(MaxNodes, "node count")
if nNodes == 0 {
return nil, errors.New("snapshot: no root node")
}
for i := uint64(0); i < nNodes && r.err == nil; i++ {
pd := r.uvarint(MaxNodes, "parent")
kind := inventory.Kind(r.byte1())
cat := inventory.Category(r.byte1())
flags := inventory.Flags(r.uvarint(1<<16-1, "flags"))
name := r.str(MaxNameLen, "name")
size, alloc, mtime := r.varint("size"), r.varint("alloc"), r.varint("mtime")
mode := r.uvarint(1<<32-1, "mode")
uid := r.uvarint(1<<32-1, "uid")
gid := r.uvarint(1<<32-1, "gid")
nlink := r.uvarint(1<<32-1, "nlink")
ext := r.uvarint(nExt, "ext")
if r.err != nil {
break
}
if kind > inventory.KindOther || cat >= inventory.NumCategories {
r.fail("node %d: invalid kind/category", i)
break
}
if size < 0 || alloc < 0 {
r.fail("node %d: negative size", i)
break
}
var id inventory.NodeID
if i == 0 {
if pd != 0 || kind != inventory.KindDir && nNodes > 1 {
r.fail("invalid root")
break
}
id = 0
rn := t.Node(0)
rn.Kind = kind
} else {
if pd == 0 || pd > i {
r.fail("node %d: invalid parent reference", i)
break
}
parent := inventory.NodeID(i - pd)
if t.Node(parent).Kind != inventory.KindDir {
r.fail("node %d: parent is not a directory", i)
break
}
if !validName(name) {
r.fail("node %d: invalid name %q", i, name)
break
}
id = t.Add(parent, name, kind)
}
n := t.Node(id)
n.Cat, n.Flags = cat, flags
n.Size, n.Alloc, n.MTime = size, alloc, mtime
n.Mode, n.UID, n.GID, n.Nlink = uint32(mode), uint32(uid), uint32(gid), uint32(nlink)
if kind != inventory.KindDir && ext < uint64(len(exts)) {
n.Ext = extIndex(exts[ext])
}
}
nErr := r.uvarint(inventory.MaxErrorRecords, "error count")
for range nErr {
node := r.uvarint(nNodes-1, "error node")
kind := inventory.ErrKind(r.byte1())
name := r.str(MaxNameLen, "error name")
msg := r.str(MaxMsgLen, "error message")
if r.err != nil {
break
}
if kind >= inventory.NumErrKinds {
kind = inventory.ErrOther
}
t.Stats.Errors = append(t.Stats.Errors, inventory.ErrorRecord{Node: inventory.NodeID(node), Kind: kind, Name: name, Msg: msg})
}
if r.err != nil {
return nil, r.err
}
// The body must end here, then the checksum follows the gzip stream.
if _, err := r.r.ReadByte(); err != io.EOF {
return nil, errors.New("snapshot: trailing data in body")
}
sum := make([]byte, sha256.Size)
if _, err := io.ReadFull(br, sum); err != nil {
return nil, errors.New("snapshot: missing checksum")
}
if !bytes.Equal(sum, h.Sum(nil)) {
return nil, errors.New("snapshot: checksum mismatch (file corrupt)")
}
t.Recompute()
st := &t.Stats
st.Start, st.End, st.Complete, st.Cancelled = m.Start, m.End, true, m.Cancelled
st.Files, st.Dirs, st.Symlinks, st.Others = m.Files, m.Dirs, m.Symlinks, m.Others
copy(st.ErrCounts[:], m.ErrCounts)
st.Excluded, st.SkippedMounts, st.VirtualSkipped = m.Excluded, m.SkippedMounts, m.VirtualSkipped
st.LoopsSkipped, st.BrokenLinks, st.HardlinkDups, st.Unscanned = m.LoopsSkipped, m.BrokenLinks, m.HardlinkDups, m.Unscanned
st.Excludes, st.OneFileSystem, st.Follow, st.Workers = m.Excludes, m.OneFileSystem, m.Follow, m.Workers
st.FromSnapshot = fmt.Sprintf("%s (%s, %s)", m.Created.Local().Format("2006-01-02 15:04"), m.Host, m.Generator)
return t, nil
}
func validName(s string) bool {
return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\x00")
}
// LoadFile opens and loads a snapshot file.
func LoadFile(path string) (*inventory.Tree, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
defer f.Close()
return Load(f)
}
internal/snapshot/snapshot_test.go
+123
View File
@@ -0,0 +1,123 @@
package snapshot
import (
"bytes"
"testing"
"github.com/andydixon/mapsize/internal/inventory"
)
func sample() *inventory.Tree {
t := inventory.New("/srv", inventory.KindDir)
a := t.Add(0, "a", inventory.KindDir)
b := t.Add(0, "b", inventory.KindDir)
f1 := t.Add(a, "x.iso", inventory.KindFile)
t.Node(f1).Size, t.Node(f1).Alloc = 1000, 1024
f2 := t.Add(b, "y.log", inventory.KindFile)
t.Node(f2).Size, t.Node(f2).Alloc = 50, 4096
t.Node(f2).Flags = inventory.FlagSparse
t.Recompute()
t.Stats.Files, t.Stats.Dirs = 2, 3
t.Stats.AddError(inventory.ErrorRecord{Node: b, Kind: inventory.ErrPermission, Msg: "denied"})
return t
}
func TestRoundTrip(t *testing.T) {
src := sample()
var buf bytes.Buffer
if err := Save(&buf, src); err != nil {
t.Fatal(err)
}
got, err := Load(bytes.NewReader(buf.Bytes()))
if err != nil {
t.Fatal(err)
}
if got.Len() != src.Len() {
t.Fatalf("len %d want %d", got.Len(), src.Len())
}
for i := range src.Len() {
a, b := src.Node(inventory.NodeID(i)), got.Node(inventory.NodeID(i))
if a.Name != b.Name || a.TotSize != b.TotSize || a.TotAlloc != b.TotAlloc || a.Flags != b.Flags ||
src.Path(inventory.NodeID(i)) != got.Path(inventory.NodeID(i)) || src.ExtName(a) != got.ExtName(b) {
t.Fatalf("node %d differs: %+v vs %+v", i, a, b)
}
}
if got.Stats.ErrCounts[inventory.ErrPermission] != 1 || len(got.Stats.Errors) != 1 {
t.Fatalf("errors not restored: %+v", got.Stats)
}
}
func TestCorruptionDetected(t *testing.T) {
var buf bytes.Buffer
Save(&buf, sample())
data := buf.Bytes()
// Flip a byte in the checksum trailer.
bad := append([]byte(nil), data...)
bad[len(bad)-1] ^= 0xff
if _, err := Load(bytes.NewReader(bad)); err == nil {
t.Fatal("checksum corruption not detected")
}
// Every truncation must fail cleanly.
for i := 0; i < len(data)-1; i += 7 {
if _, err := Load(bytes.NewReader(data[:i])); err == nil {
t.Fatalf("truncation at %d accepted", i)
}
}
}
func FuzzLoad(f *testing.F) {
var buf bytes.Buffer
Save(&buf, sample())
f.Add(buf.Bytes())
f.Add([]byte("MAPSIZE\x00\x01\x00\x01\x00"))
f.Fuzz(func(t *testing.T, data []byte) {
tr, err := Load(bytes.NewReader(data))
if err != nil {
return
}
// A successfully loaded tree must be internally consistent.
for i := 1; i < tr.Len(); i++ {
if p := tr.Node(inventory.NodeID(i)).Parent; p >= inventory.NodeID(i) {
t.Fatalf("parent %d >= child %d", p, i)
}
}
_ = tr.Path(inventory.NodeID(tr.Len() - 1))
})
}
func TestCompare(t *testing.T) {
old := sample()
nw := inventory.New("/srv", inventory.KindDir)
a := nw.Add(0, "a", inventory.KindDir)
f1 := nw.Add(a, "x.iso", inventory.KindFile)
nw.Node(f1).Size, nw.Node(f1).Alloc = 5000, 5120
c := nw.Add(0, "c", inventory.KindDir)
f3 := nw.Add(c, "z", inventory.KindFile)
nw.Node(f3).Size, nw.Node(f3).Alloc = 10, 4096
nw.Recompute()
d := Compare(old, nw)
if d.StatusOf(f1, inventory.SizeAllocated) != Grew || d.Delta(f1, inventory.SizeAllocated) != 4096 {
t.Fatalf("x.iso: %v %d", d.StatusOf(f1, 0), d.Delta(f1, 0))
}
if d.StatusOf(c, 0) != Added {
t.Fatal("c should be new")
}
if len(d.Removed) != 1 || old.Node(d.Removed[0]).Name != "b" {
t.Fatalf("removed %v", d.Removed)
}
ch := d.Changes(10, inventory.SizeAllocated)
paths := map[string]Status{}
for _, c := range ch {
paths[c.Path] = c.Status
}
if paths["/srv/a/x.iso"] != Grew || paths["/srv/b"] != Removed || paths["/srv/c"] != Added {
t.Fatalf("changes %+v", ch)
}
if _, ok := paths["/srv/a"]; ok {
t.Fatal("/srv/a should be explained by its child")
}
if _, ok := paths["/srv/c/z"]; ok {
t.Fatal("contents of new dirs should not be listed separately")
}
}