From 900931ae268a163244dbd622ce135a6470d4029d Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Fri, 2 Oct 2026 08:02:32 +0100 Subject: [PATCH] Add CLI reports, snapshot format and duplicate detection --- cmd/mapsize/main.go | 199 ++++++++++++ internal/app/app.go | 169 +++++++++++ internal/app/modes.go | 68 +++++ internal/duplicate/duplicate.go | 285 ++++++++++++++++++ internal/duplicate/duplicate_test.go | 61 ++++ internal/export/export.go | 219 ++++++++++++++ internal/snapshot/compare.go | 177 +++++++++++ internal/snapshot/snapshot.go | 433 +++++++++++++++++++++++++++ internal/snapshot/snapshot_test.go | 123 ++++++++ 9 files changed, 1734 insertions(+) create mode 100644 cmd/mapsize/main.go create mode 100644 internal/app/app.go create mode 100644 internal/app/modes.go create mode 100644 internal/duplicate/duplicate.go create mode 100644 internal/duplicate/duplicate_test.go create mode 100644 internal/export/export.go create mode 100644 internal/snapshot/compare.go create mode 100644 internal/snapshot/snapshot.go create mode 100644 internal/snapshot/snapshot_test.go diff --git a/cmd/mapsize/main.go b/cmd/mapsize/main.go new file mode 100644 index 0000000..c931aac --- /dev/null +++ b/cmd/mapsize/main.go @@ -0,0 +1,199 @@ +// Command mapsize is an interactive terminal disk usage analyser. +package main + +import ( + "flag" + "fmt" + "io" + "log/slog" + "os" + "runtime" + "runtime/pprof" + "strings" + + "github.com/andydixon/mapsize/internal/app" + "github.com/andydixon/mapsize/internal/brand" + "github.com/andydixon/mapsize/internal/config" + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/scan" + "github.com/andydixon/mapsize/internal/snapshot" + "github.com/andydixon/mapsize/internal/textutil" +) + +type multiFlag []string + +func (m *multiFlag) String() string { return strings.Join(*m, ",") } +func (m *multiFlag) Set(s string) error { *m = append(*m, s); return nil } + +func main() { + os.Exit(run(os.Args[1:])) +} + +func run(args []string) int { + settings, cfgErr := config.Load() + fs := flag.NewFlagSet(brand.Name, flag.ContinueOnError) + fs.Usage = func() { usage(fs) } + var ( + excludes multiFlag + workers = fs.String("workers", settings.Workers, "concurrency: conservative, balanced, aggressive or a number") + follow = fs.String("follow-symlinks", settings.FollowSymlinks, "none, same-filesystem or all") + oneFS = fs.Bool("one-file-system", settings.OneFileSystem, "do not cross filesystem boundaries") + oneFSx = fs.Bool("x", false, "short for --one-file-system") + readOnly = fs.Bool("read-only", false, "disable all modifying actions") + noUI = fs.Bool("no-ui", false, "scan without the interactive interface and print a summary") + jsonOut = fs.Bool("json", false, "print the inventory as JSON") + csvOut = fs.Bool("csv", false, "print the inventory as CSV") + depth = fs.Int("depth", -1, "maximum depth for --json/--csv (-1 = unlimited)") + top = fs.Int("top", 0, "print the N largest entries of the root") + largest = fs.Int("largest-files", 0, "print the N largest files") + dups = fs.Bool("duplicates", false, "find duplicate files (non-interactive)") + save = fs.String("save", "", "save a snapshot to `FILE` after scanning") + snap = fs.String("snapshot", "", "alias for --save") + load = fs.String("load", "", "open a saved snapshot `FILE` instead of scanning") + scanPath = fs.String("scan", "", "path to scan (alternative to a positional argument)") + compare = fs.Bool("compare", false, "compare two snapshots: --compare OLD NEW") + si = fs.Bool("si", settings.Units == "si", "use SI units (kB, MB) instead of IEC (KiB, MiB)") + logical = fs.Bool("apparent", settings.SizeMode == "logical", "size by logical (apparent) size instead of disk usage") + theme = fs.String("theme", settings.Theme, "colour theme: default, dark, high-contrast, mono") + noMouse = fs.Bool("no-mouse", !settings.Mouse, "disable mouse support") + logFile = fs.String("log", "", "write a debug log to `FILE`") + logLevel = fs.String("log-level", "info", "log level: error, warn, info, debug, trace") + cpuProfile = fs.String("cpuprofile", "", "write a CPU profile to `FILE`") + memProfile = fs.String("memprofile", "", "write a heap profile to `FILE` on exit") + version = fs.Bool("version", false, "print version and exit") + ) + fs.Var(&excludes, "exclude", "exclude `PATTERN` (glob on name, or path prefix if it contains a separator); repeatable") + + var positional []string + for { + if err := fs.Parse(args); err != nil { + if err == flag.ErrHelp { + return 0 + } + return 2 + } + args = fs.Args() + if len(args) == 0 { + break + } + positional = append(positional, args[0]) + args = args[1:] + } + if *version { + fmt.Printf("%s %s (%s/%s)\n", brand.Name, brand.Version, runtime.GOOS, runtime.GOARCH) + return 0 + } + if cfgErr != nil { + fmt.Fprintf(os.Stderr, "%s: warning: config: %v (using defaults)\n", brand.Name, cfgErr) + } + textutil.SI = *si + + if *logFile != "" { + closeLog, err := setupLog(*logFile, *logLevel) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err) + return 2 + } + defer closeLog() + } else { + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + } + if *cpuProfile != "" { + f, err := os.Create(*cpuProfile) + if err == nil { + pprof.StartCPUProfile(f) + defer pprof.StopCPUProfile() + } + } + if *memProfile != "" { + defer func() { + if f, err := os.Create(*memProfile); err == nil { + runtime.GC() + pprof.WriteHeapProfile(f) + f.Close() + } + }() + } + + cfg := &app.Config{ + NoUI: *noUI, JSON: *jsonOut, CSV: *csvOut, Depth: *depth, Top: *top, LargestFiles: *largest, + Duplicates: *dups, ReadOnly: *readOnly, Theme: *theme, Mouse: !*noMouse, Load: *load, + Settings: settings, + } + if *logical { + cfg.SizeMode = inventory.SizeLogical + } + cfg.Save = *save + if cfg.Save == "" { + cfg.Save = *snap + } + if *scanPath != "" { + positional = append([]string{*scanPath}, positional...) + } + if *compare { + if len(positional) != 2 { + fmt.Fprintf(os.Stderr, "%s: --compare needs exactly two snapshot files\n", brand.Name) + return 2 + } + cfg.Compare = [2]string{positional[0], positional[1]} + } else { + if len(positional) > 1 { + fmt.Fprintf(os.Stderr, "%s: only one path may be scanned at a time\n", brand.Name) + return 2 + } + if len(positional) == 1 && cfg.Load == "" && snapshot.IsSnapshot(positional[0]) { + cfg.Load = positional[0] + } else { + cfg.Paths = positional + } + } + + w, err := scan.Workers(*workers) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err) + return 2 + } + fm, err := scan.ParseFollow(*follow) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, err) + return 2 + } + cfg.ScanOpts = scan.Options{Workers: w, Follow: fm, OneFileSystem: *oneFS || *oneFSx, + Excludes: append(append([]string(nil), settings.Excludes...), excludes...)} + + if err := app.Run(cfg); err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", brand.Name, textutil.Sanitize(err.Error())) + return 1 + } + return 0 +} + +func setupLog(path, level string) (func(), error) { + lv := map[string]slog.Level{"error": slog.LevelError, "warn": slog.LevelWarn, "info": slog.LevelInfo, + "debug": slog.LevelDebug, "trace": slog.LevelDebug - 4} + l, ok := lv[level] + if !ok { + return nil, fmt.Errorf("invalid --log-level %q", level) + } + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0o600) + if err != nil { + return nil, err + } + slog.SetDefault(slog.New(slog.NewTextHandler(f, &slog.HandlerOptions{Level: l}))) + return func() { f.Close() }, nil +} + +func usage(fs *flag.FlagSet) { + w := fs.Output() + fmt.Fprintf(w, `%[1]s — interactive terminal disk usage analyser + +Usage: + %[1]s [flags] [PATH] scan PATH (default .) interactively + %[1]s [flags] SNAPSHOT%[2]s open a saved snapshot + %[1]s --compare OLD NEW compare two snapshots + %[1]s PATH --top 50 | --json | --csv | --no-ui --save FILE + +Flags: +`, brand.Name, brand.SnapshotExt) + fs.PrintDefaults() +} diff --git a/internal/app/app.go b/internal/app/app.go new file mode 100644 index 0000000..68522cc --- /dev/null +++ b/internal/app/app.go @@ -0,0 +1,169 @@ +// Package app wires the command-line modes together: interactive TUI, +// non-interactive reports, snapshots and comparisons. TUI and CLI share the +// same scanner and inventory packages. +package app + +import ( + "context" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + "time" + + "github.com/andydixon/mapsize/internal/config" + "github.com/andydixon/mapsize/internal/export" + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/scan" + "github.com/andydixon/mapsize/internal/textutil" +) + +// Config is the fully-resolved command line. +type Config struct { + Paths []string + ScanOpts scan.Options + NoUI bool + JSON, CSV bool + Depth int + Top int + LargestFiles int + Save string + Load string + Compare [2]string + Duplicates bool + ReadOnly bool + Theme string + SizeMode inventory.SizeMode + Mouse bool + Settings config.Settings + Stdout io.Writer + Stderr io.Writer +} + +// Report reports whether any non-interactive output was requested. +func (c *Config) Report() bool { + return c.NoUI || c.JSON || c.CSV || c.Top > 0 || c.LargestFiles > 0 || c.Duplicates +} + +// Run executes the configured mode. +func Run(cfg *Config) error { + if cfg.Stdout == nil { + cfg.Stdout = os.Stdout + } + if cfg.Stderr == nil { + cfg.Stderr = os.Stderr + } + if cfg.Compare[0] != "" { + return runCompare(cfg) + } + if !cfg.Report() { + return runTUI(cfg) + } + return runReport(cfg) +} + +// startSource begins a scan (or loads a snapshot). The scanner is nil when +// the tree came from a snapshot. +func startSource(ctx context.Context, cfg *Config) (*scan.Scanner, *inventory.Tree, error) { + if cfg.Load != "" { + t, err := loadSnapshot(cfg.Load) + return nil, t, err + } + root := "." + if len(cfg.Paths) > 0 { + root = cfg.Paths[0] + } + slog.Info("scan start", "workers", cfg.ScanOpts.Workers, "follow", cfg.ScanOpts.Follow.String()) + s, err := scan.Start(ctx, root, cfg.ScanOpts) + if err != nil { + return nil, nil, err + } + return s, s.Tree, nil +} + +func runReport(cfg *Config) error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + s, t, err := startSource(ctx, cfg) + if err != nil { + return err + } + if s != nil { + waitWithProgress(s, cfg.Stderr) + } + t.RLock() + defer t.RUnlock() + if s != nil && cfg.Save != "" { + if err := saveSnapshot(t, cfg.Save); err != nil { + return err + } + fmt.Fprintf(cfg.Stderr, "Saved snapshot %s\n", cfg.Save) + } + out := cfg.Stdout + switch { + case cfg.JSON: + return export.JSON(out, t, t.Root(), cfg.Depth) + case cfg.CSV: + return export.CSV(out, t, t.Root(), cfg.Depth) + } + if cfg.Duplicates { + if err := runDuplicates(ctx, cfg, t); err != nil { + return err + } + } + if cfg.Top > 0 { + ids := t.SortedChildren(t.Root(), cfg.SizeMode) + if len(ids) > cfg.Top { + ids = ids[:cfg.Top] + } + export.Table(out, t, ids, cfg.SizeMode) + fmt.Fprintln(out) + } + if cfg.LargestFiles > 0 { + ids := t.Top(t.Root(), cfg.LargestFiles, + func(n *inventory.Node) bool { + return n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0 + }, + func(n *inventory.Node) int64 { return n.Own(cfg.SizeMode) }) + export.Table(out, t, ids, cfg.SizeMode) + fmt.Fprintln(out) + } + export.Summary(out, t) + return nil +} + +// waitWithProgress blocks until the scan finishes, printing a progress line +// to w if it is a terminal. +func waitWithProgress(s *scan.Scanner, w io.Writer) { + tty := isTerminal(w) + tick := time.NewTicker(250 * time.Millisecond) + defer tick.Stop() + start := time.Now() + for { + select { + case <-s.Done(): + if tty { + fmt.Fprint(w, "\r\033[K") + } + return + case <-tick.C: + if tty { + p := s.Progress() + el := time.Since(start) + fmt.Fprintf(w, "\r\033[Kscanning… %s entries %s/s queue %d %s", + textutil.Count(p.Entries), textutil.Count(int64(float64(p.Entries)/el.Seconds())), + p.Pending, textutil.Duration(el)) + } + } + } +} + +func isTerminal(w io.Writer) bool { + f, ok := w.(*os.File) + if !ok { + return false + } + fi, err := f.Stat() + return err == nil && fi.Mode()&os.ModeCharDevice != 0 +} diff --git a/internal/app/modes.go b/internal/app/modes.go new file mode 100644 index 0000000..2a1fd06 --- /dev/null +++ b/internal/app/modes.go @@ -0,0 +1,68 @@ +package app + +import ( + "context" + "errors" + "fmt" + + "github.com/andydixon/mapsize/internal/duplicate" + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/snapshot" + "github.com/andydixon/mapsize/internal/textutil" +) + +func runTUI(*Config) error { + return errors.New("the interactive UI is not built yet; use --no-ui, --top, --json or --csv") +} + +func loadSnapshot(path string) (*inventory.Tree, error) { return snapshot.LoadFile(path) } + +func saveSnapshot(t *inventory.Tree, path string) error { return snapshot.SaveFile(path, t) } + +func runCompare(cfg *Config) error { + old, err := snapshot.LoadFile(cfg.Compare[0]) + if err != nil { + return fmt.Errorf("%s: %w", cfg.Compare[0], err) + } + nw, err := snapshot.LoadFile(cfg.Compare[1]) + if err != nil { + return fmt.Errorf("%s: %w", cfg.Compare[1], err) + } + d := snapshot.Compare(old, nw) + k := cfg.Top + if k <= 0 { + k = 30 + } + out := cfg.Stdout + ro, rn := old.Node(0), nw.Node(0) + fmt.Fprintf(out, "%s → %s\n", textutil.Sanitize(old.Stats.FromSnapshot), textutil.Sanitize(nw.Stats.FromSnapshot)) + fmt.Fprintf(out, "Total %s → %s (%s)\n\n", textutil.Size(ro.Total(cfg.SizeMode)), textutil.Size(rn.Total(cfg.SizeMode)), + textutil.SignedSize(rn.Total(cfg.SizeMode)-ro.Total(cfg.SizeMode))) + for _, c := range d.Changes(k, cfg.SizeMode) { + fmt.Fprintf(out, "%-10s %14s %s\n", c.Status, textutil.SignedSize(c.Delta), textutil.Sanitize(c.Path)) + } + return nil +} + +func runDuplicates(ctx context.Context, cfg *Config, t *inventory.Tree) error { + // The caller holds RLock; Find takes its own read lock, which is fine + // for a RWMutex only if no writer is waiting — the scan is finished. + var f duplicate.Finder + groups, err := f.Find(ctx, t, duplicate.Options{MinSize: 1}) + if err != nil { + return err + } + out := cfg.Stdout + var wasted int64 + for _, g := range groups { + wasted += g.Wasted() + fmt.Fprintf(out, "%s × %d (sha256 %x…)\n", textutil.Size(g.Size), len(g.Files), g.Hash[:6]) + for _, id := range g.Files { + fmt.Fprintf(out, " %s\n", textutil.Sanitize(t.Path(id))) + } + } + p := f.Progress() + fmt.Fprintf(out, "\n%d verified duplicate groups, %s reclaimable (%d candidates by size, %d unreadable/changed)\n\n", + len(groups), textutil.Size(wasted), p.Candidates, p.Skipped) + return nil +} diff --git a/internal/duplicate/duplicate.go b/internal/duplicate/duplicate.go new file mode 100644 index 0000000..bbf1e74 --- /dev/null +++ b/internal/duplicate/duplicate.go @@ -0,0 +1,285 @@ +// Package duplicate finds files with identical content in stages: +// +// 1. group by exact size (inventory only, no I/O) +// 2. hash three 16 KiB samples (start, middle, end) of each candidate +// 3. full SHA-256 of files whose samples still collide +// +// Only stage 3 establishes identity. Files are never declared duplicates by +// name or size alone. Reading files may update their access times. +package duplicate + +import ( + "context" + "crypto/sha256" + "io" + "os" + "sort" + "sync" + "sync/atomic" + + "github.com/andydixon/mapsize/internal/inventory" +) + +// Group is a set of files verified to have identical content. +type Group struct { + Size int64 + Hash [32]byte + Files []inventory.NodeID +} + +// Wasted returns the bytes reclaimable by keeping one copy. +func (g *Group) Wasted() int64 { return g.Size * int64(len(g.Files)-1) } + +// Progress reports activity. +type Progress struct { + Stage string + Candidates int64 // files sharing a size with another file + BytesToHash int64 + BytesHashed int64 + Skipped int64 // unreadable or changed during hashing + VerifiedGroups int64 + CandidateGroups int64 +} + +// Options tunes the search. +type Options struct { + MinSize int64 // ignore files smaller than this (default 1 byte) + Workers int // concurrent readers (default 4) +} + +type cand struct { + id inventory.NodeID + path string + size int64 +} + +const sampleSize = 16 << 10 + +// Finder runs a search and exposes live progress. +type Finder struct { + hashed, toHash, skipped atomic.Int64 + mu sync.Mutex + p Progress +} + +// Progress returns a copy of the current progress. +func (f *Finder) Progress() Progress { + f.mu.Lock() + p := f.p + f.mu.Unlock() + p.BytesHashed, p.BytesToHash, p.Skipped = f.hashed.Load(), f.toHash.Load(), f.skipped.Load() + return p +} + +func (f *Finder) stage(s string, fn func(*Progress)) { + f.mu.Lock() + f.p.Stage = s + if fn != nil { + fn(&f.p) + } + f.mu.Unlock() +} + +// Find searches t (read-locked internally only while collecting candidates). +func (f *Finder) Find(ctx context.Context, t *inventory.Tree, o Options) ([]Group, error) { + if o.MinSize < 1 { + o.MinSize = 1 + } + if o.Workers < 1 { + o.Workers = 4 + } + f.stage("grouping by size", nil) + bySize := map[int64][]cand{} + t.RLock() + t.Walk(t.Root(), func(id inventory.NodeID, n *inventory.Node) bool { + if n.Kind == inventory.KindFile && n.Flags&inventory.FlagHardlinkDup == 0 && n.Size >= o.MinSize { + bySize[n.Size] = append(bySize[n.Size], cand{id: id, size: n.Size}) + } + return true + }) + var groups [][]cand + var ncand int64 + for _, g := range bySize { + if len(g) > 1 { + for i := range g { + g[i].path = t.Path(g[i].id) + } + groups = append(groups, g) + ncand += int64(len(g)) + } + } + t.RUnlock() + f.stage("sampling", func(p *Progress) { p.Candidates = ncand; p.CandidateGroups = int64(len(groups)) }) + + // Stage 2: sample hashes. + var sampled [][]cand + for _, g := range groups { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if g[0].size <= 3*sampleSize { + sampled = append(sampled, g) // a sample would be the whole file + continue + } + sampled = append(sampled, f.split(ctx, g, o.Workers, sampleHash)...) + } + + // Stage 3: full hashes. + var total int64 + for _, g := range sampled { + total += g[0].size * int64(len(g)) + } + f.toHash.Store(total) + f.stage("hashing", nil) + var out []Group + for _, g := range sampled { + if ctx.Err() != nil { + return nil, ctx.Err() + } + for _, sub := range f.splitHash(ctx, g, o.Workers) { + grp := Group{Size: sub.size, Hash: sub.hash} + for _, c := range sub.files { + grp.Files = append(grp.Files, c.id) + } + out = append(out, grp) + } + } + sort.Slice(out, func(i, j int) bool { + if a, b := out[i].Wasted(), out[j].Wasted(); a != b { + return a > b + } + return out[i].Files[0] < out[j].Files[0] + }) + f.stage("done", func(p *Progress) { p.VerifiedGroups = int64(len(out)) }) + return out, ctx.Err() +} + +type hashed struct { + c cand + sum [32]byte + ok bool +} + +// hashAll hashes every candidate with a bounded worker pool. +func (f *Finder) hashAll(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) []hashed { + res := make([]hashed, len(g)) + var wg sync.WaitGroup + sem := make(chan struct{}, workers) + for i, c := range g { + if ctx.Err() != nil { + break + } + sem <- struct{}{} + wg.Add(1) + go func() { + defer wg.Done() + defer func() { <-sem }() + sum, ok := fn(f, c) + if !ok { + f.skipped.Add(1) + } + res[i] = hashed{c, sum, ok} + }() + } + wg.Wait() + return res +} + +func (f *Finder) split(ctx context.Context, g []cand, workers int, fn func(*Finder, cand) ([32]byte, bool)) [][]cand { + by := map[[32]byte][]cand{} + for _, h := range f.hashAll(ctx, g, workers, fn) { + if h.ok { + by[h.sum] = append(by[h.sum], h.c) + } + } + var out [][]cand + for _, v := range by { + if len(v) > 1 { + out = append(out, v) + } + } + return out +} + +type fullGroup struct { + size int64 + hash [32]byte + files []cand +} + +func (f *Finder) splitHash(ctx context.Context, g []cand, workers int) []fullGroup { + by := map[[32]byte][]cand{} + for _, h := range f.hashAll(ctx, g, workers, fullHash) { + if h.ok { + by[h.sum] = append(by[h.sum], h.c) + } + } + var out []fullGroup + for sum, v := range by { + if len(v) > 1 { + sort.Slice(v, func(i, j int) bool { return v[i].id < v[j].id }) + out = append(out, fullGroup{size: v[0].size, hash: sum, files: v}) + } + } + return out +} + +// open opens a candidate and checks it still has the inventoried size. +func open(c cand) (*os.File, bool) { + fh, err := os.Open(c.path) + if err != nil { + return nil, false + } + fi, err := fh.Stat() + if err != nil || !fi.Mode().IsRegular() || fi.Size() != c.size { + fh.Close() + return nil, false + } + return fh, true +} + +func sampleHash(_ *Finder, c cand) ([32]byte, bool) { + fh, ok := open(c) + if !ok { + return [32]byte{}, false + } + defer fh.Close() + h := sha256.New() + buf := make([]byte, sampleSize) + for _, off := range []int64{0, c.size/2 - sampleSize/2, c.size - sampleSize} { + if _, err := fh.ReadAt(buf, off); err != nil && err != io.EOF { + return [32]byte{}, false + } + h.Write(buf) + } + var s [32]byte + h.Sum(s[:0]) + return s, true +} + +func fullHash(f *Finder, c cand) ([32]byte, bool) { + fh, ok := open(c) + if !ok { + return [32]byte{}, false + } + defer fh.Close() + h := sha256.New() + n, err := io.Copy(h, &countingReader{fh, &f.hashed}) + if err != nil || n != c.size { + return [32]byte{}, false + } + var s [32]byte + h.Sum(s[:0]) + return s, true +} + +type countingReader struct { + r io.Reader + n *atomic.Int64 +} + +func (c *countingReader) Read(p []byte) (int, error) { + n, err := c.r.Read(p) + c.n.Add(int64(n)) + return n, err +} diff --git a/internal/duplicate/duplicate_test.go b/internal/duplicate/duplicate_test.go new file mode 100644 index 0000000..9f2dd3b --- /dev/null +++ b/internal/duplicate/duplicate_test.go @@ -0,0 +1,61 @@ +package duplicate + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/andydixon/mapsize/internal/scan" +) + +func TestFind(t *testing.T) { + root := t.TempDir() + big := make([]byte, 200_000) + for i := range big { + big[i] = byte(i * 7) + } + w := func(name string, b []byte) { + if err := os.WriteFile(filepath.Join(root, name), b, 0o644); err != nil { + t.Fatal(err) + } + } + w("a.bin", big) + w("b.bin", big) + // Same size, same samples, different middle-of-nowhere byte: must not match. + c := append([]byte(nil), big...) + c[20_000] ^= 1 + w("c.bin", c) + w("small1", []byte("hello")) + w("small2", []byte("hello")) + w("small3", []byte("world")) // same size, different content + + s, err := scan.Start(context.Background(), root, scan.Options{Workers: 2}) + if err != nil { + t.Fatal(err) + } + <-s.Done() + var f Finder + groups, err := f.Find(context.Background(), s.Tree, Options{}) + if err != nil { + t.Fatal(err) + } + if len(groups) != 2 { + t.Fatalf("groups=%d want 2", len(groups)) + } + names := func(g Group) (out []string) { + for _, id := range g.Files { + out = append(out, s.Tree.Node(id).Name) + } + return + } + if g := names(groups[0]); len(g) != 2 || groups[0].Size != 200_000 { + t.Fatalf("big group %v", g) + } + if g := names(groups[1]); len(g) != 2 { + t.Fatalf("small group %v", g) + } + if p := f.Progress(); p.Candidates != 6 || p.VerifiedGroups != 2 { + t.Fatalf("progress %+v", p) + } +} diff --git a/internal/export/export.go b/internal/export/export.go new file mode 100644 index 0000000..cd117be --- /dev/null +++ b/internal/export/export.go @@ -0,0 +1,219 @@ +// Package export writes inventories as JSON, CSV or plain-text reports. +// Callers must hold the tree's read lock (or own the tree exclusively). +package export + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "fmt" + "io" + "strconv" + "time" + + "github.com/andydixon/mapsize/internal/brand" + "github.com/andydixon/mapsize/internal/inventory" + "github.com/andydixon/mapsize/internal/textutil" +) + +type jsonNode struct { + Name string `json:"name"` + Type string `json:"type"` + Size int64 `json:"size"` + Allocated int64 `json:"allocated"` + Files uint32 `json:"files,omitempty"` + Dirs uint32 `json:"dirs,omitempty"` + Errors uint32 `json:"errors,omitempty"` + Modified string `json:"modified,omitempty"` + Flags []string `json:"flags,omitempty"` + Children []*jsonNode `json:"children,omitempty"` +} + +var flagNames = []struct { + f inventory.Flags + n string +}{ + {inventory.FlagHardlinkDup, "hardlink-duplicate"}, {inventory.FlagMountPoint, "mount-point"}, + {inventory.FlagVirtualFS, "virtual-fs-skipped"}, {inventory.FlagSkippedFS, "other-fs-skipped"}, + {inventory.FlagSparse, "sparse"}, {inventory.FlagError, "error"}, {inventory.FlagIncomplete, "incomplete"}, + {inventory.FlagLoop, "loop-skipped"}, {inventory.FlagFollowed, "followed-symlink"}, + {inventory.FlagAllocUnknown, "allocated-unknown"}, {inventory.FlagBrokenLink, "broken-symlink"}, + {inventory.FlagHardlinked, "hardlinked"}, +} + +// FlagList returns human-readable names for a node's flags. +func FlagList(f inventory.Flags) []string { + var out []string + for _, fn := range flagNames { + if f&fn.f != 0 { + out = append(out, fn.n) + } + } + return out +} + +// JSON writes the tree beneath root as nested JSON, down to maxDepth levels +// (maxDepth < 0 means unlimited). +func JSON(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error { + bw := bufio.NewWriter(w) + st := &t.Stats + head := map[string]any{ + "generator": brand.Name + " " + brand.Version, + "root": t.Path(root), + "complete": !st.Incomplete(), + "cancelled": st.Cancelled, + "started": st.Start.Format(time.RFC3339), + "elapsed_s": st.Elapsed().Seconds(), + "errors": errorCounts(st), + "excluded": st.Excluded, + "excludes": st.Excludes, + "hardlink_duplicates": st.HardlinkDups, + } + hb, err := json.Marshal(head) + if err != nil { + return err + } + bw.Write(hb[:len(hb)-1]) + bw.WriteString(`,"tree":`) + if err := writeNode(bw, t, root, maxDepth); err != nil { + return err + } + bw.WriteString("}\n") + return bw.Flush() +} + +func errorCounts(st *inventory.Stats) map[string]int64 { + m := map[string]int64{} + for k, c := range st.ErrCounts { + if c > 0 { + m[inventory.ErrKind(k).String()] = c + } + } + return m +} + +// writeNode streams one node; children are written recursively so memory +// stays proportional to depth, not tree size. +func writeNode(bw *bufio.Writer, t *inventory.Tree, id inventory.NodeID, depth int) error { + n := t.Node(id) + jn := jsonNode{Name: n.Name, Type: kindName(n.Kind), Size: n.TotSize, Allocated: n.TotAlloc, + Files: n.Files, Dirs: n.Dirs, Errors: n.Errors, Flags: FlagList(n.Flags)} + if n.MTime != 0 { + jn.Modified = time.Unix(0, n.MTime).UTC().Format(time.RFC3339) + } + b, err := json.Marshal(jn) + if err != nil { + return err + } + if n.FirstChild == inventory.NoNode || depth == 0 { + _, err = bw.Write(b) + return err + } + bw.Write(b[:len(b)-1]) + bw.WriteString(`,"children":[`) + first := true + for _, c := range t.SortedChildren(id, inventory.SizeAllocated) { + if !first { + bw.WriteByte(',') + } + first = false + if err := writeNode(bw, t, c, depth-1); err != nil { + return err + } + } + _, err = bw.WriteString("]}") + return err +} + +func kindName(k inventory.Kind) string { + return [...]string{"file", "dir", "symlink", "other"}[k&3] +} + +// CSV writes one row per node beneath root (depth-limited). Paths are +// sanitised so the output is safe to print to a terminal. +func CSV(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) error { + cw := csv.NewWriter(w) + cw.Write([]string{"path", "type", "size", "allocated", "files", "dirs", "errors", "modified", "flags"}) + var rec func(id inventory.NodeID, path string, depth int) error + rec = func(id inventory.NodeID, path string, depth int) error { + n := t.Node(id) + mod := "" + if n.MTime != 0 { + mod = time.Unix(0, n.MTime).UTC().Format(time.RFC3339) + } + flags := "" + for i, f := range FlagList(n.Flags) { + if i > 0 { + flags += "|" + } + flags += f + } + if err := cw.Write([]string{textutil.Sanitize(path), kindName(n.Kind), + strconv.FormatInt(n.TotSize, 10), strconv.FormatInt(n.TotAlloc, 10), + strconv.FormatUint(uint64(n.Files), 10), strconv.FormatUint(uint64(n.Dirs), 10), + strconv.FormatUint(uint64(n.Errors), 10), mod, flags}); err != nil { + return err + } + if depth == 0 { + return nil + } + for _, c := range t.SortedChildren(id, inventory.SizeAllocated) { + if err := rec(c, path+"/"+t.Node(c).Name, depth-1); err != nil { + return err + } + } + return nil + } + if err := rec(root, t.Path(root), maxDepth); err != nil { + return err + } + cw.Flush() + return cw.Error() +} + +// Table writes "size % files path" rows for the given nodes. +func Table(w io.Writer, t *inventory.Tree, ids []inventory.NodeID, m inventory.SizeMode) { + total := t.Node(t.Root()).Total(m) + fmt.Fprintf(w, "%12s %6s %12s %s\n", "SIZE", "%", "FILES", "PATH") + for _, id := range ids { + n := t.Node(id) + files := "" + if n.IsDir() { + files = textutil.Count(int64(n.Files)) + } + fmt.Fprintf(w, "%12s %6s %12s %s\n", textutil.Size(n.Total(m)), textutil.Percent(n.Total(m), total), + files, textutil.Sanitize(t.Path(id))) + } +} + +// Summary writes the scan summary, making incompleteness explicit. +func Summary(w io.Writer, t *inventory.Tree) { + st := &t.Stats + r := t.Node(t.Root()) + fmt.Fprintf(w, "Root %s\n", textutil.Sanitize(st.Root)) + fmt.Fprintf(w, "Files %s\n", textutil.Count(st.Files+st.Symlinks+st.Others)) + fmt.Fprintf(w, "Directories %s\n", textutil.Count(st.Dirs)) + fmt.Fprintf(w, "Logical size %s (%d bytes)\n", textutil.Size(r.TotSize), r.TotSize) + fmt.Fprintf(w, "Allocated size %s (%d bytes)\n", textutil.Size(r.TotAlloc), r.TotAlloc) + fmt.Fprintf(w, "Elapsed %s\n", st.Elapsed().Round(time.Millisecond)) + for k, c := range st.ErrCounts { + if c > 0 { + fmt.Fprintf(w, "%-15s %s\n", inventory.ErrKind(k).String(), textutil.Count(c)) + } + } + if st.Excluded > 0 { + fmt.Fprintf(w, "Excluded %s entries (%v)\n", textutil.Count(st.Excluded), st.Excludes) + } + if st.SkippedMounts > 0 { + fmt.Fprintf(w, "Other FS skipped %s\n", textutil.Count(st.SkippedMounts)) + } + if st.VirtualSkipped > 0 { + fmt.Fprintf(w, "Virtual FS skipped %s\n", textutil.Count(st.VirtualSkipped)) + } + if st.HardlinkDups > 0 { + fmt.Fprintf(w, "Hard-link dups %s (counted once)\n", textutil.Count(st.HardlinkDups)) + } + if st.Incomplete() { + fmt.Fprintln(w, "WARNING: totals are incomplete — some data could not be inspected.") + } +} diff --git a/internal/snapshot/compare.go b/internal/snapshot/compare.go new file mode 100644 index 0000000..c797ff4 --- /dev/null +++ b/internal/snapshot/compare.go @@ -0,0 +1,177 @@ +package snapshot + +import ( + "sort" + + "github.com/andydixon/mapsize/internal/inventory" +) + +// Diff relates two inventories by path. Both trees must not be mutated +// while a Diff is in use (snapshots and finished scans satisfy this). +type Diff struct { + Old, New *inventory.Tree + // OldOf maps a New NodeID to the matching Old NodeID, or NoNode if the + // entry is new. + OldOf []inventory.NodeID + // Removed lists the topmost Old nodes with no counterpart in New. + Removed []inventory.NodeID + // RemovedUnder sums, per New directory, the sizes of removed children + // (in both size modes) so growth colouring can account for them. + removedUnder map[inventory.NodeID][2]int64 +} + +// Compare matches nodes by name, directory by directory, from the roots. +func Compare(old, new *inventory.Tree) *Diff { + d := &Diff{Old: old, New: new, OldOf: make([]inventory.NodeID, new.Len()), + removedUnder: map[inventory.NodeID][2]int64{}} + for i := range d.OldOf { + d.OldOf[i] = inventory.NoNode + } + type pair struct{ o, n inventory.NodeID } + stack := []pair{{old.Root(), new.Root()}} + for len(stack) > 0 { + p := stack[len(stack)-1] + stack = stack[:len(stack)-1] + d.OldOf[p.n] = p.o + on, nn := old.Node(p.o), new.Node(p.n) + if !on.IsDir() || !nn.IsDir() { + continue + } + byName := map[string]inventory.NodeID{} + old.Children(p.o, func(id inventory.NodeID, n *inventory.Node) { byName[n.Name] = id }) + new.Children(p.n, func(id inventory.NodeID, n *inventory.Node) { + if oid, ok := byName[n.Name]; ok { + delete(byName, n.Name) + stack = append(stack, pair{oid, id}) + } + }) + for _, oid := range byName { + d.Removed = append(d.Removed, oid) + o := old.Node(oid) + r := d.removedUnder[p.n] + r[0] += o.TotAlloc + r[1] += o.TotSize + d.removedUnder[p.n] = r + } + } + sort.Slice(d.Removed, func(i, j int) bool { return d.Removed[i] < d.Removed[j] }) + return d +} + +// OldSize returns the matching old size of a New node and whether it existed. +func (d *Diff) OldSize(id inventory.NodeID, m inventory.SizeMode) (int64, bool) { + o := d.OldOf[id] + if o == inventory.NoNode { + return 0, false + } + return d.Old.Node(o).Total(m), true +} + +// Delta returns new minus old size for a New node (full size if new). +func (d *Diff) Delta(id inventory.NodeID, m inventory.SizeMode) int64 { + old, _ := d.OldSize(id, m) + return d.New.Node(id).Total(m) - old +} + +// Status classifies a node. +type Status uint8 + +const ( + Unchanged Status = iota + Added + Removed + Grew + Shrank +) + +func (s Status) String() string { + return [...]string{"unchanged", "new", "removed", "increased", "decreased"}[s] +} + +// StatusOf classifies a New node. +func (d *Diff) StatusOf(id inventory.NodeID, m inventory.SizeMode) Status { + if d.OldOf[id] == inventory.NoNode { + return Added + } + switch delta := d.Delta(id, m); { + case delta > 0: + return Grew + case delta < 0: + return Shrank + } + return Unchanged +} + +// Change is one row in a change report. +type Change struct { + Path string + Status Status + Delta int64 + Old int64 + New int64 + Node inventory.NodeID // in New (NoNode for removed) +} + +// Changes returns the k most significant changes. A directory is listed +// only if its change is not almost entirely explained by a single child, so +// the report points at /var/lib/postgresql rather than /, /var and /var/lib. +func (d *Diff) Changes(k int, m inventory.SizeMode) []Change { + var out []Change + for i := range d.New.Len() { + id := inventory.NodeID(i) + delta := d.Delta(id, m) + if delta == 0 { + continue + } + n := d.New.Node(id) + if d.OldOf[id] != inventory.NoNode && n.IsDir() && explainedByChild(d, id, delta, m) { + continue + } + if p := n.Parent; p != inventory.NoNode && d.OldOf[p] == inventory.NoNode { + continue // inside a wholly new directory; the directory is listed + } + old, _ := d.OldSize(id, m) + out = append(out, Change{Path: d.New.Path(id), Status: d.StatusOf(id, m), Delta: delta, Old: old, New: n.Total(m), Node: id}) + } + for _, oid := range d.Removed { + o := d.Old.Node(oid) + out = append(out, Change{Path: d.Old.Path(oid), Status: Removed, Delta: -o.Total(m), Old: o.Total(m), Node: inventory.NoNode}) + } + sort.Slice(out, func(i, j int) bool { + a, b := abs(out[i].Delta), abs(out[j].Delta) + if a != b { + return a > b + } + return out[i].Path < out[j].Path + }) + if k > 0 && len(out) > k { + out = out[:k] + } + return out +} + +func explainedByChild(d *Diff, id inventory.NodeID, delta int64, m inventory.SizeMode) bool { + explained := false + d.New.Children(id, func(c inventory.NodeID, _ *inventory.Node) { + if cd := d.Delta(c, m); (cd > 0) == (delta > 0) && abs(cd)*10 >= abs(delta)*8 { + explained = true + } + }) + if r, ok := d.removedUnder[id]; ok && !explained { + v := r[0] + if m == inventory.SizeLogical { + v = r[1] + } + if delta < 0 && v*10 >= abs(delta)*8 { + explained = true + } + } + return explained +} + +func abs(v int64) int64 { + if v < 0 { + return -v + } + return v +} diff --git a/internal/snapshot/snapshot.go b/internal/snapshot/snapshot.go new file mode 100644 index 0000000..4b5169d --- /dev/null +++ b/internal/snapshot/snapshot.go @@ -0,0 +1,433 @@ +// Package snapshot implements the versioned, checksummed binary inventory +// format and snapshot comparison. +// +// Layout (all integers little-endian / varint): +// +// magic 8 bytes brand.SnapshotMagic +// version uint16 currently 1 +// flags uint16 bit 0: body is gzip-compressed (always set by Save) +// body gzip stream of: +// meta uvarint length + JSON (root, timestamps, counters, options) +// exts uvarint count, then count × (uvarint length + bytes) +// nodes uvarint count, then per node: +// uvarint parentDelta (id-parent; 0 only for the root) +// byte kind, byte category, uvarint flags +// uvarint nameLen + name +// varint size, varint alloc, varint mtime +// uvarint mode, uid, gid, nlink, ext +// errors uvarint count, then per record: uvarint node, byte kind, +// uvarint len + name, uvarint len + message +// sha256 32 bytes over the uncompressed body +// +// Nodes are written in ID order; a parent always precedes its children, so a +// reader can rebuild the tree in one pass and malformed input cannot create +// cycles. Aggregates are recomputed on load, never trusted. +package snapshot + +import ( + "bufio" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/andydixon/mapsize/internal/brand" + "github.com/andydixon/mapsize/internal/inventory" +) + +// Version is the current format version. +const Version = 1 + +// Limits applied when reading untrusted snapshots. +const ( + MaxNodes = 1<<31 - 1 + MaxNameLen = 1 << 16 + MaxMetaLen = 16 << 20 + MaxExtLen = 64 + MaxMsgLen = 4096 + maxExts = 1<<16 - 1 + flagGzipped = 1 +) + +type meta struct { + Root string `json:"root"` + Created time.Time `json:"created"` + Start time.Time `json:"start"` + End time.Time `json:"end"` + Complete bool `json:"complete"` + Cancelled bool `json:"cancelled"` + Files int64 `json:"files"` + Dirs int64 `json:"dirs"` + Symlinks int64 `json:"symlinks"` + Others int64 `json:"others"` + ErrCounts []int64 `json:"err_counts"` + Excluded int64 `json:"excluded"` + SkippedMounts int64 `json:"skipped_mounts"` + VirtualSkipped int64 `json:"virtual_skipped"` + LoopsSkipped int64 `json:"loops_skipped"` + BrokenLinks int64 `json:"broken_links"` + HardlinkDups int64 `json:"hardlink_dups"` + Unscanned int64 `json:"unscanned"` + Excludes []string `json:"excludes"` + OneFileSystem bool `json:"one_file_system"` + Follow string `json:"follow"` + Workers int `json:"workers"` + Host string `json:"host"` + Generator string `json:"generator"` +} + +// IsSnapshot reports whether path starts with the snapshot magic. +func IsSnapshot(path string) bool { + f, err := os.Open(path) + if err != nil { + return false + } + defer f.Close() + b := make([]byte, len(brand.SnapshotMagic)) + _, err = io.ReadFull(f, b) + return err == nil && string(b) == brand.SnapshotMagic +} + +type writer struct { + w io.Writer + buf [binary.MaxVarintLen64]byte + err error +} + +func (w *writer) bytes(b []byte) { + if w.err == nil { + _, w.err = w.w.Write(b) + } +} +func (w *writer) uvarint(v uint64) { w.bytes(w.buf[:binary.PutUvarint(w.buf[:], v)]) } +func (w *writer) varint(v int64) { w.bytes(w.buf[:binary.PutVarint(w.buf[:], v)]) } +func (w *writer) str(s string) { w.uvarint(uint64(len(s))); w.bytes([]byte(s)) } + +// Save writes t to w. The caller must hold t's read lock. +func Save(out io.Writer, t *inventory.Tree) error { + bw := bufio.NewWriterSize(out, 1<<20) + hdr := make([]byte, 0, 12) + hdr = append(hdr, brand.SnapshotMagic...) + hdr = binary.LittleEndian.AppendUint16(hdr, Version) + hdr = binary.LittleEndian.AppendUint16(hdr, flagGzipped) + bw.Write(hdr) + + gz, _ := gzip.NewWriterLevel(bw, gzip.BestSpeed) + h := sha256.New() + w := &writer{w: io.MultiWriter(gz, h)} + + st := &t.Stats + host, _ := os.Hostname() + m := meta{Root: st.Root, Created: time.Now().UTC(), Start: st.Start, End: st.End, Complete: st.Complete, + Cancelled: st.Cancelled, Files: st.Files, Dirs: st.Dirs, Symlinks: st.Symlinks, Others: st.Others, + ErrCounts: st.ErrCounts[:], Excluded: st.Excluded, SkippedMounts: st.SkippedMounts, + VirtualSkipped: st.VirtualSkipped, LoopsSkipped: st.LoopsSkipped, BrokenLinks: st.BrokenLinks, + HardlinkDups: st.HardlinkDups, Unscanned: st.Unscanned, Excludes: st.Excludes, + OneFileSystem: st.OneFileSystem, Follow: st.Follow, Workers: st.Workers, Host: host, + Generator: brand.Name + " " + brand.Version} + mb, err := json.Marshal(m) + if err != nil { + return err + } + w.uvarint(uint64(len(mb))) + w.bytes(mb) + + w.uvarint(uint64(t.ExtCount())) + for i := range t.ExtCount() { + w.str(t.ExtByIndex(i)) + } + w.uvarint(uint64(t.Len())) + for i := range t.Len() { + id := inventory.NodeID(i) + n := t.Node(id) + if i == 0 { + w.uvarint(0) + } else { + w.uvarint(uint64(id - n.Parent)) + } + w.bytes([]byte{byte(n.Kind), byte(n.Cat)}) + w.uvarint(uint64(n.Flags)) + w.str(n.Name) + w.varint(n.Size) + w.varint(n.Alloc) + w.varint(n.MTime) + w.uvarint(uint64(n.Mode)) + w.uvarint(uint64(n.UID)) + w.uvarint(uint64(n.GID)) + w.uvarint(uint64(n.Nlink)) + w.uvarint(uint64(n.Ext)) + } + w.uvarint(uint64(len(st.Errors))) + for _, e := range st.Errors { + w.uvarint(uint64(e.Node)) + w.bytes([]byte{byte(e.Kind)}) + w.str(e.Name) + w.str(e.Msg) + } + if w.err != nil { + return w.err + } + if err := gz.Close(); err != nil { + return err + } + bw.Write(h.Sum(nil)) + return bw.Flush() +} + +// SaveFile writes a snapshot atomically (temp file + rename). +func SaveFile(path string, t *inventory.Tree) error { + tmp, err := os.CreateTemp(dirOf(path), ".mapsize-*.tmp") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + if err := Save(tmp, t); err != nil { + tmp.Close() + return err + } + if err := tmp.Close(); err != nil { + return err + } + return os.Rename(tmp.Name(), path) +} + +func dirOf(p string) string { + i := strings.LastIndexAny(p, `/\`) + if i < 0 { + return "." + } + if i == 0 { + return p[:1] + } + return p[:i] +} + +type reader struct { + r *bufio.Reader + err error +} + +func (r *reader) fail(format string, a ...any) { + if r.err == nil { + r.err = fmt.Errorf("snapshot: "+format, a...) + } +} + +func (r *reader) byte1() byte { + if r.err != nil { + return 0 + } + b, err := r.r.ReadByte() + if err != nil { + r.fail("truncated: %v", err) + } + return b +} + +func (r *reader) uvarint(max uint64, what string) uint64 { + if r.err != nil { + return 0 + } + v, err := binary.ReadUvarint(r.r) + if err != nil { + r.fail("bad %s: %v", what, err) + return 0 + } + if v > max { + r.fail("%s %d exceeds limit %d", what, v, max) + return 0 + } + return v +} + +func (r *reader) varint(what string) int64 { + if r.err != nil { + return 0 + } + v, err := binary.ReadVarint(r.r) + if err != nil { + r.fail("bad %s: %v", what, err) + } + return v +} + +func (r *reader) bytesN(n uint64) []byte { + if r.err != nil { + return nil + } + b := make([]byte, n) + if _, err := io.ReadFull(r.r, b); err != nil { + r.fail("truncated: %v", err) + return nil + } + return b +} + +func (r *reader) str(max uint64, what string) string { + return string(r.bytesN(r.uvarint(max, what+" length"))) +} + +// Load reads and validates a snapshot. +func Load(in io.Reader) (*inventory.Tree, error) { + br := bufio.NewReaderSize(in, 1<<20) + hdr := make([]byte, len(brand.SnapshotMagic)+4) + if _, err := io.ReadFull(br, hdr); err != nil { + return nil, errors.New("snapshot: file too short") + } + if string(hdr[:8]) != brand.SnapshotMagic { + return nil, errors.New("snapshot: not a " + brand.Name + " snapshot") + } + if v := binary.LittleEndian.Uint16(hdr[8:]); v != Version { + return nil, fmt.Errorf("snapshot: unsupported version %d (this build reads %d)", v, Version) + } + if binary.LittleEndian.Uint16(hdr[10:])&flagGzipped == 0 { + return nil, errors.New("snapshot: uncompressed bodies are not supported") + } + gz, err := gzip.NewReader(br) + if err != nil { + return nil, fmt.Errorf("snapshot: %w", err) + } + gz.Multistream(false) + h := sha256.New() + r := &reader{r: bufio.NewReaderSize(io.TeeReader(gz, h), 1<<16)} + + var m meta + if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil { + if err := json.Unmarshal(mb, &m); err != nil { + return nil, fmt.Errorf("snapshot: metadata: %w", err) + } + } + if r.err != nil { + return nil, r.err + } + t := inventory.New(m.Root, inventory.KindDir) + nExt := r.uvarint(maxExts, "extension count") + exts := make([]string, 0, min(nExt, 1024)) + for range nExt { + exts = append(exts, r.str(MaxExtLen, "extension")) + } + if r.err != nil { + return nil, r.err + } + extIndex := t.ExtIndexer() + nNodes := r.uvarint(MaxNodes, "node count") + if nNodes == 0 { + return nil, errors.New("snapshot: no root node") + } + for i := uint64(0); i < nNodes && r.err == nil; i++ { + pd := r.uvarint(MaxNodes, "parent") + kind := inventory.Kind(r.byte1()) + cat := inventory.Category(r.byte1()) + flags := inventory.Flags(r.uvarint(1<<16-1, "flags")) + name := r.str(MaxNameLen, "name") + size, alloc, mtime := r.varint("size"), r.varint("alloc"), r.varint("mtime") + mode := r.uvarint(1<<32-1, "mode") + uid := r.uvarint(1<<32-1, "uid") + gid := r.uvarint(1<<32-1, "gid") + nlink := r.uvarint(1<<32-1, "nlink") + ext := r.uvarint(nExt, "ext") + if r.err != nil { + break + } + if kind > inventory.KindOther || cat >= inventory.NumCategories { + r.fail("node %d: invalid kind/category", i) + break + } + if size < 0 || alloc < 0 { + r.fail("node %d: negative size", i) + break + } + var id inventory.NodeID + if i == 0 { + if pd != 0 || kind != inventory.KindDir && nNodes > 1 { + r.fail("invalid root") + break + } + id = 0 + rn := t.Node(0) + rn.Kind = kind + } else { + if pd == 0 || pd > i { + r.fail("node %d: invalid parent reference", i) + break + } + parent := inventory.NodeID(i - pd) + if t.Node(parent).Kind != inventory.KindDir { + r.fail("node %d: parent is not a directory", i) + break + } + if !validName(name) { + r.fail("node %d: invalid name %q", i, name) + break + } + id = t.Add(parent, name, kind) + } + n := t.Node(id) + n.Cat, n.Flags = cat, flags + n.Size, n.Alloc, n.MTime = size, alloc, mtime + n.Mode, n.UID, n.GID, n.Nlink = uint32(mode), uint32(uid), uint32(gid), uint32(nlink) + if kind != inventory.KindDir && ext < uint64(len(exts)) { + n.Ext = extIndex(exts[ext]) + } + } + nErr := r.uvarint(inventory.MaxErrorRecords, "error count") + for range nErr { + node := r.uvarint(nNodes-1, "error node") + kind := inventory.ErrKind(r.byte1()) + name := r.str(MaxNameLen, "error name") + msg := r.str(MaxMsgLen, "error message") + if r.err != nil { + break + } + if kind >= inventory.NumErrKinds { + kind = inventory.ErrOther + } + t.Stats.Errors = append(t.Stats.Errors, inventory.ErrorRecord{Node: inventory.NodeID(node), Kind: kind, Name: name, Msg: msg}) + } + if r.err != nil { + return nil, r.err + } + // The body must end here, then the checksum follows the gzip stream. + if _, err := r.r.ReadByte(); err != io.EOF { + return nil, errors.New("snapshot: trailing data in body") + } + sum := make([]byte, sha256.Size) + if _, err := io.ReadFull(br, sum); err != nil { + return nil, errors.New("snapshot: missing checksum") + } + if !bytes.Equal(sum, h.Sum(nil)) { + return nil, errors.New("snapshot: checksum mismatch (file corrupt)") + } + + t.Recompute() + st := &t.Stats + st.Start, st.End, st.Complete, st.Cancelled = m.Start, m.End, true, m.Cancelled + st.Files, st.Dirs, st.Symlinks, st.Others = m.Files, m.Dirs, m.Symlinks, m.Others + copy(st.ErrCounts[:], m.ErrCounts) + st.Excluded, st.SkippedMounts, st.VirtualSkipped = m.Excluded, m.SkippedMounts, m.VirtualSkipped + st.LoopsSkipped, st.BrokenLinks, st.HardlinkDups, st.Unscanned = m.LoopsSkipped, m.BrokenLinks, m.HardlinkDups, m.Unscanned + st.Excludes, st.OneFileSystem, st.Follow, st.Workers = m.Excludes, m.OneFileSystem, m.Follow, m.Workers + st.FromSnapshot = fmt.Sprintf("%s (%s, %s)", m.Created.Local().Format("2006-01-02 15:04"), m.Host, m.Generator) + return t, nil +} + +func validName(s string) bool { + return s != "" && s != "." && s != ".." && !strings.ContainsAny(s, "/\x00") +} + +// LoadFile opens and loads a snapshot file. +func LoadFile(path string) (*inventory.Tree, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + return Load(f) +} diff --git a/internal/snapshot/snapshot_test.go b/internal/snapshot/snapshot_test.go new file mode 100644 index 0000000..8722dc7 --- /dev/null +++ b/internal/snapshot/snapshot_test.go @@ -0,0 +1,123 @@ +package snapshot + +import ( + "bytes" + "testing" + + "github.com/andydixon/mapsize/internal/inventory" +) + +func sample() *inventory.Tree { + t := inventory.New("/srv", inventory.KindDir) + a := t.Add(0, "a", inventory.KindDir) + b := t.Add(0, "b", inventory.KindDir) + f1 := t.Add(a, "x.iso", inventory.KindFile) + t.Node(f1).Size, t.Node(f1).Alloc = 1000, 1024 + f2 := t.Add(b, "y.log", inventory.KindFile) + t.Node(f2).Size, t.Node(f2).Alloc = 50, 4096 + t.Node(f2).Flags = inventory.FlagSparse + t.Recompute() + t.Stats.Files, t.Stats.Dirs = 2, 3 + t.Stats.AddError(inventory.ErrorRecord{Node: b, Kind: inventory.ErrPermission, Msg: "denied"}) + return t +} + +func TestRoundTrip(t *testing.T) { + src := sample() + var buf bytes.Buffer + if err := Save(&buf, src); err != nil { + t.Fatal(err) + } + got, err := Load(bytes.NewReader(buf.Bytes())) + if err != nil { + t.Fatal(err) + } + if got.Len() != src.Len() { + t.Fatalf("len %d want %d", got.Len(), src.Len()) + } + for i := range src.Len() { + a, b := src.Node(inventory.NodeID(i)), got.Node(inventory.NodeID(i)) + if a.Name != b.Name || a.TotSize != b.TotSize || a.TotAlloc != b.TotAlloc || a.Flags != b.Flags || + src.Path(inventory.NodeID(i)) != got.Path(inventory.NodeID(i)) || src.ExtName(a) != got.ExtName(b) { + t.Fatalf("node %d differs: %+v vs %+v", i, a, b) + } + } + if got.Stats.ErrCounts[inventory.ErrPermission] != 1 || len(got.Stats.Errors) != 1 { + t.Fatalf("errors not restored: %+v", got.Stats) + } +} + +func TestCorruptionDetected(t *testing.T) { + var buf bytes.Buffer + Save(&buf, sample()) + data := buf.Bytes() + // Flip a byte in the checksum trailer. + bad := append([]byte(nil), data...) + bad[len(bad)-1] ^= 0xff + if _, err := Load(bytes.NewReader(bad)); err == nil { + t.Fatal("checksum corruption not detected") + } + // Every truncation must fail cleanly. + for i := 0; i < len(data)-1; i += 7 { + if _, err := Load(bytes.NewReader(data[:i])); err == nil { + t.Fatalf("truncation at %d accepted", i) + } + } +} + +func FuzzLoad(f *testing.F) { + var buf bytes.Buffer + Save(&buf, sample()) + f.Add(buf.Bytes()) + f.Add([]byte("MAPSIZE\x00\x01\x00\x01\x00")) + f.Fuzz(func(t *testing.T, data []byte) { + tr, err := Load(bytes.NewReader(data)) + if err != nil { + return + } + // A successfully loaded tree must be internally consistent. + for i := 1; i < tr.Len(); i++ { + if p := tr.Node(inventory.NodeID(i)).Parent; p >= inventory.NodeID(i) { + t.Fatalf("parent %d >= child %d", p, i) + } + } + _ = tr.Path(inventory.NodeID(tr.Len() - 1)) + }) +} + +func TestCompare(t *testing.T) { + old := sample() + nw := inventory.New("/srv", inventory.KindDir) + a := nw.Add(0, "a", inventory.KindDir) + f1 := nw.Add(a, "x.iso", inventory.KindFile) + nw.Node(f1).Size, nw.Node(f1).Alloc = 5000, 5120 + c := nw.Add(0, "c", inventory.KindDir) + f3 := nw.Add(c, "z", inventory.KindFile) + nw.Node(f3).Size, nw.Node(f3).Alloc = 10, 4096 + nw.Recompute() + + d := Compare(old, nw) + if d.StatusOf(f1, inventory.SizeAllocated) != Grew || d.Delta(f1, inventory.SizeAllocated) != 4096 { + t.Fatalf("x.iso: %v %d", d.StatusOf(f1, 0), d.Delta(f1, 0)) + } + if d.StatusOf(c, 0) != Added { + t.Fatal("c should be new") + } + if len(d.Removed) != 1 || old.Node(d.Removed[0]).Name != "b" { + t.Fatalf("removed %v", d.Removed) + } + ch := d.Changes(10, inventory.SizeAllocated) + paths := map[string]Status{} + for _, c := range ch { + paths[c.Path] = c.Status + } + if paths["/srv/a/x.iso"] != Grew || paths["/srv/b"] != Removed || paths["/srv/c"] != Added { + t.Fatalf("changes %+v", ch) + } + if _, ok := paths["/srv/a"]; ok { + t.Fatal("/srv/a should be explained by its child") + } + if _, ok := paths["/srv/c/z"]; ok { + t.Fatal("contents of new dirs should not be listed separately") + } +}