Speed up snapshot loading; polish help and table bars

Decompression and checksumming overlap with parsing via a pipe, and a
concrete-typed varint reader removes per-byte interface calls
(0.39s -> 0.29s for 458K nodes).
This commit is contained in:
Andy Dixon
2026-10-02 08:35:47 +01:00
parent 6bb8ddc0f7
commit 71556b76de
5 changed files with 81 additions and 7 deletions
+16
View File
@@ -44,6 +44,22 @@ path reconstruction for job dispatch ~7 %; aggregation is negligible.
**~158 bytes per node** (node record ~112 bytes plus name storage and **~158 bytes per node** (node record ~112 bytes plus name storage and
directory side tables). Extrapolated, 10 M entries need ~1.6 GiB. directory side tables). Extrapolated, 10 M entries need ~1.6 GiB.
## Snapshots
Same 458,510-node tree:
| operation | result |
|---|---:|
| snapshot file size (gzip level 1) | 6.7 MB (~15 bytes/node) |
| scan + save | 1.03 s |
| load (`mapsize --no-ui file.msz`), 3 runs | 0.28 / 0.29 / 0.32 s, 81 MB RSS |
| warm-cache rescan for comparison | 0.55 s |
Loading decompresses and verifies SHA-256 on a separate goroutine, overlapping
with parsing. Against a warm cache the win is ~1.9×; against a cold cache or
a network filesystem, where scans are dominated by I/O latency, it is far
larger (not yet measured here).
## UI ## UI
`go test ./internal/tui -bench Big` on a synthetic in-memory tree of `go test ./internal/tui -bench Big` on a synthetic in-memory tree of
+43 -3
View File
@@ -235,7 +235,7 @@ func (r *reader) uvarint(max uint64, what string) uint64 {
if r.err != nil { if r.err != nil {
return 0 return 0
} }
v, err := binary.ReadUvarint(r.r) v, err := readUvarint(r.r)
if err != nil { if err != nil {
r.fail("bad %s: %v", what, err) r.fail("bad %s: %v", what, err)
return 0 return 0
@@ -251,13 +251,44 @@ func (r *reader) varint(what string) int64 {
if r.err != nil { if r.err != nil {
return 0 return 0
} }
v, err := binary.ReadVarint(r.r) ux, err := readUvarint(r.r)
if err != nil { if err != nil {
r.fail("bad %s: %v", what, err) r.fail("bad %s: %v", what, err)
} }
v := int64(ux >> 1) // zig-zag, as encoding/binary.PutVarint
if ux&1 != 0 {
v = ^v
}
return v return v
} }
var errOverflow = errors.New("varint overflows 64 bits")
// readUvarint is encoding/binary.ReadUvarint specialised to *bufio.Reader
// (no interface dispatch per byte; this is the loader's hot loop).
func readUvarint(br *bufio.Reader) (uint64, error) {
var x uint64
var s uint
for i := 0; i < binary.MaxVarintLen64; i++ {
b, err := br.ReadByte()
if err != nil {
if i > 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
return x, err
}
if b < 0x80 {
if i == binary.MaxVarintLen64-1 && b > 1 {
return x, errOverflow
}
return x | uint64(b)<<s, nil
}
x |= uint64(b&0x7f) << s
s += 7
}
return x, errOverflow
}
func (r *reader) bytesN(n uint64) []byte { func (r *reader) bytesN(n uint64) []byte {
if r.err != nil { if r.err != nil {
return nil return nil
@@ -295,8 +326,17 @@ func Load(in io.Reader) (*inventory.Tree, error) {
return nil, fmt.Errorf("snapshot: %w", err) return nil, fmt.Errorf("snapshot: %w", err)
} }
gz.Multistream(false) gz.Multistream(false)
// Decompression and hashing run on their own goroutine, overlapping
// with parsing. The pipe's close orders the hash and br accesses: the
// parser only sees EOF after the copier has finished with both.
h := sha256.New() h := sha256.New()
r := &reader{r: bufio.NewReaderSize(io.TeeReader(gz, h), 1<<16)} pr, pw := io.Pipe()
defer pr.Close() // unblocks the copier if we bail out early
go func() {
_, err := io.CopyBuffer(pw, io.TeeReader(gz, h), make([]byte, 256<<10))
pw.CloseWithError(err)
}()
r := &reader{r: bufio.NewReaderSize(pr, 1<<16)}
var m meta var m meta
if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil { if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil {
+16
View File
@@ -1,7 +1,9 @@
package snapshot package snapshot
import ( import (
"bufio"
"bytes" "bytes"
"encoding/binary"
"testing" "testing"
"github.com/andydixon/mapsize/internal/inventory" "github.com/andydixon/mapsize/internal/inventory"
@@ -121,3 +123,17 @@ func TestCompare(t *testing.T) {
t.Fatal("contents of new dirs should not be listed separately") t.Fatal("contents of new dirs should not be listed separately")
} }
} }
func TestReadUvarintMatchesStdlib(t *testing.T) {
for _, v := range []uint64{0, 1, 127, 128, 300, 1 << 35, 1<<64 - 1} {
buf := binary.AppendUvarint(nil, v)
got, err := readUvarint(bufio.NewReader(bytes.NewReader(buf)))
if err != nil || got != v {
t.Fatalf("%d: got %d %v", v, got, err)
}
}
bad := bytes.Repeat([]byte{0xff}, 11)
if _, err := readUvarint(bufio.NewReader(bytes.NewReader(bad))); err == nil {
t.Fatal("overflow not detected")
}
}
+5 -3
View File
@@ -230,7 +230,7 @@ type helpModal struct{}
func newHelpModal(*Model) modal { return helpModal{} } func newHelpModal(*Model) modal { return helpModal{} }
func (helpModal) width() int { return 74 } func (helpModal) width() int { return 78 }
func (helpModal) title() string { return "Keyboard reference" } func (helpModal) title() string { return "Keyboard reference" }
func (helpModal) footer(*Model) string { return "Esc close · ↑↓ scroll" } func (helpModal) footer(*Model) string { return "Esc close · ↑↓ scroll" }
@@ -246,7 +246,7 @@ func (helpModal) content(m *Model, _ int) []line {
} }
out = append(out, styled(s, h)) out = append(out, styled(s, h))
} }
row := func(keys, desc string) { out = append(out, line{{" " + textutil.PadRight(keys, 16), &k}, {desc, &d}}) } row := func(keys, desc string) { out = append(out, line{{" " + textutil.PadRight(keys, 19), &k}, {desc, &d}}) }
sec("NAVIGATION") sec("NAVIGATION")
row("↑ ↓ ← → hjkl", "move spatially through the treemap") row("↑ ↓ ← → hjkl", "move spatially through the treemap")
row("Enter", "inspect the selected item") row("Enter", "inspect the selected item")
@@ -260,7 +260,9 @@ func (helpModal) content(m *Model, _ int) []line {
row("Esc", "clear the filter") row("Esc", "clear the filter")
out = append(out, styled(" ubuntu *.iso size > 5GB ext IN (iso,qcow2) age > 365d", Style{FG: t.Muted})) out = append(out, styled(" ubuntu *.iso size > 5GB ext IN (iso,qcow2) age > 365d", Style{FG: t.Muted}))
out = append(out, styled(" path contains cache AND NOT type = dir owner = andy flag = sparse", Style{FG: t.Muted})) out = append(out, styled(" path contains cache AND NOT type = dir owner = andy flag = sparse", Style{FG: t.Muted}))
out = append(out, styled(" fields: "+filter.Fields, Style{FG: t.Muted})) fields := strings.Fields(filter.Fields)
out = append(out, styled(" fields: "+strings.Join(fields[:7], " "), Style{FG: t.Muted}))
out = append(out, styled(" "+strings.Join(fields[7:], " "), Style{FG: t.Muted}))
out = append(out, styled(" sizes: KiB/MiB/GiB (1024), kB/MB/GB (1000); ages: s min h d w mo y", Style{FG: t.Muted})) out = append(out, styled(" sizes: KiB/MiB/GiB (1024), kB/MB/GB (1000); ages: s min h d w mo y", Style{FG: t.Muted}))
sec("VIEWS") sec("VIEWS")
row("Tab Shift+Tab", "next / previous view") row("Tab Shift+Tab", "next / previous view")
+1 -1
View File
@@ -126,7 +126,7 @@ func (tb *table) paint(m *Model, cv *Canvas, r treemap.Rect, cols []column, rows
if !sel && ci < len(row.styles) && row.styles[ci] != nil { if !sel && ci < len(row.styles) && row.styles[ci] != nil {
st = *row.styles[ci] st = *row.styles[ci]
} }
if c.title == "" && row.bar >= 0 && c.width > 0 { if c.title == "" && row.bar >= 0 {
m.paintBar(cv, x, y, w, row.bar, row.barCol, sel) m.paintBar(cv, x, y, w, row.bar, row.barCol, sel)
} else if c.right { } else if c.right {
cv.Text(x, y, textutil.PadLeft(s, w), w, st) cv.Text(x, y, textutil.PadLeft(s, w), w, st)