Speed up snapshot loading; polish help and table bars
Decompression and checksumming overlap with parsing via a pipe, and a concrete-typed varint reader removes per-byte interface calls (0.39s -> 0.29s for 458K nodes).
This commit is contained in:
Andy Dixon
@@ -44,6 +44,22 @@ path reconstruction for job dispatch ~7 %; aggregation is negligible.
|
||||
**~158 bytes per node** (node record ~112 bytes plus name storage and
|
||||
directory side tables). Extrapolated, 10 M entries need ~1.6 GiB.
|
||||
|
||||
## Snapshots
|
||||
|
||||
Same 458,510-node tree:
|
||||
|
||||
| operation | result |
|
||||
|---|---:|
|
||||
| snapshot file size (gzip level 1) | 6.7 MB (~15 bytes/node) |
|
||||
| scan + save | 1.03 s |
|
||||
| load (`mapsize --no-ui file.msz`), 3 runs | 0.28 / 0.29 / 0.32 s, 81 MB RSS |
|
||||
| warm-cache rescan for comparison | 0.55 s |
|
||||
|
||||
Loading decompresses and verifies SHA-256 on a separate goroutine, overlapping
|
||||
with parsing. Against a warm cache the win is ~1.9×; against a cold cache or
|
||||
a network filesystem, where scans are dominated by I/O latency, it is far
|
||||
larger (not yet measured here).
|
||||
|
||||
## UI
|
||||
|
||||
`go test ./internal/tui -bench Big` on a synthetic in-memory tree of
|
||||
|
||||
@@ -235,7 +235,7 @@ func (r *reader) uvarint(max uint64, what string) uint64 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
v, err := binary.ReadUvarint(r.r)
|
||||
v, err := readUvarint(r.r)
|
||||
if err != nil {
|
||||
r.fail("bad %s: %v", what, err)
|
||||
return 0
|
||||
@@ -251,13 +251,44 @@ func (r *reader) varint(what string) int64 {
|
||||
if r.err != nil {
|
||||
return 0
|
||||
}
|
||||
v, err := binary.ReadVarint(r.r)
|
||||
ux, err := readUvarint(r.r)
|
||||
if err != nil {
|
||||
r.fail("bad %s: %v", what, err)
|
||||
}
|
||||
v := int64(ux >> 1) // zig-zag, as encoding/binary.PutVarint
|
||||
if ux&1 != 0 {
|
||||
v = ^v
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
var errOverflow = errors.New("varint overflows 64 bits")
|
||||
|
||||
// readUvarint is encoding/binary.ReadUvarint specialised to *bufio.Reader
|
||||
// (no interface dispatch per byte; this is the loader's hot loop).
|
||||
func readUvarint(br *bufio.Reader) (uint64, error) {
|
||||
var x uint64
|
||||
var s uint
|
||||
for i := 0; i < binary.MaxVarintLen64; i++ {
|
||||
b, err := br.ReadByte()
|
||||
if err != nil {
|
||||
if i > 0 && err == io.EOF {
|
||||
err = io.ErrUnexpectedEOF
|
||||
}
|
||||
return x, err
|
||||
}
|
||||
if b < 0x80 {
|
||||
if i == binary.MaxVarintLen64-1 && b > 1 {
|
||||
return x, errOverflow
|
||||
}
|
||||
return x | uint64(b)<<s, nil
|
||||
}
|
||||
x |= uint64(b&0x7f) << s
|
||||
s += 7
|
||||
}
|
||||
return x, errOverflow
|
||||
}
|
||||
|
||||
func (r *reader) bytesN(n uint64) []byte {
|
||||
if r.err != nil {
|
||||
return nil
|
||||
@@ -295,8 +326,17 @@ func Load(in io.Reader) (*inventory.Tree, error) {
|
||||
return nil, fmt.Errorf("snapshot: %w", err)
|
||||
}
|
||||
gz.Multistream(false)
|
||||
// Decompression and hashing run on their own goroutine, overlapping
|
||||
// with parsing. The pipe's close orders the hash and br accesses: the
|
||||
// parser only sees EOF after the copier has finished with both.
|
||||
h := sha256.New()
|
||||
r := &reader{r: bufio.NewReaderSize(io.TeeReader(gz, h), 1<<16)}
|
||||
pr, pw := io.Pipe()
|
||||
defer pr.Close() // unblocks the copier if we bail out early
|
||||
go func() {
|
||||
_, err := io.CopyBuffer(pw, io.TeeReader(gz, h), make([]byte, 256<<10))
|
||||
pw.CloseWithError(err)
|
||||
}()
|
||||
r := &reader{r: bufio.NewReaderSize(pr, 1<<16)}
|
||||
|
||||
var m meta
|
||||
if mb := r.bytesN(r.uvarint(MaxMetaLen, "metadata length")); r.err == nil {
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/mapsize/internal/inventory"
|
||||
@@ -121,3 +123,17 @@ func TestCompare(t *testing.T) {
|
||||
t.Fatal("contents of new dirs should not be listed separately")
|
||||
}
|
||||
}
|
||||
|
||||
func TestReadUvarintMatchesStdlib(t *testing.T) {
|
||||
for _, v := range []uint64{0, 1, 127, 128, 300, 1 << 35, 1<<64 - 1} {
|
||||
buf := binary.AppendUvarint(nil, v)
|
||||
got, err := readUvarint(bufio.NewReader(bytes.NewReader(buf)))
|
||||
if err != nil || got != v {
|
||||
t.Fatalf("%d: got %d %v", v, got, err)
|
||||
}
|
||||
}
|
||||
bad := bytes.Repeat([]byte{0xff}, 11)
|
||||
if _, err := readUvarint(bufio.NewReader(bytes.NewReader(bad))); err == nil {
|
||||
t.Fatal("overflow not detected")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -230,7 +230,7 @@ type helpModal struct{}
|
||||
|
||||
func newHelpModal(*Model) modal { return helpModal{} }
|
||||
|
||||
func (helpModal) width() int { return 74 }
|
||||
func (helpModal) width() int { return 78 }
|
||||
func (helpModal) title() string { return "Keyboard reference" }
|
||||
func (helpModal) footer(*Model) string { return "Esc close · ↑↓ scroll" }
|
||||
|
||||
@@ -246,7 +246,7 @@ func (helpModal) content(m *Model, _ int) []line {
|
||||
}
|
||||
out = append(out, styled(s, h))
|
||||
}
|
||||
row := func(keys, desc string) { out = append(out, line{{" " + textutil.PadRight(keys, 16), &k}, {desc, &d}}) }
|
||||
row := func(keys, desc string) { out = append(out, line{{" " + textutil.PadRight(keys, 19), &k}, {desc, &d}}) }
|
||||
sec("NAVIGATION")
|
||||
row("↑ ↓ ← → hjkl", "move spatially through the treemap")
|
||||
row("Enter", "inspect the selected item")
|
||||
@@ -260,7 +260,9 @@ func (helpModal) content(m *Model, _ int) []line {
|
||||
row("Esc", "clear the filter")
|
||||
out = append(out, styled(" ubuntu *.iso size > 5GB ext IN (iso,qcow2) age > 365d", Style{FG: t.Muted}))
|
||||
out = append(out, styled(" path contains cache AND NOT type = dir owner = andy flag = sparse", Style{FG: t.Muted}))
|
||||
out = append(out, styled(" fields: "+filter.Fields, Style{FG: t.Muted}))
|
||||
fields := strings.Fields(filter.Fields)
|
||||
out = append(out, styled(" fields: "+strings.Join(fields[:7], " "), Style{FG: t.Muted}))
|
||||
out = append(out, styled(" "+strings.Join(fields[7:], " "), Style{FG: t.Muted}))
|
||||
out = append(out, styled(" sizes: KiB/MiB/GiB (1024), kB/MB/GB (1000); ages: s min h d w mo y", Style{FG: t.Muted}))
|
||||
sec("VIEWS")
|
||||
row("Tab Shift+Tab", "next / previous view")
|
||||
|
||||
@@ -126,7 +126,7 @@ func (tb *table) paint(m *Model, cv *Canvas, r treemap.Rect, cols []column, rows
|
||||
if !sel && ci < len(row.styles) && row.styles[ci] != nil {
|
||||
st = *row.styles[ci]
|
||||
}
|
||||
if c.title == "" && row.bar >= 0 && c.width > 0 {
|
||||
if c.title == "" && row.bar >= 0 {
|
||||
m.paintBar(cv, x, y, w, row.bar, row.barCol, sel)
|
||||
} else if c.right {
|
||||
cv.Text(x, y, textutil.PadLeft(s, w), w, st)
|
||||
|
||||
Reference in New Issue
Block a user