Fix CSV path joining and relative exclusions; add scan benchmark script
This commit is contained in:
@@ -21,6 +21,26 @@ bin/treegen --out /tmp/t --directories 50000 --files 1000000 --max-depth 20 --sp
|
|||||||
bin/mapsize /tmp/t
|
bin/mapsize /tmp/t
|
||||||
```
|
```
|
||||||
|
|
||||||
|
## Benchmarks
|
||||||
|
|
||||||
|
```sh
|
||||||
|
make benchmark # Go micro-benchmarks (layout, render, filter, scan)
|
||||||
|
scripts/bench-scan.sh /path 1 2 4 8 # wall time and peak RSS per worker count
|
||||||
|
sudo DROP_CACHES=1 scripts/bench-scan.sh /path # cold-cache runs
|
||||||
|
MAPSIZE_MEM_ROOT=/path go test ./internal/scan -run TestMemoryPerNode -v
|
||||||
|
```
|
||||||
|
|
||||||
|
Paste results into PERFORMANCE.md with the machine description. Never edit
|
||||||
|
numbers by hand.
|
||||||
|
|
||||||
|
## Terminal restoration checks
|
||||||
|
|
||||||
|
```sh
|
||||||
|
MAPSIZE_DEBUG_PANIC=ui bin/mapsize . # panics on the first key press
|
||||||
|
MAPSIZE_DEBUG_PANIC=scanner bin/mapsize / # panics in a scanner goroutine
|
||||||
|
stty -a # echo/icanon must be back on
|
||||||
|
```
|
||||||
|
|
||||||
## Profiling
|
## Profiling
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"path/filepath"
|
||||||
"strconv"
|
"strconv"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@@ -158,7 +159,7 @@ func CSV(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) er
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
|
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
|
||||||
if err := rec(c, path+"/"+t.Node(c).Name, depth-1); err != nil {
|
if err := rec(c, filepath.Join(path, t.Node(c).Name), depth-1); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -138,6 +138,17 @@ type inodeKey struct{ dev, ino uint64 }
|
|||||||
// Start stats root, creates the tree and begins scanning in the background.
|
// Start stats root, creates the tree and begins scanning in the background.
|
||||||
func Start(ctx context.Context, root string, opts Options) (*Scanner, error) {
|
func Start(ctx context.Context, root string, opts Options) (*Scanner, error) {
|
||||||
opts.defaults()
|
opts.defaults()
|
||||||
|
// Path-style exclusions are matched against absolute paths.
|
||||||
|
excl := make([]string, len(opts.Excludes))
|
||||||
|
for i, p := range opts.Excludes {
|
||||||
|
excl[i] = p
|
||||||
|
if strings.ContainsAny(p, `/\`) {
|
||||||
|
if a, err := filepath.Abs(p); err == nil {
|
||||||
|
excl[i] = a
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
opts.Excludes = excl
|
||||||
abs, err := filepath.Abs(root)
|
abs, err := filepath.Abs(root)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
|
|||||||
@@ -237,6 +237,12 @@ func TestExcludes(t *testing.T) {
|
|||||||
if tr.Stats.Excluded != 2 || tr.Node(0).Files != 1 {
|
if tr.Stats.Excluded != 2 || tr.Node(0).Files != 1 {
|
||||||
t.Fatalf("excluded=%d files=%d", tr.Stats.Excluded, tr.Node(0).Files)
|
t.Fatalf("excluded=%d files=%d", tr.Stats.Excluded, tr.Node(0).Files)
|
||||||
}
|
}
|
||||||
|
// Relative path patterns resolve against the working directory.
|
||||||
|
t.Chdir(root)
|
||||||
|
tr = runScan(t, root, Options{Workers: 2, Excludes: []string{"./skip"}})
|
||||||
|
if tr.Stats.Excluded != 1 {
|
||||||
|
t.Fatalf("relative exclude: excluded=%d", tr.Stats.Excluded)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestUnicodeAndHostileNames(t *testing.T) {
|
func TestUnicodeAndHostileNames(t *testing.T) {
|
||||||
|
|||||||
Executable
+36
@@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Records scan benchmarks as Markdown table rows for PERFORMANCE.md.
|
||||||
|
#
|
||||||
|
# scripts/bench-scan.sh PATH [workers...] (default workers: 1 2 4 8 16)
|
||||||
|
#
|
||||||
|
# Linux only (uses /usr/bin/time, df, lsblk). Runs each configuration three
|
||||||
|
# times against a warm cache; to measure a cold cache, run as root with
|
||||||
|
# DROP_CACHES=1, which drops the page cache before every run.
|
||||||
|
set -euo pipefail
|
||||||
|
path=${1:?usage: bench-scan.sh PATH [workers...]}
|
||||||
|
shift || true
|
||||||
|
workers=${*:-1 2 4 8 16}
|
||||||
|
bin=${MAPSIZE:-bin/mapsize}
|
||||||
|
[[ -x $bin ]] || make build >/dev/null
|
||||||
|
|
||||||
|
fs=$(df -T "$path" | awk 'NR==2 {print $2}')
|
||||||
|
dev=$(df "$path" | awk 'NR==2 {print $1}')
|
||||||
|
medium=$(lsblk -no ROTA "$dev" 2>/dev/null | head -1 | sed 's/1/rotational/;s/0/solid-state/')
|
||||||
|
counts=$("$bin" --no-ui "$path" 2>/dev/null | awk '/^Files/ {f=$2} /^Directories/ {d=$2} END {print f" files, "d" dirs"}')
|
||||||
|
|
||||||
|
echo "<!-- $(date -u +%FT%TZ) $(uname -srm), $(nproc) CPUs -->"
|
||||||
|
echo "Path: \`$path\` · $fs on ${medium:-unknown medium} · $counts"
|
||||||
|
echo
|
||||||
|
echo "| workers | cache | wall (s) | peak RSS (MB) |"
|
||||||
|
echo "|---:|---|---:|---:|"
|
||||||
|
for w in $workers; do
|
||||||
|
for i in 1 2 3; do
|
||||||
|
cache=warm
|
||||||
|
if [[ ${DROP_CACHES:-} == 1 ]]; then
|
||||||
|
sync; echo 3 > /proc/sys/vm/drop_caches; cache=cold
|
||||||
|
fi
|
||||||
|
out=$( { /usr/bin/time -f "%e %M" "$bin" --no-ui --workers "$w" "$path" >/dev/null; } 2>&1 | tail -1)
|
||||||
|
read -r secs kb <<<"$out"
|
||||||
|
echo "| $w | $cache | $secs | $((kb / 1024)) |"
|
||||||
|
done
|
||||||
|
done
|
||||||
Reference in New Issue
Block a user