Fix CSV path joining and relative exclusions; add scan benchmark script
This commit is contained in:
@@ -21,6 +21,26 @@ bin/treegen --out /tmp/t --directories 50000 --files 1000000 --max-depth 20 --sp
|
||||
bin/mapsize /tmp/t
|
||||
```
|
||||
|
||||
## Benchmarks
|
||||
|
||||
```sh
|
||||
make benchmark # Go micro-benchmarks (layout, render, filter, scan)
|
||||
scripts/bench-scan.sh /path 1 2 4 8 # wall time and peak RSS per worker count
|
||||
sudo DROP_CACHES=1 scripts/bench-scan.sh /path # cold-cache runs
|
||||
MAPSIZE_MEM_ROOT=/path go test ./internal/scan -run TestMemoryPerNode -v
|
||||
```
|
||||
|
||||
Paste results into PERFORMANCE.md with the machine description. Never edit
|
||||
numbers by hand.
|
||||
|
||||
## Terminal restoration checks
|
||||
|
||||
```sh
|
||||
MAPSIZE_DEBUG_PANIC=ui bin/mapsize . # panics on the first key press
|
||||
MAPSIZE_DEBUG_PANIC=scanner bin/mapsize / # panics in a scanner goroutine
|
||||
stty -a # echo/icanon must be back on
|
||||
```
|
||||
|
||||
## Profiling
|
||||
|
||||
```sh
|
||||
|
||||
@@ -8,6 +8,7 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
@@ -158,7 +159,7 @@ func CSV(w io.Writer, t *inventory.Tree, root inventory.NodeID, maxDepth int) er
|
||||
return nil
|
||||
}
|
||||
for _, c := range t.SortedChildren(id, inventory.SizeAllocated) {
|
||||
if err := rec(c, path+"/"+t.Node(c).Name, depth-1); err != nil {
|
||||
if err := rec(c, filepath.Join(path, t.Node(c).Name), depth-1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@@ -138,6 +138,17 @@ type inodeKey struct{ dev, ino uint64 }
|
||||
// Start stats root, creates the tree and begins scanning in the background.
|
||||
func Start(ctx context.Context, root string, opts Options) (*Scanner, error) {
|
||||
opts.defaults()
|
||||
// Path-style exclusions are matched against absolute paths.
|
||||
excl := make([]string, len(opts.Excludes))
|
||||
for i, p := range opts.Excludes {
|
||||
excl[i] = p
|
||||
if strings.ContainsAny(p, `/\`) {
|
||||
if a, err := filepath.Abs(p); err == nil {
|
||||
excl[i] = a
|
||||
}
|
||||
}
|
||||
}
|
||||
opts.Excludes = excl
|
||||
abs, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@@ -237,6 +237,12 @@ func TestExcludes(t *testing.T) {
|
||||
if tr.Stats.Excluded != 2 || tr.Node(0).Files != 1 {
|
||||
t.Fatalf("excluded=%d files=%d", tr.Stats.Excluded, tr.Node(0).Files)
|
||||
}
|
||||
// Relative path patterns resolve against the working directory.
|
||||
t.Chdir(root)
|
||||
tr = runScan(t, root, Options{Workers: 2, Excludes: []string{"./skip"}})
|
||||
if tr.Stats.Excluded != 1 {
|
||||
t.Fatalf("relative exclude: excluded=%d", tr.Stats.Excluded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnicodeAndHostileNames(t *testing.T) {
|
||||
|
||||
Executable
+36
@@ -0,0 +1,36 @@
|
||||
#!/usr/bin/env bash
|
||||
# Records scan benchmarks as Markdown table rows for PERFORMANCE.md.
|
||||
#
|
||||
# scripts/bench-scan.sh PATH [workers...] (default workers: 1 2 4 8 16)
|
||||
#
|
||||
# Linux only (uses /usr/bin/time, df, lsblk). Runs each configuration three
|
||||
# times against a warm cache; to measure a cold cache, run as root with
|
||||
# DROP_CACHES=1, which drops the page cache before every run.
|
||||
set -euo pipefail
|
||||
path=${1:?usage: bench-scan.sh PATH [workers...]}
|
||||
shift || true
|
||||
workers=${*:-1 2 4 8 16}
|
||||
bin=${MAPSIZE:-bin/mapsize}
|
||||
[[ -x $bin ]] || make build >/dev/null
|
||||
|
||||
fs=$(df -T "$path" | awk 'NR==2 {print $2}')
|
||||
dev=$(df "$path" | awk 'NR==2 {print $1}')
|
||||
medium=$(lsblk -no ROTA "$dev" 2>/dev/null | head -1 | sed 's/1/rotational/;s/0/solid-state/')
|
||||
counts=$("$bin" --no-ui "$path" 2>/dev/null | awk '/^Files/ {f=$2} /^Directories/ {d=$2} END {print f" files, "d" dirs"}')
|
||||
|
||||
echo "<!-- $(date -u +%FT%TZ) $(uname -srm), $(nproc) CPUs -->"
|
||||
echo "Path: \`$path\` · $fs on ${medium:-unknown medium} · $counts"
|
||||
echo
|
||||
echo "| workers | cache | wall (s) | peak RSS (MB) |"
|
||||
echo "|---:|---|---:|---:|"
|
||||
for w in $workers; do
|
||||
for i in 1 2 3; do
|
||||
cache=warm
|
||||
if [[ ${DROP_CACHES:-} == 1 ]]; then
|
||||
sync; echo 3 > /proc/sys/vm/drop_caches; cache=cold
|
||||
fi
|
||||
out=$( { /usr/bin/time -f "%e %M" "$bin" --no-ui --workers "$w" "$path" >/dev/null; } 2>&1 | tail -1)
|
||||
read -r secs kb <<<"$out"
|
||||
echo "| $w | $cache | $secs | $((kb / 1024)) |"
|
||||
done
|
||||
done
|
||||
Reference in New Issue
Block a user