andy/mapsize
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

rust: port snapshot

Browse Source
This commit is contained in:
Andy Dixon
2026-10-08 17:19:48 +01:00
parent a530bc7a75
commit 45c185a46a
4 changed files with 1150 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rust/src/lib.rs
+1
View File
@@ -7,6 +7,7 @@ pub mod export;
pub mod inventory; pub mod inventory;
pub mod platform; pub mod platform;
pub mod scan; pub mod scan;
pub mod snapshot;
pub mod textutil; pub mod textutil;
/// Boxed error used across mode boundaries. /// Boxed error used across mode boundaries.
rust/src/snapshot/compare.rs
+160
View File
@@ -0,0 +1,160 @@
use crate::inventory::{sat_add, NodeId, SizeMode, Tree, NO_NODE};
use std::collections::HashMap;
use std::fmt;
/// Relates two inventories by path. It owns the old tree; methods that need
/// the new one take it as a parameter. The new tree must not be mutated
/// while a Diff is in use (snapshots and finished scans satisfy this).
pub struct Diff {
pub old: Tree,
/// Maps a new NodeId to the matching old NodeId, or NO_NODE if the entry
/// is new.
pub old_of: Vec<NodeId>,
/// The topmost old nodes with no counterpart in the new tree.
pub removed: Vec<NodeId>,
/// Per new directory, the sizes of removed children ([allocated,
/// logical]) so growth colouring can account for them.
removed_under: HashMap<NodeId, [i64; 2]>,
}
/// Matches nodes by name, directory by directory, from the roots.
pub fn compare(old: Tree, new: &Tree) -> Diff {
let mut old_of = vec![NO_NODE; new.len()];
let mut removed = Vec::new();
let mut removed_under: HashMap<NodeId, [i64; 2]> = HashMap::new();
let mut stack = vec![(old.root(), new.root())];
while let Some((o, n)) = stack.pop() {
old_of[n as usize] = o;
if !old.node(o).is_dir() || !new.node(n).is_dir() {
continue;
}
let mut by_name: HashMap<&[u8], NodeId> = old.children(o).map(|(id, c)| (&c.name[..], id)).collect();
for (id, c) in new.children(n) {
if let Some(oid) = by_name.remove(&c.name[..]) {
stack.push((oid, id));
}
}
for oid in by_name.into_values() {
removed.push(oid);
let on = old.node(oid);
let r = removed_under.entry(n).or_default();
r[0] = sat_add(r[0], on.tot_alloc);
r[1] = sat_add(r[1], on.tot_size);
}
}
removed.sort_unstable();
Diff { old, old_of, removed, removed_under }
}
/// Classifies a node.
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
pub enum Status {
Unchanged,
Added,
Removed,
Grew,
Shrank,
}
impl Status {
pub fn name(self) -> &'static str {
["unchanged", "new", "removed", "increased", "decreased"][self as usize]
}
}
impl fmt::Display for Status {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(self.name())
}
}
/// One row in a change report.
#[derive(Clone, Debug)]
pub struct Change {
pub path: String,
pub status: Status,
pub delta: i64,
pub old: i64,
pub new: i64,
pub node: NodeId, // in the new tree (NO_NODE for removed)
}
impl Diff {
/// The matching old size of a new node, if it existed.
pub fn old_size(&self, id: NodeId, m: SizeMode) -> Option<i64> {
let o = self.old_of[id as usize];
(o != NO_NODE).then(|| self.old.node(o).total(m))
}
/// New minus old size for a new node (full size if new).
pub fn delta(&self, new: &Tree, id: NodeId, m: SizeMode) -> i64 {
new.node(id).total(m) - self.old_size(id, m).unwrap_or(0)
}
/// Classifies a new node.
pub fn status_of(&self, new: &Tree, id: NodeId, m: SizeMode) -> Status {
if self.old_of[id as usize] == NO_NODE {
return Status::Added;
}
match self.delta(new, id, m) {
d if d > 0 => Status::Grew,
d if d < 0 => Status::Shrank,
_ => Status::Unchanged,
}
}
/// The k most significant changes (all if k is 0). A directory is listed
/// only if its change is not almost entirely explained by a single
/// child, so the report points at /var/lib/postgresql rather than /,
/// /var and /var/lib.
pub fn changes(&self, new: &Tree, k: usize, m: SizeMode) -> Vec<Change> {
let mut out = Vec::new();
for id in 0..new.len() as NodeId {
let delta = self.delta(new, id, m);
if delta == 0 {
continue;
}
let n = new.node(id);
if self.old_of[id as usize] != NO_NODE && n.is_dir() && self.explained_by_child(new, id, delta, m) {
continue;
}
if n.parent != NO_NODE && self.old_of[n.parent as usize] == NO_NODE {
continue; // inside a wholly new directory; the directory is listed
}
out.push(Change {
path: new.path_string(id),
status: self.status_of(new, id, m),
delta,
old: self.old_size(id, m).unwrap_or(0),
new: n.total(m),
node: id,
});
}
for &oid in &self.removed {
let o = self.old.node(oid).total(m);
out.push(Change { path: self.old.path_string(oid), status: Status::Removed, delta: -o, old: o, new: 0, node: NO_NODE });
}
out.sort_by(|a, b| b.delta.unsigned_abs().cmp(&a.delta.unsigned_abs()).then_with(|| a.path.cmp(&b.path)));
if k > 0 {
out.truncate(k);
}
out
}
// Products are widened to i128 so saturated (untrusted) totals cannot wrap.
fn explained_by_child(&self, new: &Tree, id: NodeId, delta: i64, m: SizeMode) -> bool {
let whole = delta.unsigned_abs() as i128 * 8;
let by_child = new.children(id).any(|(c, _)| {
let cd = self.delta(new, c, m);
(cd > 0) == (delta > 0) && cd.unsigned_abs() as i128 * 10 >= whole
});
by_child
|| match self.removed_under.get(&id) {
Some(r) => {
let v = if m == SizeMode::Logical { r[1] } else { r[0] };
delta < 0 && v as i128 * 10 >= whole
}
None => false,
}
}
}
rust/src/snapshot/mod.rs
+989
View File
@@ -0,0 +1,989 @@
//! The versioned, checksummed binary inventory format and snapshot
//! comparison.
//!
//! Layout (all integers little-endian / varint):
//!
//! ```text
//! magic 8 bytes brand::SNAPSHOT_MAGIC
//! version uint16 currently 1
//! flags uint16 bit 0: body is gzip-compressed (always set by save)
//! body gzip stream of:
//! meta uvarint length + JSON (root, timestamps, counters, options)
//! exts uvarint count, then count × (uvarint length + bytes)
//! nodes uvarint count, then per node:
//! uvarint parentDelta (id-parent; 0 only for the root)
//! byte kind, byte category, uvarint flags
//! uvarint nameLen + name
//! varint size, varint alloc, varint mtime
//! uvarint mode, uid, gid, nlink, ext
//! errors uvarint count, then per record: uvarint node, byte kind,
//! uvarint len + name, uvarint len + message
//! sha256 32 bytes over the uncompressed body
//! ```
//!
//! Varints are Go's encoding/binary (zig-zag for signed). Nodes are written
//! in ID order; a parent always precedes its children, so a reader can
//! rebuild the tree in one pass and malformed input cannot create cycles.
//! Aggregates are recomputed on load, never trusted.
mod compare;
pub use compare::*;
use crate::brand;
use crate::inventory::{Category, ErrKind, ErrorRecord, Kind, NodeId, Tree, MAX_ERROR_RECORDS, NUM_CATEGORIES};
use crossbeam_channel::Receiver;
use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::fs::{self, File, OpenOptions};
use std::io::{self, BufReader, BufWriter, Read, Write};
use std::os::unix::fs::OpenOptionsExt;
use std::path::{Path, PathBuf};
use std::time::{Duration, SystemTime, UNIX_EPOCH};
/// The current format version.
pub const VERSION: u16 = 1;
// Limits applied when reading untrusted snapshots.
pub const MAX_NODES: u64 = (1 << 31) - 1;
pub const MAX_DEPTH: u16 = 4096; // far beyond any real path; bounds recursion in exporters
pub const MAX_SIZE: i64 = 1 << 60; // per-node size cap (1 EiB); keeps aggregates from overflowing
/// Caps the decompressed body for load. load_file uses a limit proportional
/// to the file size instead (see body_limit).
pub const DEFAULT_MAX_BODY: i64 = 4 << 30;
pub const MAX_NAME_LEN: u64 = 1 << 16;
pub const MAX_META_LEN: u64 = 16 << 20;
pub const MAX_EXT_LEN: u64 = 64;
pub const MAX_MSG_LEN: u64 = 4096;
const MAX_EXTS: u64 = (1 << 16) - 1;
const FLAG_GZIPPED: u16 = 1;
#[derive(Serialize, Deserialize, Default)]
#[serde(default)]
struct Meta {
root: String,
created: GoTime,
start: GoTime,
end: GoTime,
complete: bool,
cancelled: bool,
files: i64,
dirs: i64,
symlinks: i64,
others: i64,
err_counts: Option<Vec<i64>>,
excluded: i64,
skipped_mounts: i64,
virtual_skipped: i64,
loops_skipped: i64,
broken_links: i64,
hardlink_dups: i64,
unscanned: i64,
excludes: Option<Vec<String>>,
one_file_system: bool,
follow: String,
workers: i64,
host: String,
generator: String,
}
/// Reports whether path starts with the snapshot magic.
pub fn is_snapshot(path: &Path) -> bool {
let mut b = [0u8; 8];
File::open(path).and_then(|mut f| f.read_exact(&mut b)).is_ok() && &b == brand::SNAPSHOT_MAGIC
}
/// Accumulates the body; flushes it through the hash and gzip in chunks.
struct Writer<W: Write> {
gz: GzEncoder<W>,
h: Sha256,
buf: Vec<u8>,
}
impl<W: Write> Writer<W> {
fn bytes(&mut self, b: &[u8]) -> io::Result<()> {
self.buf.extend_from_slice(b);
if self.buf.len() >= 256 << 10 {
self.flush()?;
}
Ok(())
}
fn flush(&mut self) -> io::Result<()> {
self.h.update(&self.buf);
self.gz.write_all(&self.buf)?;
self.buf.clear();
Ok(())
}
fn uvarint(&mut self, mut v: u64) -> io::Result<()> {
let (mut b, mut i) = ([0u8; 10], 0);
while v >= 0x80 {
b[i] = v as u8 | 0x80;
v >>= 7;
i += 1;
}
b[i] = v as u8;
self.bytes(&b[..=i])
}
fn varint(&mut self, v: i64) -> io::Result<()> {
self.uvarint(((v << 1) ^ (v >> 63)) as u64) // zig-zag, as encoding/binary.PutVarint
}
fn str(&mut self, s: &[u8]) -> io::Result<()> {
self.uvarint(s.len() as u64)?;
self.bytes(s)
}
}
/// Writes t to out. The caller holds t's read lock.
pub fn save(out: &mut dyn Write, t: &Tree) -> io::Result<()> {
let mut bw = BufWriter::with_capacity(1 << 20, out);
bw.write_all(brand::SNAPSHOT_MAGIC)?;
bw.write_all(&VERSION.to_le_bytes())?;
bw.write_all(&FLAG_GZIPPED.to_le_bytes())?;
let mut w = Writer { gz: GzEncoder::new(bw, Compression::fast()), h: Sha256::new(), buf: Vec::with_capacity(256 << 10) };
let st = &t.stats;
let m = Meta {
root: String::from_utf8_lossy(&st.root).into_owned(),
created: GoTime::from_system(SystemTime::now()),
start: GoTime::from_system(st.start).local(),
end: st.end.map(|e| GoTime::from_system(e).local()).unwrap_or_default(),
complete: st.complete,
cancelled: st.cancelled,
files: st.files,
dirs: st.dirs,
symlinks: st.symlinks,
others: st.others,
err_counts: Some(st.err_counts.to_vec()),
excluded: st.excluded,
skipped_mounts: st.skipped_mounts,
virtual_skipped: st.virtual_skipped,
loops_skipped: st.loops_skipped,
broken_links: st.broken_links,
hardlink_dups: st.hardlink_dups,
unscanned: st.unscanned,
excludes: Some(st.excludes.clone()),
one_file_system: st.one_file_system,
follow: st.follow.clone(),
workers: st.workers as i64,
host: hostname(),
generator: format!("{} {}", brand::NAME, brand::VERSION),
};
let mb = serde_json::to_vec(&m)?;
w.uvarint(mb.len() as u64)?;
w.bytes(&mb)?;
w.uvarint(t.ext_count() as u64)?;
for i in 0..t.ext_count() {
w.str(t.ext_by_index(i).as_bytes())?;
}
w.uvarint(t.len() as u64)?;
for id in 0..t.len() as NodeId {
let n = t.node(id);
w.uvarint(if id == 0 { 0 } else { (id - n.parent) as u64 })?;
w.bytes(&[n.kind as u8, n.cat as u8])?;
w.uvarint(n.flags as u64)?;
w.str(&n.name)?;
w.varint(n.size)?;
w.varint(n.alloc)?;
w.varint(n.mtime)?;
for v in [n.mode, n.uid, n.gid, n.nlink, n.ext as u32] {
w.uvarint(v as u64)?;
}
}
w.uvarint(st.errors.len() as u64)?;
for e in &st.errors {
w.uvarint(e.node as u64)?;
w.bytes(&[e.kind as u8])?;
w.str(&e.name)?;
w.str(e.msg.as_bytes())?;
}
w.flush()?;
let mut bw = w.gz.finish()?;
bw.write_all(&w.h.finalize())?;
bw.flush()
}
/// Writes a snapshot atomically (temp file + rename).
pub fn save_file(path: &Path, t: &Tree) -> io::Result<()> {
let dir = match path.parent() {
Some(d) if !d.as_os_str().is_empty() => d,
_ => Path::new("."),
};
let (tmp, mut f) = create_temp(dir)?;
let res = save(&mut f, t).and_then(|_| {
drop(f);
fs::rename(&tmp, path)
});
if res.is_err() {
let _ = fs::remove_file(&tmp);
}
res
}
/// Creates a new 0600 file named .mapsize-*.tmp in dir (like os.CreateTemp).
fn create_temp(dir: &Path) -> io::Result<(PathBuf, File)> {
let seed = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().subsec_nanos() ^ std::process::id();
for i in 0..10000u32 {
let p = dir.join(format!(".mapsize-{}.tmp", seed.wrapping_add(i.wrapping_mul(7919))));
match OpenOptions::new().write(true).create_new(true).mode(0o600).open(&p) {
Ok(f) => return Ok((p, f)),
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
Err(e) => return Err(e),
}
}
Err(io::Error::new(io::ErrorKind::AlreadyExists, "snapshot: cannot create temporary file"))
}
fn hostname() -> String {
let mut b = [0u8; 256];
if unsafe { libc::gethostname(b.as_mut_ptr() as *mut libc::c_char, b.len()) } != 0 {
return String::new();
}
let n = b.iter().position(|&c| c == 0).unwrap_or(b.len());
String::from_utf8_lossy(&b[..n]).into_owned()
}
/// The decompressed body, fed in chunks by the decompressing thread.
struct Body {
rx: Receiver<Result<Vec<u8>, String>>,
buf: Vec<u8>,
pos: usize,
err: Option<String>,
}
const EOF: &str = "EOF";
const UNEXPECTED_EOF: &str = "unexpected EOF";
impl Body {
fn new(rx: Receiver<Result<Vec<u8>, String>>) -> Body {
Body { rx, buf: Vec::new(), pos: 0, err: None }
}
/// Ensures unread bytes are buffered; false at end of stream.
fn fill(&mut self) -> Result<bool, String> {
while self.pos == self.buf.len() {
if let Some(e) = &self.err {
return Err(e.clone());
}
match self.rx.recv() {
Ok(Ok(b)) => (self.buf, self.pos) = (b, 0),
Ok(Err(e)) => self.err = Some(e),
Err(_) => return Ok(false),
}
}
Ok(true)
}
/// The next byte; Err("EOF") at end of stream.
#[inline]
fn byte(&mut self) -> Result<u8, String> {
if self.pos == self.buf.len() && !self.fill()? {
return Err(EOF.into());
}
self.pos += 1;
Ok(self.buf[self.pos - 1])
}
/// Fills out completely (errors like io.ReadFull).
fn read_full(&mut self, out: &mut [u8]) -> Result<(), String> {
let mut done = 0;
while done < out.len() {
if !self.fill()? {
return Err(if done == 0 { EOF } else { UNEXPECTED_EOF }.into());
}
let k = (out.len() - done).min(self.buf.len() - self.pos);
out[done..done + k].copy_from_slice(&self.buf[self.pos..self.pos + k]);
self.pos += k;
done += k;
}
Ok(())
}
fn byte1(&mut self) -> Result<u8, String> {
self.byte().map_err(|e| format!("snapshot: truncated: {e}"))
}
fn uvarint(&mut self, max: u64, what: &str) -> Result<u64, String> {
let v = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?;
if v > max {
return Err(format!("snapshot: {what} {v} exceeds limit {max}"));
}
Ok(v)
}
fn varint(&mut self, what: &str) -> Result<i64, String> {
let ux = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?;
let v = (ux >> 1) as i64; // zig-zag, as encoding/binary.PutVarint
Ok(if ux & 1 != 0 { !v } else { v })
}
fn bytes_n(&mut self, n: u64) -> Result<Vec<u8>, String> {
let mut b = vec![0u8; n as usize];
self.read_full(&mut b).map_err(|e| format!("snapshot: truncated: {e}"))?;
Ok(b)
}
fn str(&mut self, max: u64, what: &str) -> Result<Vec<u8>, String> {
let n = self.uvarint(max, &format!("{what} length"))?;
self.bytes_n(n)
}
}
/// encoding/binary.ReadUvarint over the body.
fn read_uvarint(b: &mut Body) -> Result<u64, String> {
let (mut x, mut s) = (0u64, 0u32);
for i in 0..10 {
let c = match b.byte() {
Ok(c) => c,
Err(e) if i > 0 && e == EOF => return Err(UNEXPECTED_EOF.into()),
Err(e) => return Err(e),
};
if c < 0x80 {
if i == 9 && c > 1 {
break;
}
return Ok(x | (c as u64) << s);
}
x |= ((c & 0x7f) as u64) << s;
s += 7;
}
Err("varint overflows 64 bits".into())
}
const BOMB: &str = "snapshot: decompressed size exceeds limit (possible decompression bomb)";
/// Allows decompressed bodies up to 64× the compressed file size (real
/// snapshots compress ~6×), with a 64 MiB floor for tiny files.
fn body_limit(file_size: i64) -> i64 {
(64 << 20).max(file_size.saturating_mul(64))
}
/// Caps the node count for a file of file_size bytes. Real snapshots spend
/// ~15 compressed bytes per node; a crafted one describes a node in a
/// fraction of a byte, so without this a 1 MB file can demand gigabytes of
/// memory while staying within body_limit.
fn node_limit(file_size: i64) -> u64 {
(1u64 << 20).max(file_size.max(0) as u64)
}
/// Reads and validates a snapshot, allowing up to DEFAULT_MAX_BODY bytes of
/// decompressed data.
pub fn load(r: impl Read) -> Result<Tree, crate::Error> {
load_limit(r, DEFAULT_MAX_BODY)
}
/// load with an explicit decompressed-size limit.
pub fn load_limit(r: impl Read, max_body: i64) -> Result<Tree, crate::Error> {
load_inner(r, max_body, MAX_NODES).map_err(Into::into)
}
/// Opens and loads a snapshot file.
pub fn load_file(path: &Path) -> Result<Tree, crate::Error> {
let f = File::open(path)?;
let (limit, nodes) = match f.metadata() {
Ok(m) => (body_limit(m.len() as i64), node_limit(m.len() as i64)),
Err(_) => (DEFAULT_MAX_BODY, MAX_NODES),
};
load_inner(f, limit, nodes).map_err(Into::into)
}
fn load_inner(input: impl Read, max_body: i64, max_nodes: u64) -> Result<Tree, String> {
let mut br = BufReader::with_capacity(1 << 20, input);
let mut hdr = [0u8; 12];
br.read_exact(&mut hdr).map_err(|_| "snapshot: file too short".to_string())?;
if &hdr[..8] != brand::SNAPSHOT_MAGIC {
return Err(format!("snapshot: not a {} snapshot", brand::NAME));
}
let v = u16::from_le_bytes([hdr[8], hdr[9]]);
if v != VERSION {
return Err(format!("snapshot: unsupported version {v} (this build reads {VERSION})"));
}
if u16::from_le_bytes([hdr[10], hdr[11]]) & FLAG_GZIPPED == 0 {
return Err("snapshot: uncompressed bodies are not supported".into());
}
// Decompression and hashing run on this thread while a scoped thread
// parses, overlapping the two. The parser only sees end of stream once
// the channel's sender is dropped, i.e. after decompression finished.
let mut h = Sha256::new();
let res = {
let mut gz = GzDecoder::new(&mut br); // single member, like gz.Multistream(false)
std::thread::scope(|s| {
let (tx, rx) = crossbeam_channel::bounded(4);
let parser = s.spawn(move || parse(&mut Body::new(rx), max_nodes));
let mut left = max_body;
loop {
if left <= 0 {
let _ = tx.send(Err(BOMB.to_string()));
break;
}
let mut buf = vec![0u8; (256 << 10).min(left as usize)];
match gz.read(&mut buf) {
Ok(0) => break,
Ok(n) => {
buf.truncate(n);
left -= n as i64;
h.update(&buf);
if tx.send(Ok(buf)).is_err() {
break; // parser bailed out
}
}
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
Err(e) => {
let _ = tx.send(Err(format!("gzip: {e}")));
break;
}
}
}
drop(tx);
parser.join().unwrap_or_else(|p| std::panic::resume_unwind(p))
})
};
let (mut t, m) = res?;
// The checksum follows the gzip stream.
let mut sum = [0u8; 32];
br.read_exact(&mut sum).map_err(|_| "snapshot: missing checksum".to_string())?;
if sum[..] != h.finalize()[..] {
return Err("snapshot: checksum mismatch (file corrupt)".into());
}
t.recompute();
let st = &mut t.stats;
st.start = m.start.to_system();
st.end = (!m.end.is_zero()).then(|| m.end.to_system());
st.complete = true;
st.cancelled = m.cancelled;
(st.files, st.dirs, st.symlinks, st.others) = (m.files, m.dirs, m.symlinks, m.others);
for (d, s) in st.err_counts.iter_mut().zip(m.err_counts.unwrap_or_default()) {
*d = s;
}
(st.excluded, st.skipped_mounts, st.virtual_skipped) = (m.excluded, m.skipped_mounts, m.virtual_skipped);
(st.loops_skipped, st.broken_links, st.hardlink_dups, st.unscanned) =
(m.loops_skipped, m.broken_links, m.hardlink_dups, m.unscanned);
st.excludes = m.excludes.unwrap_or_default();
st.one_file_system = m.one_file_system;
st.follow = m.follow;
st.workers = m.workers.max(0) as usize;
st.from_snapshot = format!("{} ({}, {})", m.created.local_minutes(), m.host, m.generator);
Ok(t)
}
/// Parses the decompressed body up to (and including) its end.
fn parse(r: &mut Body, max_nodes: u64) -> Result<(Tree, Meta), String> {
let n = r.uvarint(MAX_META_LEN, "metadata length")?;
let mb = r.bytes_n(n)?;
let m: Meta = serde_json::from_slice(&mb).map_err(|e| format!("snapshot: metadata: {e}"))?;
let mut t = Tree::new(m.root.as_bytes(), Kind::Dir);
let n_ext = r.uvarint(MAX_EXTS, "extension count")?;
let mut exts = Vec::with_capacity(n_ext.min(1024) as usize);
for _ in 0..n_ext {
exts.push(String::from_utf8_lossy(&r.str(MAX_EXT_LEN, "extension")?).into_owned());
}
let n_nodes = r.uvarint(max_nodes, "node count")?;
if n_nodes == 0 {
return Err("snapshot: no root node".into());
}
let mut depth: Vec<u16> = vec![0]; // per node, for the depth limit; parents precede children
for i in 0..n_nodes {
let pd = r.uvarint(MAX_NODES, "parent")?;
let kind = r.byte1()?;
let cat = r.byte1()?;
let flags = r.uvarint((1 << 16) - 1, "flags")? as u16;
let name = r.str(MAX_NAME_LEN, "name")?;
let (size, alloc, mtime) = (r.varint("size")?, r.varint("alloc")?, r.varint("mtime")?);
let mode = r.uvarint(u32::MAX as u64, "mode")? as u32;
let uid = r.uvarint(u32::MAX as u64, "uid")? as u32;
let gid = r.uvarint(u32::MAX as u64, "gid")? as u32;
let nlink = r.uvarint(u32::MAX as u64, "nlink")? as u32;
let ext = r.uvarint(n_ext, "ext")?;
if kind > Kind::Other as u8 || cat as usize >= NUM_CATEGORIES {
return Err(format!("snapshot: node {i}: invalid kind/category"));
}
let kind = Kind::from_u8(kind);
if size < 0 || alloc < 0 || size > MAX_SIZE || alloc > MAX_SIZE {
return Err(format!("snapshot: node {i}: size out of range"));
}
let id = if i == 0 {
if pd != 0 || kind != Kind::Dir && n_nodes > 1 {
return Err("snapshot: invalid root".into());
}
t.node_mut(0).kind = kind;
0
} else {
if pd == 0 || pd > i {
return Err(format!("snapshot: node {i}: invalid parent reference"));
}
let parent = (i - pd) as NodeId;
if t.node(parent).kind != Kind::Dir {
return Err(format!("snapshot: node {i}: parent is not a directory"));
}
if !valid_name(&name) {
return Err(format!("snapshot: node {i}: invalid name {:?}", String::from_utf8_lossy(&name)));
}
let d = depth[parent as usize] + 1;
if d > MAX_DEPTH {
return Err(format!("snapshot: node {i}: deeper than {MAX_DEPTH} levels"));
}
depth.push(d);
t.add(parent, &name, kind)
};
let n = t.node_mut(id);
n.cat = Category::from_u8(cat);
n.flags = flags;
(n.size, n.alloc, n.mtime) = (size, alloc, mtime);
(n.mode, n.uid, n.gid, n.nlink) = (mode, uid, gid, nlink);
if kind != Kind::Dir && (ext as usize) < exts.len() {
let e = t.intern_ext(&exts[ext as usize]);
t.node_mut(id).ext = e;
}
}
let n_err = r.uvarint(MAX_ERROR_RECORDS as u64, "error count")?;
for _ in 0..n_err {
let node = r.uvarint(n_nodes - 1, "error node")? as NodeId;
let kind = ErrKind::from_u8(r.byte1()?); // out-of-range kinds become Other
let name = r.str(MAX_NAME_LEN, "error name")?;
let msg = String::from_utf8_lossy(&r.str(MAX_MSG_LEN, "error message")?).into_owned();
t.stats.errors.push(ErrorRecord { node, kind, name, msg });
}
// The body must end here.
if r.byte() != Err(EOF.into()) {
return Err("snapshot: trailing data in body".into());
}
Ok((t, m))
}
fn valid_name(s: &[u8]) -> bool {
!s.is_empty() && s != b"." && s != b".." && !s.iter().any(|&c| c == b'/' || c == 0)
}
/// A Go time.Time as JSON carries it: unix seconds plus nanoseconds, and
/// the UTC offset it is written with. The default is Go's zero time
/// (0001-01-01T00:00:00Z).
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
struct GoTime {
secs: i64,
nanos: u32,
off: i32, // seconds east of UTC; presentation only
}
const GO_ZERO_SECS: i64 = -62_135_596_800;
impl Default for GoTime {
fn default() -> Self {
GoTime { secs: GO_ZERO_SECS, nanos: 0, off: 0 }
}
}
impl GoTime {
fn from_system(t: SystemTime) -> GoTime {
match t.duration_since(UNIX_EPOCH) {
Ok(d) => GoTime { secs: d.as_secs() as i64, nanos: d.subsec_nanos(), off: 0 },
Err(e) => {
let d = e.duration();
let (s, n) = (d.as_secs() as i64, d.subsec_nanos());
let (secs, nanos) = if n == 0 { (-s, 0) } else { (-s - 1, 1_000_000_000 - n) };
GoTime { secs, nanos, off: 0 }
}
}
}
/// The same instant in the local time zone (as Go's time.Now()).
fn local(self) -> GoTime {
let off = localtime(self.secs).map_or(0, |tm| tm.tm_gmtoff as i32);
GoTime { off, ..self }
}
fn is_zero(self) -> bool {
(self.secs, self.nanos) == (GO_ZERO_SECS, 0)
}
fn to_system(self) -> SystemTime {
if self.secs >= 0 {
UNIX_EPOCH + Duration::new(self.secs as u64, self.nanos)
} else {
UNIX_EPOCH - Duration::from_secs(self.secs.unsigned_abs()) + Duration::from_nanos(self.nanos as u64)
}
}
/// RFC 3339 with trimmed nanoseconds, like Go's MarshalJSON.
fn format(self) -> String {
let secs = self.secs + self.off as i64;
let (y, mo, d) = civil_from_days(secs.div_euclid(86400));
let rem = secs.rem_euclid(86400);
let mut s = format!("{y:04}-{mo:02}-{d:02}T{:02}:{:02}:{:02}", rem / 3600, rem / 60 % 60, rem % 60);
if self.nanos != 0 {
s += format!(".{:09}", self.nanos).trim_end_matches('0');
}
if self.off == 0 {
return s + "Z";
}
let a = self.off.unsigned_abs() / 60;
format!("{s}{}{:02}:{:02}", if self.off < 0 { '-' } else { '+' }, a / 60, a % 60)
}
/// Parses RFC 3339 ("2026-10-08T10:40:00.123456789+01:00").
fn parse(s: &str) -> Option<GoTime> {
let b = s.as_bytes();
let num = |r: std::ops::Range<usize>| -> Option<i64> {
let p = b.get(r)?;
p.iter().all(u8::is_ascii_digit).then(|| p.iter().fold(0i64, |a, &c| a * 10 + (c - b'0') as i64))
};
if b.len() < 20 || b[4] != b'-' || b[7] != b'-' || b[10] != b'T' || b[13] != b':' || b[16] != b':' {
return None;
}
let (y, mo, d, h, mi, se) = (num(0..4)?, num(5..7)?, num(8..10)?, num(11..13)?, num(14..16)?, num(17..19)?);
if !(1..=12).contains(&mo) || !(1..=31).contains(&d) || h > 23 || mi > 59 || se > 59 {
return None;
}
let mut i = 19;
let mut nanos = 0u32;
if b[i] == b'.' {
let start = i + 1;
i = start;
while i < b.len() && b[i].is_ascii_digit() {
if i - start < 9 {
nanos = nanos * 10 + (b[i] - b'0') as u32;
}
i += 1;
}
if i == start {
return None;
}
for _ in (i - start)..9 {
nanos *= 10;
}
}
let off = match b.get(i)? {
b'Z' if b.len() == i + 1 => 0,
&c @ (b'+' | b'-') if b.len() == i + 6 && b[i + 3] == b':' => {
let (oh, om) = (num(i + 1..i + 3)?, num(i + 4..i + 6)?);
if oh > 23 || om > 59 {
return None;
}
(oh * 3600 + om * 60) * if c == b'-' { -1 } else { 1 }
}
_ => return None,
};
let secs = days_from_civil(y, mo, d) * 86400 + h * 3600 + mi * 60 + se - off;
Some(GoTime { secs, nanos, off: off as i32 })
}
/// "2006-01-02 15:04" in local time.
fn local_minutes(self) -> String {
let Some(tm) = localtime(self.secs) else { return self.format() };
format!("{:04}-{:02}-{:02} {:02}:{:02}", tm.tm_year as i64 + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min)
}
}
fn localtime(secs: i64) -> Option<libc::tm> {
let t = secs as libc::time_t;
let mut tm: libc::tm = unsafe { std::mem::zeroed() };
(!unsafe { libc::localtime_r(&t, &mut tm) }.is_null()).then_some(tm)
}
impl Serialize for GoTime {
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
s.serialize_str(&self.format())
}
}
impl<'de> Deserialize<'de> for GoTime {
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
// null leaves the zero time, as in Go.
match Option::<String>::deserialize(d)? {
None => Ok(GoTime::default()),
Some(s) => GoTime::parse(&s).ok_or_else(|| serde::de::Error::custom(format!("parsing time {s:?}"))),
}
}
}
/// Civil-from-days (Howard Hinnant).
fn civil_from_days(days: i64) -> (i64, i64, i64) {
let z = days + 719468;
let era = z.div_euclid(146097);
let doe = z - era * 146097;
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
let mp = (5 * doy + 2) / 153;
let d = doy - (153 * mp + 2) / 5 + 1;
let m = if mp < 10 { mp + 3 } else { mp - 9 };
(yoe + era * 400 + (m <= 2) as i64, m, d)
}
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
let y = if m <= 2 { y - 1 } else { y };
let era = y.div_euclid(400);
let yoe = y - era * 400;
let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1;
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
era * 146097 + doe - 719468
}
#[cfg(test)]
mod tests {
use super::*;
use crate::inventory::{SizeMode, FLAG_SPARSE};
use std::collections::HashMap;
fn sample() -> Tree {
let mut t = Tree::new(b"/srv", Kind::Dir);
let a = t.add(0, b"a", Kind::Dir);
let b = t.add(0, b"b", Kind::Dir);
let f1 = t.add(a, b"x.iso", Kind::File);
(t.node_mut(f1).size, t.node_mut(f1).alloc) = (1000, 1024);
let f2 = t.add(b, b"y.log", Kind::File);
(t.node_mut(f2).size, t.node_mut(f2).alloc) = (50, 4096);
t.node_mut(f2).flags = FLAG_SPARSE;
t.recompute();
(t.stats.files, t.stats.dirs) = (2, 3);
t.stats.add_error(ErrorRecord { node: b, kind: ErrKind::Permission, msg: "denied".into(), ..Default::default() });
t
}
fn saved(t: &Tree) -> Vec<u8> {
let mut buf = Vec::new();
save(&mut buf, t).unwrap();
buf
}
#[test]
fn round_trip() {
let src = sample();
let got = load(&saved(&src)[..]).unwrap();
assert_eq!(got.len(), src.len());
for i in 0..src.len() as NodeId {
let (a, b) = (src.node(i), got.node(i));
assert!(
a.name == b.name && a.tot_size == b.tot_size && a.tot_alloc == b.tot_alloc && a.flags == b.flags
&& src.path(i) == got.path(i) && src.ext_name(a) == got.ext_name(b),
"node {i} differs: {a:?} vs {b:?}"
);
}
assert_eq!(got.stats.err_counts[ErrKind::Permission as usize], 1);
assert_eq!(got.stats.errors.len(), 1);
assert_eq!(got.stats.start, src.stats.start);
assert_eq!(got.stats.end, None);
}
#[test]
fn corruption_detected() {
let data = saved(&sample());
let mut bad = data.clone();
*bad.last_mut().unwrap() ^= 0xff; // flip a byte in the checksum trailer
assert!(load(&bad[..]).is_err(), "checksum corruption not detected");
// Every truncation must fail cleanly.
for i in (0..data.len() - 1).step_by(7) {
assert!(load(&data[..i]).is_err(), "truncation at {i} accepted");
}
}
#[test]
fn compare_trees() {
let old = sample();
let mut nw = Tree::new(b"/srv", Kind::Dir);
let a = nw.add(0, b"a", Kind::Dir);
let f1 = nw.add(a, b"x.iso", Kind::File);
(nw.node_mut(f1).size, nw.node_mut(f1).alloc) = (5000, 5120);
let c = nw.add(0, b"c", Kind::Dir);
let f3 = nw.add(c, b"z", Kind::File);
(nw.node_mut(f3).size, nw.node_mut(f3).alloc) = (10, 4096);
nw.recompute();
let d = compare(old, &nw);
let al = SizeMode::Allocated;
assert_eq!(d.status_of(&nw, f1, al), Status::Grew);
assert_eq!(d.delta(&nw, f1, al), 4096);
assert_eq!(d.status_of(&nw, c, al), Status::Added);
assert_eq!(d.removed.len(), 1);
assert_eq!(&d.old.node(d.removed[0]).name[..], b"b");
let ch = d.changes(&nw, 10, al);
let paths: HashMap<&str, Status> = ch.iter().map(|c| (c.path.as_str(), c.status)).collect();
assert_eq!(paths.get("/srv/a/x.iso"), Some(&Status::Grew), "{ch:?}");
assert_eq!(paths.get("/srv/b"), Some(&Status::Removed), "{ch:?}");
assert_eq!(paths.get("/srv/c"), Some(&Status::Added), "{ch:?}");
assert!(!paths.contains_key("/srv/a"), "/srv/a should be explained by its child");
assert!(!paths.contains_key("/srv/c/z"), "contents of new dirs should not be listed separately");
}
fn body(b: &[u8]) -> Body {
let (tx, rx) = crossbeam_channel::unbounded();
tx.send(Ok(b.to_vec())).unwrap();
Body::new(rx)
}
fn put_uvarint(v: u64) -> Vec<u8> {
let mut w = Writer { gz: GzEncoder::new(Vec::new(), Compression::fast()), h: Sha256::new(), buf: Vec::new() };
w.uvarint(v).unwrap();
w.buf
}
#[test]
fn read_uvarint_matches_go() {
for v in [0, 1, 127, 128, 300, 1 << 35, u64::MAX] {
assert_eq!(read_uvarint(&mut body(&put_uvarint(v))), Ok(v));
}
assert_eq!(put_uvarint(300), [0xac, 0x02]);
assert!(read_uvarint(&mut body(&[0xff; 11])).is_err(), "overflow not detected");
// Zig-zag as Go's PutVarint.
for (v, enc) in [(0i64, 0u64), (-1, 1), (1, 2), (-2, 3), (i64::MIN, u64::MAX)] {
assert_eq!(((v << 1) ^ (v >> 63)) as u64, enc);
assert_eq!(body(&put_uvarint(enc)).varint("v"), Ok(v));
}
}
#[test]
fn go_time_json() {
let t = GoTime::parse("2026-10-08T10:40:00.123456789+01:00").unwrap();
assert_eq!((t.secs, t.nanos, t.off), (1_791_452_400, 123_456_789, 3600));
assert_eq!(t.format(), "2026-10-08T10:40:00.123456789+01:00");
assert_eq!(GoTime { off: 0, ..t }.format(), "2026-10-08T09:40:00.123456789Z");
assert_eq!(GoTime { off: -5 * 3600 - 1800, ..t }.format(), "2026-10-08T04:10:00.123456789-05:30");
assert_eq!(GoTime::parse("2026-10-08T09:40:00.1Z").unwrap().format(), "2026-10-08T09:40:00.1Z");
assert_eq!(GoTime::parse("0001-01-01T00:00:00Z"), Some(GoTime::default()));
assert_eq!(GoTime::default().format(), "0001-01-01T00:00:00Z");
assert_eq!(GoTime::from_system(GoTime::default().to_system()), GoTime::default());
for bad in ["", "2026-10-08", "2026-13-08T00:00:00Z", "2026-10-08T00:00:00", "2026-10-08T00:00:00.Z"] {
assert_eq!(GoTime::parse(bad), None, "{bad}");
}
}
/// A tiny tree saved by the Go binary
/// (tree/{a/x.iso 10000 B, a/b/note.txt, c/y.log 3000 B, c/link -> ../a}).
#[test]
fn loads_go_snapshot() {
let t = load_file(Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/src/snapshot/testdata/go.msz"))).unwrap();
let r = t.node(0);
assert_eq!((r.tot_size, r.tot_alloc), (13310, 20480));
assert_eq!((t.stats.files, t.stats.dirs, t.stats.symlinks), (3, 4, 1));
assert_eq!(t.len(), 8);
assert!(t.stats.root.ends_with(b"/tree"));
assert!(t.stats.end.is_some() && t.stats.complete);
assert!(t.stats.from_snapshot.contains("mapsize"), "{}", t.stats.from_snapshot);
let iso = (0..t.len() as NodeId).find(|&i| &t.node(i).name[..] == b"x.iso").unwrap();
assert_eq!(t.node(iso).size, 10000);
assert_eq!(t.ext_name(t.node(iso)), "iso");
assert!(t.path_string(iso).ends_with("/tree/a/x.iso"));
// And it survives a Rust round trip unchanged.
let again = load(&saved(&t)[..]).unwrap();
assert_eq!((again.node(0).tot_size, again.len(), again.stats.start), (13310, 8, t.stats.start));
}
// Hostile input.
/// Builds a well-formed (correctly checksummed) but hostile snapshot.
fn craft(n: usize, meta: &str, rec: impl Fn(usize, &mut Writer<Vec<u8>>)) -> Vec<u8> {
let mut out = Vec::new();
out.extend_from_slice(brand::SNAPSHOT_MAGIC);
out.extend_from_slice(&VERSION.to_le_bytes());
out.extend_from_slice(&FLAG_GZIPPED.to_le_bytes());
let mut w = Writer { gz: GzEncoder::new(out, Compression::best()), h: Sha256::new(), buf: Vec::new() };
w.uvarint(meta.len() as u64).unwrap();
w.bytes(meta.as_bytes()).unwrap();
w.uvarint(1).unwrap();
w.str(b"").unwrap();
w.uvarint(n as u64).unwrap();
for i in 0..n {
rec(i, &mut w);
}
w.uvarint(0).unwrap();
w.flush().unwrap();
let mut out = w.gz.finish().unwrap();
let sum = w.h.finalize();
out.extend_from_slice(&sum);
out
}
fn node(w: &mut Writer<Vec<u8>>, pd: u64, kind: Kind, name: &str, size: i64) {
w.uvarint(pd).unwrap();
w.bytes(&[kind as u8, 0]).unwrap();
w.uvarint(0).unwrap();
w.str(name.as_bytes()).unwrap();
w.varint(size).unwrap();
w.varint(size).unwrap();
w.varint(0).unwrap();
for _ in 0..5 {
w.uvarint(0).unwrap();
}
}
fn chain(n: usize) -> Vec<u8> {
craft(n, r#"{"root":"/r"}"#, |i, w| if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::Dir, "a", 0) })
}
#[test]
fn deep_chain_rejected() {
load(&chain(MAX_DEPTH as usize + 1)[..]).expect("depth at the limit must load");
let err = load(&chain(MAX_DEPTH as usize + 2)[..]).err().expect("over-deep chain accepted");
assert!(err.to_string().contains("deeper"), "{err}");
}
#[test]
fn decompression_bomb_limited() {
let flat = craft(100_000, r#"{"root":"/r"}"#, |i, w| {
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", 0) }
});
let err = load_limit(&flat[..], 64 << 10).err().expect("bomb not limited");
assert!(err.to_string().contains("limit"), "{err}");
assert!(body_limit(flat.len() as i64) >= 64 << 20, "body_limit floor");
}
#[test]
fn huge_sizes_do_not_wrap() {
let data = craft(3, r#"{"root":"/r"}"#, |i, w| {
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", MAX_SIZE) }
});
let tr = load(&data[..]).unwrap();
assert!(tr.node(0).tot_size >= 0, "aggregate wrapped negative");
let over = craft(2, r#"{"root":"/r"}"#, |i, w| {
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::File, "f", MAX_SIZE + 1) }
});
assert!(load(&over[..]).is_err(), "size above MAX_SIZE accepted");
}
#[test]
fn hostile_metadata_sanitised_in_summary() {
let data = craft(1, r#"{"root":"/r\u001b[2J","excludes":["\u001b]0;PWNED\u0007"],"excluded":1}"#, |_, w| {
node(w, 0, Kind::Dir, "", 0)
});
let mut tr = load(&data[..]).unwrap();
tr.stats.excluded = 1;
let mut b = Vec::new();
crate::export::summary(&mut b, &tr).unwrap();
assert!(!b.iter().any(|&c| c == 0x1b || c == 0x07), "raw control characters in summary: {:?}", String::from_utf8_lossy(&b));
}
/// Blocks of one directory plus K-1 files repeat byte for byte, so a
/// small file describes millions of nodes; load_file must refuse before
/// allocating.
#[test]
fn node_amplification_limited() {
const K: usize = 1000;
const N: usize = 1_200_001;
let data = craft(N, r#"{"root":"/r"}"#, |i, w| match i {
0 => node(w, 0, Kind::Dir, "", 0),
1 => node(w, 1, Kind::Dir, "d", 0),
_ if (i - 1) % K == 0 => node(w, K as u64, Kind::Dir, "d", 0),
_ => node(w, ((i - 1) % K) as u64, Kind::File, "f", 0),
});
load(&data[..]).expect("crafted tree is otherwise valid");
let p = std::env::temp_dir().join(format!("mapsize-amp-{}.msz", std::process::id()));
fs::write(&p, &data).unwrap();
let res = load_file(&p);
let _ = fs::remove_file(&p);
let err = res.err().unwrap_or_else(|| panic!("{N} nodes from a {}-byte file accepted", data.len()));
assert!(err.to_string().contains("node count"), "{err}");
}
#[test]
fn save_file_is_atomic_and_detected() {
let dir = std::env::temp_dir().join(format!("mapsize-save-{}", std::process::id()));
fs::create_dir_all(&dir).unwrap();
let p = dir.join("s.msz");
save_file(&p, &sample()).unwrap();
assert!(is_snapshot(&p));
assert_eq!(load_file(&p).unwrap().len(), 5);
assert_eq!(fs::read_dir(&dir).unwrap().count(), 1, "temp file left behind");
fs::remove_dir_all(&dir).unwrap();
}
}
rust/src/snapshot/testdata/go.msz
Vendored
BIN
View File
Binary file not shown.