Rewrite mapsize in Rust and remove the Go implementation #1
@@ -9,6 +9,7 @@ pub mod filter;
|
||||
pub mod inventory;
|
||||
pub mod platform;
|
||||
pub mod scan;
|
||||
pub mod snapshot;
|
||||
pub mod textutil;
|
||||
pub mod treemap;
|
||||
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
use crate::inventory::{sat_add, NodeId, SizeMode, Tree, NO_NODE};
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
|
||||
/// Relates two inventories by path. It owns the old tree; methods that need
|
||||
/// the new one take it as a parameter. The new tree must not be mutated
|
||||
/// while a Diff is in use (snapshots and finished scans satisfy this).
|
||||
pub struct Diff {
|
||||
pub old: Tree,
|
||||
/// Maps a new NodeId to the matching old NodeId, or NO_NODE if the entry
|
||||
/// is new.
|
||||
pub old_of: Vec<NodeId>,
|
||||
/// The topmost old nodes with no counterpart in the new tree.
|
||||
pub removed: Vec<NodeId>,
|
||||
/// Per new directory, the sizes of removed children ([allocated,
|
||||
/// logical]) so growth colouring can account for them.
|
||||
removed_under: HashMap<NodeId, [i64; 2]>,
|
||||
}
|
||||
|
||||
/// Matches nodes by name, directory by directory, from the roots.
|
||||
pub fn compare(old: Tree, new: &Tree) -> Diff {
|
||||
let mut old_of = vec![NO_NODE; new.len()];
|
||||
let mut removed = Vec::new();
|
||||
let mut removed_under: HashMap<NodeId, [i64; 2]> = HashMap::new();
|
||||
let mut stack = vec![(old.root(), new.root())];
|
||||
while let Some((o, n)) = stack.pop() {
|
||||
old_of[n as usize] = o;
|
||||
if !old.node(o).is_dir() || !new.node(n).is_dir() {
|
||||
continue;
|
||||
}
|
||||
let mut by_name: HashMap<&[u8], NodeId> = old.children(o).map(|(id, c)| (&c.name[..], id)).collect();
|
||||
for (id, c) in new.children(n) {
|
||||
if let Some(oid) = by_name.remove(&c.name[..]) {
|
||||
stack.push((oid, id));
|
||||
}
|
||||
}
|
||||
for oid in by_name.into_values() {
|
||||
removed.push(oid);
|
||||
let on = old.node(oid);
|
||||
let r = removed_under.entry(n).or_default();
|
||||
r[0] = sat_add(r[0], on.tot_alloc);
|
||||
r[1] = sat_add(r[1], on.tot_size);
|
||||
}
|
||||
}
|
||||
removed.sort_unstable();
|
||||
Diff { old, old_of, removed, removed_under }
|
||||
}
|
||||
|
||||
/// Classifies a node.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
|
||||
pub enum Status {
|
||||
Unchanged,
|
||||
Added,
|
||||
Removed,
|
||||
Grew,
|
||||
Shrank,
|
||||
}
|
||||
|
||||
impl Status {
|
||||
pub fn name(self) -> &'static str {
|
||||
["unchanged", "new", "removed", "increased", "decreased"][self as usize]
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Status {
|
||||
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
|
||||
f.write_str(self.name())
|
||||
}
|
||||
}
|
||||
|
||||
/// One row in a change report.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Change {
|
||||
pub path: String,
|
||||
pub status: Status,
|
||||
pub delta: i64,
|
||||
pub old: i64,
|
||||
pub new: i64,
|
||||
pub node: NodeId, // in the new tree (NO_NODE for removed)
|
||||
}
|
||||
|
||||
impl Diff {
|
||||
/// The matching old size of a new node, if it existed.
|
||||
pub fn old_size(&self, id: NodeId, m: SizeMode) -> Option<i64> {
|
||||
let o = self.old_of[id as usize];
|
||||
(o != NO_NODE).then(|| self.old.node(o).total(m))
|
||||
}
|
||||
|
||||
/// New minus old size for a new node (full size if new).
|
||||
pub fn delta(&self, new: &Tree, id: NodeId, m: SizeMode) -> i64 {
|
||||
new.node(id).total(m) - self.old_size(id, m).unwrap_or(0)
|
||||
}
|
||||
|
||||
/// Classifies a new node.
|
||||
pub fn status_of(&self, new: &Tree, id: NodeId, m: SizeMode) -> Status {
|
||||
if self.old_of[id as usize] == NO_NODE {
|
||||
return Status::Added;
|
||||
}
|
||||
match self.delta(new, id, m) {
|
||||
d if d > 0 => Status::Grew,
|
||||
d if d < 0 => Status::Shrank,
|
||||
_ => Status::Unchanged,
|
||||
}
|
||||
}
|
||||
|
||||
/// The k most significant changes (all if k is 0). A directory is listed
|
||||
/// only if its change is not almost entirely explained by a single
|
||||
/// child, so the report points at /var/lib/postgresql rather than /,
|
||||
/// /var and /var/lib.
|
||||
pub fn changes(&self, new: &Tree, k: usize, m: SizeMode) -> Vec<Change> {
|
||||
let mut out = Vec::new();
|
||||
for id in 0..new.len() as NodeId {
|
||||
let delta = self.delta(new, id, m);
|
||||
if delta == 0 {
|
||||
continue;
|
||||
}
|
||||
let n = new.node(id);
|
||||
if self.old_of[id as usize] != NO_NODE && n.is_dir() && self.explained_by_child(new, id, delta, m) {
|
||||
continue;
|
||||
}
|
||||
if n.parent != NO_NODE && self.old_of[n.parent as usize] == NO_NODE {
|
||||
continue; // inside a wholly new directory; the directory is listed
|
||||
}
|
||||
out.push(Change {
|
||||
path: new.path_string(id),
|
||||
status: self.status_of(new, id, m),
|
||||
delta,
|
||||
old: self.old_size(id, m).unwrap_or(0),
|
||||
new: n.total(m),
|
||||
node: id,
|
||||
});
|
||||
}
|
||||
for &oid in &self.removed {
|
||||
let o = self.old.node(oid).total(m);
|
||||
out.push(Change { path: self.old.path_string(oid), status: Status::Removed, delta: -o, old: o, new: 0, node: NO_NODE });
|
||||
}
|
||||
out.sort_by(|a, b| b.delta.unsigned_abs().cmp(&a.delta.unsigned_abs()).then_with(|| a.path.cmp(&b.path)));
|
||||
if k > 0 {
|
||||
out.truncate(k);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
// Products are widened to i128 so saturated (untrusted) totals cannot wrap.
|
||||
fn explained_by_child(&self, new: &Tree, id: NodeId, delta: i64, m: SizeMode) -> bool {
|
||||
let whole = delta.unsigned_abs() as i128 * 8;
|
||||
let by_child = new.children(id).any(|(c, _)| {
|
||||
let cd = self.delta(new, c, m);
|
||||
(cd > 0) == (delta > 0) && cd.unsigned_abs() as i128 * 10 >= whole
|
||||
});
|
||||
by_child
|
||||
|| match self.removed_under.get(&id) {
|
||||
Some(r) => {
|
||||
let v = if m == SizeMode::Logical { r[1] } else { r[0] };
|
||||
delta < 0 && v as i128 * 10 >= whole
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,989 @@
|
||||
//! The versioned, checksummed binary inventory format and snapshot
|
||||
//! comparison.
|
||||
//!
|
||||
//! Layout (all integers little-endian / varint):
|
||||
//!
|
||||
//! ```text
|
||||
//! magic 8 bytes brand::SNAPSHOT_MAGIC
|
||||
//! version uint16 currently 1
|
||||
//! flags uint16 bit 0: body is gzip-compressed (always set by save)
|
||||
//! body gzip stream of:
|
||||
//! meta uvarint length + JSON (root, timestamps, counters, options)
|
||||
//! exts uvarint count, then count × (uvarint length + bytes)
|
||||
//! nodes uvarint count, then per node:
|
||||
//! uvarint parentDelta (id-parent; 0 only for the root)
|
||||
//! byte kind, byte category, uvarint flags
|
||||
//! uvarint nameLen + name
|
||||
//! varint size, varint alloc, varint mtime
|
||||
//! uvarint mode, uid, gid, nlink, ext
|
||||
//! errors uvarint count, then per record: uvarint node, byte kind,
|
||||
//! uvarint len + name, uvarint len + message
|
||||
//! sha256 32 bytes over the uncompressed body
|
||||
//! ```
|
||||
//!
|
||||
//! Varints are Go's encoding/binary (zig-zag for signed). Nodes are written
|
||||
//! in ID order; a parent always precedes its children, so a reader can
|
||||
//! rebuild the tree in one pass and malformed input cannot create cycles.
|
||||
//! Aggregates are recomputed on load, never trusted.
|
||||
|
||||
mod compare;
|
||||
|
||||
pub use compare::*;
|
||||
|
||||
use crate::brand;
|
||||
use crate::inventory::{Category, ErrKind, ErrorRecord, Kind, NodeId, Tree, MAX_ERROR_RECORDS, NUM_CATEGORIES};
|
||||
use crossbeam_channel::Receiver;
|
||||
use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{self, BufReader, BufWriter, Read, Write};
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
/// The current format version.
|
||||
pub const VERSION: u16 = 1;
|
||||
|
||||
// Limits applied when reading untrusted snapshots.
|
||||
pub const MAX_NODES: u64 = (1 << 31) - 1;
|
||||
pub const MAX_DEPTH: u16 = 4096; // far beyond any real path; bounds recursion in exporters
|
||||
pub const MAX_SIZE: i64 = 1 << 60; // per-node size cap (1 EiB); keeps aggregates from overflowing
|
||||
/// Caps the decompressed body for load. load_file uses a limit proportional
|
||||
/// to the file size instead (see body_limit).
|
||||
pub const DEFAULT_MAX_BODY: i64 = 4 << 30;
|
||||
pub const MAX_NAME_LEN: u64 = 1 << 16;
|
||||
pub const MAX_META_LEN: u64 = 16 << 20;
|
||||
pub const MAX_EXT_LEN: u64 = 64;
|
||||
pub const MAX_MSG_LEN: u64 = 4096;
|
||||
const MAX_EXTS: u64 = (1 << 16) - 1;
|
||||
const FLAG_GZIPPED: u16 = 1;
|
||||
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
#[serde(default)]
|
||||
struct Meta {
|
||||
root: String,
|
||||
created: GoTime,
|
||||
start: GoTime,
|
||||
end: GoTime,
|
||||
complete: bool,
|
||||
cancelled: bool,
|
||||
files: i64,
|
||||
dirs: i64,
|
||||
symlinks: i64,
|
||||
others: i64,
|
||||
err_counts: Option<Vec<i64>>,
|
||||
excluded: i64,
|
||||
skipped_mounts: i64,
|
||||
virtual_skipped: i64,
|
||||
loops_skipped: i64,
|
||||
broken_links: i64,
|
||||
hardlink_dups: i64,
|
||||
unscanned: i64,
|
||||
excludes: Option<Vec<String>>,
|
||||
one_file_system: bool,
|
||||
follow: String,
|
||||
workers: i64,
|
||||
host: String,
|
||||
generator: String,
|
||||
}
|
||||
|
||||
/// Reports whether path starts with the snapshot magic.
|
||||
pub fn is_snapshot(path: &Path) -> bool {
|
||||
let mut b = [0u8; 8];
|
||||
File::open(path).and_then(|mut f| f.read_exact(&mut b)).is_ok() && &b == brand::SNAPSHOT_MAGIC
|
||||
}
|
||||
|
||||
/// Accumulates the body; flushes it through the hash and gzip in chunks.
|
||||
struct Writer<W: Write> {
|
||||
gz: GzEncoder<W>,
|
||||
h: Sha256,
|
||||
buf: Vec<u8>,
|
||||
}
|
||||
|
||||
impl<W: Write> Writer<W> {
|
||||
fn bytes(&mut self, b: &[u8]) -> io::Result<()> {
|
||||
self.buf.extend_from_slice(b);
|
||||
if self.buf.len() >= 256 << 10 {
|
||||
self.flush()?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.h.update(&self.buf);
|
||||
self.gz.write_all(&self.buf)?;
|
||||
self.buf.clear();
|
||||
Ok(())
|
||||
}
|
||||
fn uvarint(&mut self, mut v: u64) -> io::Result<()> {
|
||||
let (mut b, mut i) = ([0u8; 10], 0);
|
||||
while v >= 0x80 {
|
||||
b[i] = v as u8 | 0x80;
|
||||
v >>= 7;
|
||||
i += 1;
|
||||
}
|
||||
b[i] = v as u8;
|
||||
self.bytes(&b[..=i])
|
||||
}
|
||||
fn varint(&mut self, v: i64) -> io::Result<()> {
|
||||
self.uvarint(((v << 1) ^ (v >> 63)) as u64) // zig-zag, as encoding/binary.PutVarint
|
||||
}
|
||||
fn str(&mut self, s: &[u8]) -> io::Result<()> {
|
||||
self.uvarint(s.len() as u64)?;
|
||||
self.bytes(s)
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes t to out. The caller holds t's read lock.
|
||||
pub fn save(out: &mut dyn Write, t: &Tree) -> io::Result<()> {
|
||||
let mut bw = BufWriter::with_capacity(1 << 20, out);
|
||||
bw.write_all(brand::SNAPSHOT_MAGIC)?;
|
||||
bw.write_all(&VERSION.to_le_bytes())?;
|
||||
bw.write_all(&FLAG_GZIPPED.to_le_bytes())?;
|
||||
let mut w = Writer { gz: GzEncoder::new(bw, Compression::fast()), h: Sha256::new(), buf: Vec::with_capacity(256 << 10) };
|
||||
|
||||
let st = &t.stats;
|
||||
let m = Meta {
|
||||
root: String::from_utf8_lossy(&st.root).into_owned(),
|
||||
created: GoTime::from_system(SystemTime::now()),
|
||||
start: GoTime::from_system(st.start).local(),
|
||||
end: st.end.map(|e| GoTime::from_system(e).local()).unwrap_or_default(),
|
||||
complete: st.complete,
|
||||
cancelled: st.cancelled,
|
||||
files: st.files,
|
||||
dirs: st.dirs,
|
||||
symlinks: st.symlinks,
|
||||
others: st.others,
|
||||
err_counts: Some(st.err_counts.to_vec()),
|
||||
excluded: st.excluded,
|
||||
skipped_mounts: st.skipped_mounts,
|
||||
virtual_skipped: st.virtual_skipped,
|
||||
loops_skipped: st.loops_skipped,
|
||||
broken_links: st.broken_links,
|
||||
hardlink_dups: st.hardlink_dups,
|
||||
unscanned: st.unscanned,
|
||||
excludes: Some(st.excludes.clone()),
|
||||
one_file_system: st.one_file_system,
|
||||
follow: st.follow.clone(),
|
||||
workers: st.workers as i64,
|
||||
host: hostname(),
|
||||
generator: format!("{} {}", brand::NAME, brand::VERSION),
|
||||
};
|
||||
let mb = serde_json::to_vec(&m)?;
|
||||
w.uvarint(mb.len() as u64)?;
|
||||
w.bytes(&mb)?;
|
||||
|
||||
w.uvarint(t.ext_count() as u64)?;
|
||||
for i in 0..t.ext_count() {
|
||||
w.str(t.ext_by_index(i).as_bytes())?;
|
||||
}
|
||||
w.uvarint(t.len() as u64)?;
|
||||
for id in 0..t.len() as NodeId {
|
||||
let n = t.node(id);
|
||||
w.uvarint(if id == 0 { 0 } else { (id - n.parent) as u64 })?;
|
||||
w.bytes(&[n.kind as u8, n.cat as u8])?;
|
||||
w.uvarint(n.flags as u64)?;
|
||||
w.str(&n.name)?;
|
||||
w.varint(n.size)?;
|
||||
w.varint(n.alloc)?;
|
||||
w.varint(n.mtime)?;
|
||||
for v in [n.mode, n.uid, n.gid, n.nlink, n.ext as u32] {
|
||||
w.uvarint(v as u64)?;
|
||||
}
|
||||
}
|
||||
w.uvarint(st.errors.len() as u64)?;
|
||||
for e in &st.errors {
|
||||
w.uvarint(e.node as u64)?;
|
||||
w.bytes(&[e.kind as u8])?;
|
||||
w.str(&e.name)?;
|
||||
w.str(e.msg.as_bytes())?;
|
||||
}
|
||||
w.flush()?;
|
||||
let mut bw = w.gz.finish()?;
|
||||
bw.write_all(&w.h.finalize())?;
|
||||
bw.flush()
|
||||
}
|
||||
|
||||
/// Writes a snapshot atomically (temp file + rename).
|
||||
pub fn save_file(path: &Path, t: &Tree) -> io::Result<()> {
|
||||
let dir = match path.parent() {
|
||||
Some(d) if !d.as_os_str().is_empty() => d,
|
||||
_ => Path::new("."),
|
||||
};
|
||||
let (tmp, mut f) = create_temp(dir)?;
|
||||
let res = save(&mut f, t).and_then(|_| {
|
||||
drop(f);
|
||||
fs::rename(&tmp, path)
|
||||
});
|
||||
if res.is_err() {
|
||||
let _ = fs::remove_file(&tmp);
|
||||
}
|
||||
res
|
||||
}
|
||||
|
||||
/// Creates a new 0600 file named .mapsize-*.tmp in dir (like os.CreateTemp).
|
||||
fn create_temp(dir: &Path) -> io::Result<(PathBuf, File)> {
|
||||
let seed = SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().subsec_nanos() ^ std::process::id();
|
||||
for i in 0..10000u32 {
|
||||
let p = dir.join(format!(".mapsize-{}.tmp", seed.wrapping_add(i.wrapping_mul(7919))));
|
||||
match OpenOptions::new().write(true).create_new(true).mode(0o600).open(&p) {
|
||||
Ok(f) => return Ok((p, f)),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Err(io::Error::new(io::ErrorKind::AlreadyExists, "snapshot: cannot create temporary file"))
|
||||
}
|
||||
|
||||
fn hostname() -> String {
|
||||
let mut b = [0u8; 256];
|
||||
if unsafe { libc::gethostname(b.as_mut_ptr() as *mut libc::c_char, b.len()) } != 0 {
|
||||
return String::new();
|
||||
}
|
||||
let n = b.iter().position(|&c| c == 0).unwrap_or(b.len());
|
||||
String::from_utf8_lossy(&b[..n]).into_owned()
|
||||
}
|
||||
|
||||
/// The decompressed body, fed in chunks by the decompressing thread.
|
||||
struct Body {
|
||||
rx: Receiver<Result<Vec<u8>, String>>,
|
||||
buf: Vec<u8>,
|
||||
pos: usize,
|
||||
err: Option<String>,
|
||||
}
|
||||
|
||||
const EOF: &str = "EOF";
|
||||
const UNEXPECTED_EOF: &str = "unexpected EOF";
|
||||
|
||||
impl Body {
|
||||
fn new(rx: Receiver<Result<Vec<u8>, String>>) -> Body {
|
||||
Body { rx, buf: Vec::new(), pos: 0, err: None }
|
||||
}
|
||||
|
||||
/// Ensures unread bytes are buffered; false at end of stream.
|
||||
fn fill(&mut self) -> Result<bool, String> {
|
||||
while self.pos == self.buf.len() {
|
||||
if let Some(e) = &self.err {
|
||||
return Err(e.clone());
|
||||
}
|
||||
match self.rx.recv() {
|
||||
Ok(Ok(b)) => (self.buf, self.pos) = (b, 0),
|
||||
Ok(Err(e)) => self.err = Some(e),
|
||||
Err(_) => return Ok(false),
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// The next byte; Err("EOF") at end of stream.
|
||||
#[inline]
|
||||
fn byte(&mut self) -> Result<u8, String> {
|
||||
if self.pos == self.buf.len() && !self.fill()? {
|
||||
return Err(EOF.into());
|
||||
}
|
||||
self.pos += 1;
|
||||
Ok(self.buf[self.pos - 1])
|
||||
}
|
||||
|
||||
/// Fills out completely (errors like io.ReadFull).
|
||||
fn read_full(&mut self, out: &mut [u8]) -> Result<(), String> {
|
||||
let mut done = 0;
|
||||
while done < out.len() {
|
||||
if !self.fill()? {
|
||||
return Err(if done == 0 { EOF } else { UNEXPECTED_EOF }.into());
|
||||
}
|
||||
let k = (out.len() - done).min(self.buf.len() - self.pos);
|
||||
out[done..done + k].copy_from_slice(&self.buf[self.pos..self.pos + k]);
|
||||
self.pos += k;
|
||||
done += k;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn byte1(&mut self) -> Result<u8, String> {
|
||||
self.byte().map_err(|e| format!("snapshot: truncated: {e}"))
|
||||
}
|
||||
|
||||
fn uvarint(&mut self, max: u64, what: &str) -> Result<u64, String> {
|
||||
let v = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?;
|
||||
if v > max {
|
||||
return Err(format!("snapshot: {what} {v} exceeds limit {max}"));
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
fn varint(&mut self, what: &str) -> Result<i64, String> {
|
||||
let ux = read_uvarint(self).map_err(|e| format!("snapshot: bad {what}: {e}"))?;
|
||||
let v = (ux >> 1) as i64; // zig-zag, as encoding/binary.PutVarint
|
||||
Ok(if ux & 1 != 0 { !v } else { v })
|
||||
}
|
||||
|
||||
fn bytes_n(&mut self, n: u64) -> Result<Vec<u8>, String> {
|
||||
let mut b = vec![0u8; n as usize];
|
||||
self.read_full(&mut b).map_err(|e| format!("snapshot: truncated: {e}"))?;
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
fn str(&mut self, max: u64, what: &str) -> Result<Vec<u8>, String> {
|
||||
let n = self.uvarint(max, &format!("{what} length"))?;
|
||||
self.bytes_n(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// encoding/binary.ReadUvarint over the body.
|
||||
fn read_uvarint(b: &mut Body) -> Result<u64, String> {
|
||||
let (mut x, mut s) = (0u64, 0u32);
|
||||
for i in 0..10 {
|
||||
let c = match b.byte() {
|
||||
Ok(c) => c,
|
||||
Err(e) if i > 0 && e == EOF => return Err(UNEXPECTED_EOF.into()),
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
if c < 0x80 {
|
||||
if i == 9 && c > 1 {
|
||||
break;
|
||||
}
|
||||
return Ok(x | (c as u64) << s);
|
||||
}
|
||||
x |= ((c & 0x7f) as u64) << s;
|
||||
s += 7;
|
||||
}
|
||||
Err("varint overflows 64 bits".into())
|
||||
}
|
||||
|
||||
const BOMB: &str = "snapshot: decompressed size exceeds limit (possible decompression bomb)";
|
||||
|
||||
/// Allows decompressed bodies up to 64× the compressed file size (real
|
||||
/// snapshots compress ~6×), with a 64 MiB floor for tiny files.
|
||||
fn body_limit(file_size: i64) -> i64 {
|
||||
(64 << 20).max(file_size.saturating_mul(64))
|
||||
}
|
||||
|
||||
/// Caps the node count for a file of file_size bytes. Real snapshots spend
|
||||
/// ~15 compressed bytes per node; a crafted one describes a node in a
|
||||
/// fraction of a byte, so without this a 1 MB file can demand gigabytes of
|
||||
/// memory while staying within body_limit.
|
||||
fn node_limit(file_size: i64) -> u64 {
|
||||
(1u64 << 20).max(file_size.max(0) as u64)
|
||||
}
|
||||
|
||||
/// Reads and validates a snapshot, allowing up to DEFAULT_MAX_BODY bytes of
|
||||
/// decompressed data.
|
||||
pub fn load(r: impl Read) -> Result<Tree, crate::Error> {
|
||||
load_limit(r, DEFAULT_MAX_BODY)
|
||||
}
|
||||
|
||||
/// load with an explicit decompressed-size limit.
|
||||
pub fn load_limit(r: impl Read, max_body: i64) -> Result<Tree, crate::Error> {
|
||||
load_inner(r, max_body, MAX_NODES).map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Opens and loads a snapshot file.
|
||||
pub fn load_file(path: &Path) -> Result<Tree, crate::Error> {
|
||||
let f = File::open(path)?;
|
||||
let (limit, nodes) = match f.metadata() {
|
||||
Ok(m) => (body_limit(m.len() as i64), node_limit(m.len() as i64)),
|
||||
Err(_) => (DEFAULT_MAX_BODY, MAX_NODES),
|
||||
};
|
||||
load_inner(f, limit, nodes).map_err(Into::into)
|
||||
}
|
||||
|
||||
fn load_inner(input: impl Read, max_body: i64, max_nodes: u64) -> Result<Tree, String> {
|
||||
let mut br = BufReader::with_capacity(1 << 20, input);
|
||||
let mut hdr = [0u8; 12];
|
||||
br.read_exact(&mut hdr).map_err(|_| "snapshot: file too short".to_string())?;
|
||||
if &hdr[..8] != brand::SNAPSHOT_MAGIC {
|
||||
return Err(format!("snapshot: not a {} snapshot", brand::NAME));
|
||||
}
|
||||
let v = u16::from_le_bytes([hdr[8], hdr[9]]);
|
||||
if v != VERSION {
|
||||
return Err(format!("snapshot: unsupported version {v} (this build reads {VERSION})"));
|
||||
}
|
||||
if u16::from_le_bytes([hdr[10], hdr[11]]) & FLAG_GZIPPED == 0 {
|
||||
return Err("snapshot: uncompressed bodies are not supported".into());
|
||||
}
|
||||
// Decompression and hashing run on this thread while a scoped thread
|
||||
// parses, overlapping the two. The parser only sees end of stream once
|
||||
// the channel's sender is dropped, i.e. after decompression finished.
|
||||
let mut h = Sha256::new();
|
||||
let res = {
|
||||
let mut gz = GzDecoder::new(&mut br); // single member, like gz.Multistream(false)
|
||||
std::thread::scope(|s| {
|
||||
let (tx, rx) = crossbeam_channel::bounded(4);
|
||||
let parser = s.spawn(move || parse(&mut Body::new(rx), max_nodes));
|
||||
let mut left = max_body;
|
||||
loop {
|
||||
if left <= 0 {
|
||||
let _ = tx.send(Err(BOMB.to_string()));
|
||||
break;
|
||||
}
|
||||
let mut buf = vec![0u8; (256 << 10).min(left as usize)];
|
||||
match gz.read(&mut buf) {
|
||||
Ok(0) => break,
|
||||
Ok(n) => {
|
||||
buf.truncate(n);
|
||||
left -= n as i64;
|
||||
h.update(&buf);
|
||||
if tx.send(Ok(buf)).is_err() {
|
||||
break; // parser bailed out
|
||||
}
|
||||
}
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(e) => {
|
||||
let _ = tx.send(Err(format!("gzip: {e}")));
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
drop(tx);
|
||||
parser.join().unwrap_or_else(|p| std::panic::resume_unwind(p))
|
||||
})
|
||||
};
|
||||
let (mut t, m) = res?;
|
||||
// The checksum follows the gzip stream.
|
||||
let mut sum = [0u8; 32];
|
||||
br.read_exact(&mut sum).map_err(|_| "snapshot: missing checksum".to_string())?;
|
||||
if sum[..] != h.finalize()[..] {
|
||||
return Err("snapshot: checksum mismatch (file corrupt)".into());
|
||||
}
|
||||
|
||||
t.recompute();
|
||||
let st = &mut t.stats;
|
||||
st.start = m.start.to_system();
|
||||
st.end = (!m.end.is_zero()).then(|| m.end.to_system());
|
||||
st.complete = true;
|
||||
st.cancelled = m.cancelled;
|
||||
(st.files, st.dirs, st.symlinks, st.others) = (m.files, m.dirs, m.symlinks, m.others);
|
||||
for (d, s) in st.err_counts.iter_mut().zip(m.err_counts.unwrap_or_default()) {
|
||||
*d = s;
|
||||
}
|
||||
(st.excluded, st.skipped_mounts, st.virtual_skipped) = (m.excluded, m.skipped_mounts, m.virtual_skipped);
|
||||
(st.loops_skipped, st.broken_links, st.hardlink_dups, st.unscanned) =
|
||||
(m.loops_skipped, m.broken_links, m.hardlink_dups, m.unscanned);
|
||||
st.excludes = m.excludes.unwrap_or_default();
|
||||
st.one_file_system = m.one_file_system;
|
||||
st.follow = m.follow;
|
||||
st.workers = m.workers.max(0) as usize;
|
||||
st.from_snapshot = format!("{} ({}, {})", m.created.local_minutes(), m.host, m.generator);
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
/// Parses the decompressed body up to (and including) its end.
|
||||
fn parse(r: &mut Body, max_nodes: u64) -> Result<(Tree, Meta), String> {
|
||||
let n = r.uvarint(MAX_META_LEN, "metadata length")?;
|
||||
let mb = r.bytes_n(n)?;
|
||||
let m: Meta = serde_json::from_slice(&mb).map_err(|e| format!("snapshot: metadata: {e}"))?;
|
||||
let mut t = Tree::new(m.root.as_bytes(), Kind::Dir);
|
||||
|
||||
let n_ext = r.uvarint(MAX_EXTS, "extension count")?;
|
||||
let mut exts = Vec::with_capacity(n_ext.min(1024) as usize);
|
||||
for _ in 0..n_ext {
|
||||
exts.push(String::from_utf8_lossy(&r.str(MAX_EXT_LEN, "extension")?).into_owned());
|
||||
}
|
||||
let n_nodes = r.uvarint(max_nodes, "node count")?;
|
||||
if n_nodes == 0 {
|
||||
return Err("snapshot: no root node".into());
|
||||
}
|
||||
let mut depth: Vec<u16> = vec![0]; // per node, for the depth limit; parents precede children
|
||||
for i in 0..n_nodes {
|
||||
let pd = r.uvarint(MAX_NODES, "parent")?;
|
||||
let kind = r.byte1()?;
|
||||
let cat = r.byte1()?;
|
||||
let flags = r.uvarint((1 << 16) - 1, "flags")? as u16;
|
||||
let name = r.str(MAX_NAME_LEN, "name")?;
|
||||
let (size, alloc, mtime) = (r.varint("size")?, r.varint("alloc")?, r.varint("mtime")?);
|
||||
let mode = r.uvarint(u32::MAX as u64, "mode")? as u32;
|
||||
let uid = r.uvarint(u32::MAX as u64, "uid")? as u32;
|
||||
let gid = r.uvarint(u32::MAX as u64, "gid")? as u32;
|
||||
let nlink = r.uvarint(u32::MAX as u64, "nlink")? as u32;
|
||||
let ext = r.uvarint(n_ext, "ext")?;
|
||||
if kind > Kind::Other as u8 || cat as usize >= NUM_CATEGORIES {
|
||||
return Err(format!("snapshot: node {i}: invalid kind/category"));
|
||||
}
|
||||
let kind = Kind::from_u8(kind);
|
||||
if size < 0 || alloc < 0 || size > MAX_SIZE || alloc > MAX_SIZE {
|
||||
return Err(format!("snapshot: node {i}: size out of range"));
|
||||
}
|
||||
let id = if i == 0 {
|
||||
if pd != 0 || kind != Kind::Dir && n_nodes > 1 {
|
||||
return Err("snapshot: invalid root".into());
|
||||
}
|
||||
t.node_mut(0).kind = kind;
|
||||
0
|
||||
} else {
|
||||
if pd == 0 || pd > i {
|
||||
return Err(format!("snapshot: node {i}: invalid parent reference"));
|
||||
}
|
||||
let parent = (i - pd) as NodeId;
|
||||
if t.node(parent).kind != Kind::Dir {
|
||||
return Err(format!("snapshot: node {i}: parent is not a directory"));
|
||||
}
|
||||
if !valid_name(&name) {
|
||||
return Err(format!("snapshot: node {i}: invalid name {:?}", String::from_utf8_lossy(&name)));
|
||||
}
|
||||
let d = depth[parent as usize] + 1;
|
||||
if d > MAX_DEPTH {
|
||||
return Err(format!("snapshot: node {i}: deeper than {MAX_DEPTH} levels"));
|
||||
}
|
||||
depth.push(d);
|
||||
t.add(parent, &name, kind)
|
||||
};
|
||||
let n = t.node_mut(id);
|
||||
n.cat = Category::from_u8(cat);
|
||||
n.flags = flags;
|
||||
(n.size, n.alloc, n.mtime) = (size, alloc, mtime);
|
||||
(n.mode, n.uid, n.gid, n.nlink) = (mode, uid, gid, nlink);
|
||||
if kind != Kind::Dir && (ext as usize) < exts.len() {
|
||||
let e = t.intern_ext(&exts[ext as usize]);
|
||||
t.node_mut(id).ext = e;
|
||||
}
|
||||
}
|
||||
let n_err = r.uvarint(MAX_ERROR_RECORDS as u64, "error count")?;
|
||||
for _ in 0..n_err {
|
||||
let node = r.uvarint(n_nodes - 1, "error node")? as NodeId;
|
||||
let kind = ErrKind::from_u8(r.byte1()?); // out-of-range kinds become Other
|
||||
let name = r.str(MAX_NAME_LEN, "error name")?;
|
||||
let msg = String::from_utf8_lossy(&r.str(MAX_MSG_LEN, "error message")?).into_owned();
|
||||
t.stats.errors.push(ErrorRecord { node, kind, name, msg });
|
||||
}
|
||||
// The body must end here.
|
||||
if r.byte() != Err(EOF.into()) {
|
||||
return Err("snapshot: trailing data in body".into());
|
||||
}
|
||||
Ok((t, m))
|
||||
}
|
||||
|
||||
fn valid_name(s: &[u8]) -> bool {
|
||||
!s.is_empty() && s != b"." && s != b".." && !s.iter().any(|&c| c == b'/' || c == 0)
|
||||
}
|
||||
|
||||
/// A Go time.Time as JSON carries it: unix seconds plus nanoseconds, and
|
||||
/// the UTC offset it is written with. The default is Go's zero time
|
||||
/// (0001-01-01T00:00:00Z).
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
struct GoTime {
|
||||
secs: i64,
|
||||
nanos: u32,
|
||||
off: i32, // seconds east of UTC; presentation only
|
||||
}
|
||||
|
||||
const GO_ZERO_SECS: i64 = -62_135_596_800;
|
||||
|
||||
impl Default for GoTime {
|
||||
fn default() -> Self {
|
||||
GoTime { secs: GO_ZERO_SECS, nanos: 0, off: 0 }
|
||||
}
|
||||
}
|
||||
|
||||
impl GoTime {
|
||||
fn from_system(t: SystemTime) -> GoTime {
|
||||
match t.duration_since(UNIX_EPOCH) {
|
||||
Ok(d) => GoTime { secs: d.as_secs() as i64, nanos: d.subsec_nanos(), off: 0 },
|
||||
Err(e) => {
|
||||
let d = e.duration();
|
||||
let (s, n) = (d.as_secs() as i64, d.subsec_nanos());
|
||||
let (secs, nanos) = if n == 0 { (-s, 0) } else { (-s - 1, 1_000_000_000 - n) };
|
||||
GoTime { secs, nanos, off: 0 }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The same instant in the local time zone (as Go's time.Now()).
|
||||
fn local(self) -> GoTime {
|
||||
let off = localtime(self.secs).map_or(0, |tm| tm.tm_gmtoff as i32);
|
||||
GoTime { off, ..self }
|
||||
}
|
||||
|
||||
fn is_zero(self) -> bool {
|
||||
(self.secs, self.nanos) == (GO_ZERO_SECS, 0)
|
||||
}
|
||||
|
||||
fn to_system(self) -> SystemTime {
|
||||
if self.secs >= 0 {
|
||||
UNIX_EPOCH + Duration::new(self.secs as u64, self.nanos)
|
||||
} else {
|
||||
UNIX_EPOCH - Duration::from_secs(self.secs.unsigned_abs()) + Duration::from_nanos(self.nanos as u64)
|
||||
}
|
||||
}
|
||||
|
||||
/// RFC 3339 with trimmed nanoseconds, like Go's MarshalJSON.
|
||||
fn format(self) -> String {
|
||||
let secs = self.secs + self.off as i64;
|
||||
let (y, mo, d) = civil_from_days(secs.div_euclid(86400));
|
||||
let rem = secs.rem_euclid(86400);
|
||||
let mut s = format!("{y:04}-{mo:02}-{d:02}T{:02}:{:02}:{:02}", rem / 3600, rem / 60 % 60, rem % 60);
|
||||
if self.nanos != 0 {
|
||||
s += format!(".{:09}", self.nanos).trim_end_matches('0');
|
||||
}
|
||||
if self.off == 0 {
|
||||
return s + "Z";
|
||||
}
|
||||
let a = self.off.unsigned_abs() / 60;
|
||||
format!("{s}{}{:02}:{:02}", if self.off < 0 { '-' } else { '+' }, a / 60, a % 60)
|
||||
}
|
||||
|
||||
/// Parses RFC 3339 ("2026-10-08T10:40:00.123456789+01:00").
|
||||
fn parse(s: &str) -> Option<GoTime> {
|
||||
let b = s.as_bytes();
|
||||
let num = |r: std::ops::Range<usize>| -> Option<i64> {
|
||||
let p = b.get(r)?;
|
||||
p.iter().all(u8::is_ascii_digit).then(|| p.iter().fold(0i64, |a, &c| a * 10 + (c - b'0') as i64))
|
||||
};
|
||||
if b.len() < 20 || b[4] != b'-' || b[7] != b'-' || b[10] != b'T' || b[13] != b':' || b[16] != b':' {
|
||||
return None;
|
||||
}
|
||||
let (y, mo, d, h, mi, se) = (num(0..4)?, num(5..7)?, num(8..10)?, num(11..13)?, num(14..16)?, num(17..19)?);
|
||||
if !(1..=12).contains(&mo) || !(1..=31).contains(&d) || h > 23 || mi > 59 || se > 59 {
|
||||
return None;
|
||||
}
|
||||
let mut i = 19;
|
||||
let mut nanos = 0u32;
|
||||
if b[i] == b'.' {
|
||||
let start = i + 1;
|
||||
i = start;
|
||||
while i < b.len() && b[i].is_ascii_digit() {
|
||||
if i - start < 9 {
|
||||
nanos = nanos * 10 + (b[i] - b'0') as u32;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if i == start {
|
||||
return None;
|
||||
}
|
||||
for _ in (i - start)..9 {
|
||||
nanos *= 10;
|
||||
}
|
||||
}
|
||||
let off = match b.get(i)? {
|
||||
b'Z' if b.len() == i + 1 => 0,
|
||||
&c @ (b'+' | b'-') if b.len() == i + 6 && b[i + 3] == b':' => {
|
||||
let (oh, om) = (num(i + 1..i + 3)?, num(i + 4..i + 6)?);
|
||||
if oh > 23 || om > 59 {
|
||||
return None;
|
||||
}
|
||||
(oh * 3600 + om * 60) * if c == b'-' { -1 } else { 1 }
|
||||
}
|
||||
_ => return None,
|
||||
};
|
||||
let secs = days_from_civil(y, mo, d) * 86400 + h * 3600 + mi * 60 + se - off;
|
||||
Some(GoTime { secs, nanos, off: off as i32 })
|
||||
}
|
||||
|
||||
/// "2006-01-02 15:04" in local time.
|
||||
fn local_minutes(self) -> String {
|
||||
let Some(tm) = localtime(self.secs) else { return self.format() };
|
||||
format!("{:04}-{:02}-{:02} {:02}:{:02}", tm.tm_year as i64 + 1900, tm.tm_mon + 1, tm.tm_mday, tm.tm_hour, tm.tm_min)
|
||||
}
|
||||
}
|
||||
|
||||
fn localtime(secs: i64) -> Option<libc::tm> {
|
||||
let t = secs as libc::time_t;
|
||||
let mut tm: libc::tm = unsafe { std::mem::zeroed() };
|
||||
(!unsafe { libc::localtime_r(&t, &mut tm) }.is_null()).then_some(tm)
|
||||
}
|
||||
|
||||
impl Serialize for GoTime {
|
||||
fn serialize<S: serde::Serializer>(&self, s: S) -> Result<S::Ok, S::Error> {
|
||||
s.serialize_str(&self.format())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for GoTime {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(d: D) -> Result<Self, D::Error> {
|
||||
// null leaves the zero time, as in Go.
|
||||
match Option::<String>::deserialize(d)? {
|
||||
None => Ok(GoTime::default()),
|
||||
Some(s) => GoTime::parse(&s).ok_or_else(|| serde::de::Error::custom(format!("parsing time {s:?}"))),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Civil-from-days (Howard Hinnant).
|
||||
fn civil_from_days(days: i64) -> (i64, i64, i64) {
|
||||
let z = days + 719468;
|
||||
let era = z.div_euclid(146097);
|
||||
let doe = z - era * 146097;
|
||||
let yoe = (doe - doe / 1460 + doe / 36524 - doe / 146096) / 365;
|
||||
let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
|
||||
let mp = (5 * doy + 2) / 153;
|
||||
let d = doy - (153 * mp + 2) / 5 + 1;
|
||||
let m = if mp < 10 { mp + 3 } else { mp - 9 };
|
||||
(yoe + era * 400 + (m <= 2) as i64, m, d)
|
||||
}
|
||||
|
||||
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
||||
let y = if m <= 2 { y - 1 } else { y };
|
||||
let era = y.div_euclid(400);
|
||||
let yoe = y - era * 400;
|
||||
let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1;
|
||||
let doe = yoe * 365 + yoe / 4 - yoe / 100 + doy;
|
||||
era * 146097 + doe - 719468
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::inventory::{SizeMode, FLAG_SPARSE};
|
||||
use std::collections::HashMap;
|
||||
|
||||
fn sample() -> Tree {
|
||||
let mut t = Tree::new(b"/srv", Kind::Dir);
|
||||
let a = t.add(0, b"a", Kind::Dir);
|
||||
let b = t.add(0, b"b", Kind::Dir);
|
||||
let f1 = t.add(a, b"x.iso", Kind::File);
|
||||
(t.node_mut(f1).size, t.node_mut(f1).alloc) = (1000, 1024);
|
||||
let f2 = t.add(b, b"y.log", Kind::File);
|
||||
(t.node_mut(f2).size, t.node_mut(f2).alloc) = (50, 4096);
|
||||
t.node_mut(f2).flags = FLAG_SPARSE;
|
||||
t.recompute();
|
||||
(t.stats.files, t.stats.dirs) = (2, 3);
|
||||
t.stats.add_error(ErrorRecord { node: b, kind: ErrKind::Permission, msg: "denied".into(), ..Default::default() });
|
||||
t
|
||||
}
|
||||
|
||||
fn saved(t: &Tree) -> Vec<u8> {
|
||||
let mut buf = Vec::new();
|
||||
save(&mut buf, t).unwrap();
|
||||
buf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip() {
|
||||
let src = sample();
|
||||
let got = load(&saved(&src)[..]).unwrap();
|
||||
assert_eq!(got.len(), src.len());
|
||||
for i in 0..src.len() as NodeId {
|
||||
let (a, b) = (src.node(i), got.node(i));
|
||||
assert!(
|
||||
a.name == b.name && a.tot_size == b.tot_size && a.tot_alloc == b.tot_alloc && a.flags == b.flags
|
||||
&& src.path(i) == got.path(i) && src.ext_name(a) == got.ext_name(b),
|
||||
"node {i} differs: {a:?} vs {b:?}"
|
||||
);
|
||||
}
|
||||
assert_eq!(got.stats.err_counts[ErrKind::Permission as usize], 1);
|
||||
assert_eq!(got.stats.errors.len(), 1);
|
||||
assert_eq!(got.stats.start, src.stats.start);
|
||||
assert_eq!(got.stats.end, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn corruption_detected() {
|
||||
let data = saved(&sample());
|
||||
let mut bad = data.clone();
|
||||
*bad.last_mut().unwrap() ^= 0xff; // flip a byte in the checksum trailer
|
||||
assert!(load(&bad[..]).is_err(), "checksum corruption not detected");
|
||||
// Every truncation must fail cleanly.
|
||||
for i in (0..data.len() - 1).step_by(7) {
|
||||
assert!(load(&data[..i]).is_err(), "truncation at {i} accepted");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compare_trees() {
|
||||
let old = sample();
|
||||
let mut nw = Tree::new(b"/srv", Kind::Dir);
|
||||
let a = nw.add(0, b"a", Kind::Dir);
|
||||
let f1 = nw.add(a, b"x.iso", Kind::File);
|
||||
(nw.node_mut(f1).size, nw.node_mut(f1).alloc) = (5000, 5120);
|
||||
let c = nw.add(0, b"c", Kind::Dir);
|
||||
let f3 = nw.add(c, b"z", Kind::File);
|
||||
(nw.node_mut(f3).size, nw.node_mut(f3).alloc) = (10, 4096);
|
||||
nw.recompute();
|
||||
|
||||
let d = compare(old, &nw);
|
||||
let al = SizeMode::Allocated;
|
||||
assert_eq!(d.status_of(&nw, f1, al), Status::Grew);
|
||||
assert_eq!(d.delta(&nw, f1, al), 4096);
|
||||
assert_eq!(d.status_of(&nw, c, al), Status::Added);
|
||||
assert_eq!(d.removed.len(), 1);
|
||||
assert_eq!(&d.old.node(d.removed[0]).name[..], b"b");
|
||||
let ch = d.changes(&nw, 10, al);
|
||||
let paths: HashMap<&str, Status> = ch.iter().map(|c| (c.path.as_str(), c.status)).collect();
|
||||
assert_eq!(paths.get("/srv/a/x.iso"), Some(&Status::Grew), "{ch:?}");
|
||||
assert_eq!(paths.get("/srv/b"), Some(&Status::Removed), "{ch:?}");
|
||||
assert_eq!(paths.get("/srv/c"), Some(&Status::Added), "{ch:?}");
|
||||
assert!(!paths.contains_key("/srv/a"), "/srv/a should be explained by its child");
|
||||
assert!(!paths.contains_key("/srv/c/z"), "contents of new dirs should not be listed separately");
|
||||
}
|
||||
|
||||
fn body(b: &[u8]) -> Body {
|
||||
let (tx, rx) = crossbeam_channel::unbounded();
|
||||
tx.send(Ok(b.to_vec())).unwrap();
|
||||
Body::new(rx)
|
||||
}
|
||||
|
||||
fn put_uvarint(v: u64) -> Vec<u8> {
|
||||
let mut w = Writer { gz: GzEncoder::new(Vec::new(), Compression::fast()), h: Sha256::new(), buf: Vec::new() };
|
||||
w.uvarint(v).unwrap();
|
||||
w.buf
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_uvarint_matches_go() {
|
||||
for v in [0, 1, 127, 128, 300, 1 << 35, u64::MAX] {
|
||||
assert_eq!(read_uvarint(&mut body(&put_uvarint(v))), Ok(v));
|
||||
}
|
||||
assert_eq!(put_uvarint(300), [0xac, 0x02]);
|
||||
assert!(read_uvarint(&mut body(&[0xff; 11])).is_err(), "overflow not detected");
|
||||
// Zig-zag as Go's PutVarint.
|
||||
for (v, enc) in [(0i64, 0u64), (-1, 1), (1, 2), (-2, 3), (i64::MIN, u64::MAX)] {
|
||||
assert_eq!(((v << 1) ^ (v >> 63)) as u64, enc);
|
||||
assert_eq!(body(&put_uvarint(enc)).varint("v"), Ok(v));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_time_json() {
|
||||
let t = GoTime::parse("2026-10-08T10:40:00.123456789+01:00").unwrap();
|
||||
assert_eq!((t.secs, t.nanos, t.off), (1_791_452_400, 123_456_789, 3600));
|
||||
assert_eq!(t.format(), "2026-10-08T10:40:00.123456789+01:00");
|
||||
assert_eq!(GoTime { off: 0, ..t }.format(), "2026-10-08T09:40:00.123456789Z");
|
||||
assert_eq!(GoTime { off: -5 * 3600 - 1800, ..t }.format(), "2026-10-08T04:10:00.123456789-05:30");
|
||||
assert_eq!(GoTime::parse("2026-10-08T09:40:00.1Z").unwrap().format(), "2026-10-08T09:40:00.1Z");
|
||||
assert_eq!(GoTime::parse("0001-01-01T00:00:00Z"), Some(GoTime::default()));
|
||||
assert_eq!(GoTime::default().format(), "0001-01-01T00:00:00Z");
|
||||
assert_eq!(GoTime::from_system(GoTime::default().to_system()), GoTime::default());
|
||||
for bad in ["", "2026-10-08", "2026-13-08T00:00:00Z", "2026-10-08T00:00:00", "2026-10-08T00:00:00.Z"] {
|
||||
assert_eq!(GoTime::parse(bad), None, "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
/// A tiny tree saved by the Go binary
|
||||
/// (tree/{a/x.iso 10000 B, a/b/note.txt, c/y.log 3000 B, c/link -> ../a}).
|
||||
#[test]
|
||||
fn loads_go_snapshot() {
|
||||
let t = load_file(Path::new(concat!(env!("CARGO_MANIFEST_DIR"), "/src/snapshot/testdata/go.msz"))).unwrap();
|
||||
let r = t.node(0);
|
||||
assert_eq!((r.tot_size, r.tot_alloc), (13310, 20480));
|
||||
assert_eq!((t.stats.files, t.stats.dirs, t.stats.symlinks), (3, 4, 1));
|
||||
assert_eq!(t.len(), 8);
|
||||
assert!(t.stats.root.ends_with(b"/tree"));
|
||||
assert!(t.stats.end.is_some() && t.stats.complete);
|
||||
assert!(t.stats.from_snapshot.contains("mapsize"), "{}", t.stats.from_snapshot);
|
||||
let iso = (0..t.len() as NodeId).find(|&i| &t.node(i).name[..] == b"x.iso").unwrap();
|
||||
assert_eq!(t.node(iso).size, 10000);
|
||||
assert_eq!(t.ext_name(t.node(iso)), "iso");
|
||||
assert!(t.path_string(iso).ends_with("/tree/a/x.iso"));
|
||||
// And it survives a Rust round trip unchanged.
|
||||
let again = load(&saved(&t)[..]).unwrap();
|
||||
assert_eq!((again.node(0).tot_size, again.len(), again.stats.start), (13310, 8, t.stats.start));
|
||||
}
|
||||
|
||||
// Hostile input.
|
||||
|
||||
/// Builds a well-formed (correctly checksummed) but hostile snapshot.
|
||||
fn craft(n: usize, meta: &str, rec: impl Fn(usize, &mut Writer<Vec<u8>>)) -> Vec<u8> {
|
||||
let mut out = Vec::new();
|
||||
out.extend_from_slice(brand::SNAPSHOT_MAGIC);
|
||||
out.extend_from_slice(&VERSION.to_le_bytes());
|
||||
out.extend_from_slice(&FLAG_GZIPPED.to_le_bytes());
|
||||
let mut w = Writer { gz: GzEncoder::new(out, Compression::best()), h: Sha256::new(), buf: Vec::new() };
|
||||
w.uvarint(meta.len() as u64).unwrap();
|
||||
w.bytes(meta.as_bytes()).unwrap();
|
||||
w.uvarint(1).unwrap();
|
||||
w.str(b"").unwrap();
|
||||
w.uvarint(n as u64).unwrap();
|
||||
for i in 0..n {
|
||||
rec(i, &mut w);
|
||||
}
|
||||
w.uvarint(0).unwrap();
|
||||
w.flush().unwrap();
|
||||
let mut out = w.gz.finish().unwrap();
|
||||
let sum = w.h.finalize();
|
||||
out.extend_from_slice(&sum);
|
||||
out
|
||||
}
|
||||
|
||||
fn node(w: &mut Writer<Vec<u8>>, pd: u64, kind: Kind, name: &str, size: i64) {
|
||||
w.uvarint(pd).unwrap();
|
||||
w.bytes(&[kind as u8, 0]).unwrap();
|
||||
w.uvarint(0).unwrap();
|
||||
w.str(name.as_bytes()).unwrap();
|
||||
w.varint(size).unwrap();
|
||||
w.varint(size).unwrap();
|
||||
w.varint(0).unwrap();
|
||||
for _ in 0..5 {
|
||||
w.uvarint(0).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
fn chain(n: usize) -> Vec<u8> {
|
||||
craft(n, r#"{"root":"/r"}"#, |i, w| if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::Dir, "a", 0) })
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn deep_chain_rejected() {
|
||||
load(&chain(MAX_DEPTH as usize + 1)[..]).expect("depth at the limit must load");
|
||||
let err = load(&chain(MAX_DEPTH as usize + 2)[..]).err().expect("over-deep chain accepted");
|
||||
assert!(err.to_string().contains("deeper"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decompression_bomb_limited() {
|
||||
let flat = craft(100_000, r#"{"root":"/r"}"#, |i, w| {
|
||||
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", 0) }
|
||||
});
|
||||
let err = load_limit(&flat[..], 64 << 10).err().expect("bomb not limited");
|
||||
assert!(err.to_string().contains("limit"), "{err}");
|
||||
assert!(body_limit(flat.len() as i64) >= 64 << 20, "body_limit floor");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn huge_sizes_do_not_wrap() {
|
||||
let data = craft(3, r#"{"root":"/r"}"#, |i, w| {
|
||||
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, i as u64, Kind::File, "f", MAX_SIZE) }
|
||||
});
|
||||
let tr = load(&data[..]).unwrap();
|
||||
assert!(tr.node(0).tot_size >= 0, "aggregate wrapped negative");
|
||||
let over = craft(2, r#"{"root":"/r"}"#, |i, w| {
|
||||
if i == 0 { node(w, 0, Kind::Dir, "", 0) } else { node(w, 1, Kind::File, "f", MAX_SIZE + 1) }
|
||||
});
|
||||
assert!(load(&over[..]).is_err(), "size above MAX_SIZE accepted");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_metadata_sanitised_in_summary() {
|
||||
let data = craft(1, r#"{"root":"/r\u001b[2J","excludes":["\u001b]0;PWNED\u0007"],"excluded":1}"#, |_, w| {
|
||||
node(w, 0, Kind::Dir, "", 0)
|
||||
});
|
||||
let mut tr = load(&data[..]).unwrap();
|
||||
tr.stats.excluded = 1;
|
||||
let mut b = Vec::new();
|
||||
crate::export::summary(&mut b, &tr).unwrap();
|
||||
assert!(!b.iter().any(|&c| c == 0x1b || c == 0x07), "raw control characters in summary: {:?}", String::from_utf8_lossy(&b));
|
||||
}
|
||||
|
||||
/// Blocks of one directory plus K-1 files repeat byte for byte, so a
|
||||
/// small file describes millions of nodes; load_file must refuse before
|
||||
/// allocating.
|
||||
#[test]
|
||||
fn node_amplification_limited() {
|
||||
const K: usize = 1000;
|
||||
const N: usize = 1_200_001;
|
||||
let data = craft(N, r#"{"root":"/r"}"#, |i, w| match i {
|
||||
0 => node(w, 0, Kind::Dir, "", 0),
|
||||
1 => node(w, 1, Kind::Dir, "d", 0),
|
||||
_ if (i - 1) % K == 0 => node(w, K as u64, Kind::Dir, "d", 0),
|
||||
_ => node(w, ((i - 1) % K) as u64, Kind::File, "f", 0),
|
||||
});
|
||||
load(&data[..]).expect("crafted tree is otherwise valid");
|
||||
let p = std::env::temp_dir().join(format!("mapsize-amp-{}.msz", std::process::id()));
|
||||
fs::write(&p, &data).unwrap();
|
||||
let res = load_file(&p);
|
||||
let _ = fs::remove_file(&p);
|
||||
let err = res.err().unwrap_or_else(|| panic!("{N} nodes from a {}-byte file accepted", data.len()));
|
||||
assert!(err.to_string().contains("node count"), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn save_file_is_atomic_and_detected() {
|
||||
let dir = std::env::temp_dir().join(format!("mapsize-save-{}", std::process::id()));
|
||||
fs::create_dir_all(&dir).unwrap();
|
||||
let p = dir.join("s.msz");
|
||||
save_file(&p, &sample()).unwrap();
|
||||
assert!(is_snapshot(&p));
|
||||
assert_eq!(load_file(&p).unwrap().len(), 5);
|
||||
assert_eq!(fs::read_dir(&dir).unwrap().count(), 1, "temp file left behind");
|
||||
fs::remove_dir_all(&dir).unwrap();
|
||||
}
|
||||
}
|
||||
Vendored
BIN
Binary file not shown.
Reference in New Issue
Block a user