From a6610449f2710a7cace37cf27b5d2c997225154c Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Thu, 8 Oct 2026 10:21:44 +0100 Subject: [PATCH] Store, validated types, bounded pipeline, no-clobber extraction The store reads and writes the Go on-disk format (it opens the oracle's fixture vault). Object I/O runs on one bounded worker pool with eight reusable cleared buffers; sequential and parallel runs produce identical bytes under seeded test randomness. Extraction publishes with RENAME_NOREPLACE/link and only after full authentication; failed or after-publish writes keep crash-safe ordering. Co-Authored-By: Claude --- Cargo.lock | 1 + Cargo.toml | 2 +- src/crypto/fury.rs | 30 +- src/crypto/stream.rs | 2 +- src/error.rs | 75 ++++ src/lib.rs | 3 + src/pipeline.rs | 222 ++++++++++++ src/secret.rs | 7 +- src/store/mod.rs | 795 +++++++++++++++++++++++++++++++++++++++++++ src/store/tests.rs | 492 ++++++++++++++++++++++++++ src/store/types.rs | 375 ++++++++++++++++++++ 11 files changed, 1996 insertions(+), 8 deletions(-) create mode 100644 src/error.rs create mode 100644 src/pipeline.rs create mode 100644 src/store/mod.rs create mode 100644 src/store/tests.rs create mode 100644 src/store/types.rs diff --git a/Cargo.lock b/Cargo.lock index 43d439c..131a37a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1236,6 +1236,7 @@ version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879" dependencies = [ + "serde", "zeroize_derive", ] diff --git a/Cargo.toml b/Cargo.toml index 11c2565..1278e88 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,7 +17,7 @@ hmac = "0.13" sha2 = { version = "0.11", default-features = false } sha3 = { version = "0.11", default-features = false } ml-kem = { version = "0.3.2", features = ["zeroize", "hazmat"] } -zeroize = { version = "1.9", features = ["derive"] } +zeroize = { version = "1.9", features = ["derive", "serde"] } getrandom = "0.4" base64 = "0.23" serde = { version = "1", features = ["derive"] } diff --git a/src/crypto/fury.rs b/src/crypto/fury.rs index 42ada9b..d9afd69 100644 --- a/src/crypto/fury.rs +++ b/src/crypto/fury.rs @@ -130,6 +130,29 @@ pub fn decrypt_reader(src: R, id: &Identity) -> Result(src: R, id: &Identity, max_plaintext: usize) -> Result { + let mut buf = SecretVec::with_capacity(max_ciphertext(max_plaintext) + 1); + decrypt_bounded_into(src, id, max_plaintext, &mut buf)?; + Ok(buf) +} + +/// The largest STREAM payload holding at most `max_plaintext` bytes. +pub fn max_ciphertext(max_plaintext: usize) -> usize { + max_plaintext + max_plaintext.div_ceil(stream::CHUNK_SIZE).max(1) * stream::TAG_SIZE +} + +/// As [`decrypt_bounded`], into a caller-supplied buffer whose capacity +/// must be at least `max_ciphertext(max_plaintext) + 1`. On error the +/// buffer is left empty. +pub fn decrypt_bounded_into(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> { + buf.clear(); + let r = decrypt_into(src, id, max_plaintext, buf); + if r.is_err() { + buf.clear(); + } + r +} + +fn decrypt_into(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> { let (h, mut payload) = header::parse(src).map_err(Error::Header)?; let fk = file_key(&h, id)?; let mut nonce = [0u8; NONCE_SIZE]; @@ -137,16 +160,15 @@ pub fn decrypt_bounded(src: R, id: &Identity, max_plaintext: usize) -> .read_exact(&mut nonce) .map_err(|e| Error::Io(io::Error::new(e.kind(), format!("failed to read nonce: {e}"))))?; let key = payload_key(fk.expose(), &nonce); - let chunks = max_plaintext.div_ceil(stream::CHUNK_SIZE).max(1); - let max_ct = max_plaintext + chunks * stream::TAG_SIZE; - let mut buf = SecretVec::zeroed(max_ct + 1); + let max_ct = max_ciphertext(max_plaintext); + buf.resize_within_capacity(max_ct + 1); let n = stream::read_full(&mut payload, buf.expose_mut())?; if n > max_ct { return Err(Error::TooLarge); } let plain = stream::open_all(key.expose(), &mut buf.expose_mut()[..n]).map_err(|e| Error::Io(e.into()))?; buf.resize_within_capacity(plain); - Ok(buf) + Ok(()) } #[cfg(test)] diff --git a/src/crypto/stream.rs b/src/crypto/stream.rs index ef69c5f..e3d434e 100644 --- a/src/crypto/stream.rs +++ b/src/crypto/stream.rs @@ -185,7 +185,7 @@ impl Drop for StreamWriter { } /// Reads until `buf` is full or the source ends; returns the count. -pub fn read_full(src: &mut impl Read, buf: &mut [u8]) -> io::Result { +pub fn read_full(src: &mut R, buf: &mut [u8]) -> io::Result { let mut n = 0; while n < buf.len() { match src.read(&mut buf[n..]) { diff --git a/src/error.rs b/src/error.rs new file mode 100644 index 0000000..cee618b --- /dev/null +++ b/src/error.rs @@ -0,0 +1,75 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! The command-level error: a lower-case message without a trailing full +//! stop, printed as `vault: MESSAGE`. + +use std::fmt; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Error { + pub msg: String, + pub kind: Kind, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Kind { + Other, + /// A source ended before its declared size. + SourceShort, + /// A file was written under its final name, but a later durability step + /// failed: what it depends on must not be deleted. + AfterPublish, + /// The thing to remove or read was not there. + NotFound, +} + +impl Error { + pub fn new(msg: impl Into) -> Error { + Error { + msg: msg.into(), + kind: Kind::Other, + } + } + + pub fn with_kind(mut self, kind: Kind) -> Error { + self.kind = kind; + self + } + + pub fn io(op: &str, path: &std::path::Path, e: &std::io::Error) -> Error { + Error::new(crate::util::io_msg(op, path, e)) + } +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.msg) + } +} + +impl std::error::Error for Error {} + +impl From for Error { + fn from(s: String) -> Self { + Error::new(s) + } +} + +impl From<&str> for Error { + fn from(s: &str) -> Self { + Error::new(s) + } +} + +impl From for Error { + fn from(e: std::io::Error) -> Self { + Error::new(crate::util::io_reason(&e)) + } +} + +pub type Result = std::result::Result; + +#[macro_export] +macro_rules! bail { + ($($t:tt)*) => { return Err($crate::error::Error::new(format!($($t)*))) }; +} diff --git a/src/lib.rs b/src/lib.rs index 8c54538..22a607c 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -7,8 +7,11 @@ #![deny(unsafe_op_in_unsafe_fn)] pub mod crypto; +pub mod error; pub mod keyfile; +pub mod pipeline; pub mod platform; pub mod rng; pub mod secret; +pub mod store; pub mod util; diff --git a/src/pipeline.rs b/src/pipeline.rs new file mode 100644 index 0000000..a3cc0b7 --- /dev/null +++ b/src/pipeline.rs @@ -0,0 +1,222 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! One bounded worker pool per process, and the bounded buffer pool that +//! object I/O borrows from. +//! +//! The pool runs a fixed number of threads fed from a bounded queue, so +//! submitting blocks once the queue is full (backpressure). Coordinators — +//! the code that reads a source, writes an output in order, or collects +//! results — always run on the caller's thread, never on a worker, so no +//! worker ever waits for another and nested waiting cannot deadlock. +//! Dropping the pool closes the queue and joins every thread; jobs already +//! queued still run (each is bounded file I/O and crypto), and their +//! results are dropped if nobody waits for them. A panicking job is turned +//! into an error for whoever waits on it and does not take the worker down. + +use crate::secret::SecretVec; +use std::sync::mpsc::{Receiver, SyncSender, sync_channel}; +use std::sync::{Arc, Mutex}; +use std::thread::JoinHandle; + +/// The largest worker count accepted from the user. +pub const MAX_WORKERS: usize = 256; +/// The default cap: past this, the work is bound by storage, not CPU. +pub const DEFAULT_MAX_WORKERS: usize = 8; +/// Object-sized buffers in flight at once for one file. +pub const OBJECT_BUFFERS: usize = 8; + +type Job = Box; + +pub struct Pool { + tx: Option>, + threads: Vec>, +} + +impl Pool { + pub fn new(workers: usize) -> Pool { + assert!((1..=MAX_WORKERS).contains(&workers), "worker count out of range"); + let (tx, rx) = sync_channel::(workers * 2); + let rx = Arc::new(Mutex::new(rx)); + let threads = (0..workers) + .map(|i| { + let rx = Arc::clone(&rx); + std::thread::Builder::new() + .name(format!("vault-worker-{i}")) + .spawn(move || { + loop { + // Hold the lock only to take a job, not to run it. + let job = match rx.lock() { + Ok(r) => r.recv(), + Err(p) => p.into_inner().recv(), + }; + match job { + Ok(job) => job(), + Err(_) => return, + } + } + }) + .expect("spawn worker thread") + }) + .collect(); + Pool { tx: Some(tx), threads } + } + + /// The default worker count: available parallelism, at most 8. + pub fn default_workers() -> usize { + std::thread::available_parallelism().map_or(1, |n| n.get()).min(DEFAULT_MAX_WORKERS) + } + + pub fn workers(&self) -> usize { + self.threads.len() + } + + /// Queues `f`, blocking while the queue is full. + pub fn submit(&self, f: impl FnOnce() -> T + Send + 'static) -> Ticket { + let (rtx, rrx) = sync_channel(1); + let job: Job = Box::new(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|_| "internal error: a worker panicked".to_string()); + let _ = rtx.send(r); + }); + self.tx.as_ref().expect("pool open").send(job).expect("workers alive"); + Ticket(rrx) + } +} + +impl Drop for Pool { + fn drop(&mut self) { + drop(self.tx.take()); + for t in self.threads.drain(..) { + let _ = t.join(); + } + } +} + +/// The pending result of a submitted job. +pub struct Ticket(Receiver>); + +impl Ticket { + pub fn wait(self) -> Result { + self.0.recv().unwrap_or_else(|_| Err("internal error: a worker vanished".into())) + } +} + +/// A fixed set of reusable, equally sized secret buffers. Taking one blocks +/// until one is free; a buffer is cleared when it comes back, and every +/// buffer is cleared again when the pool is dropped. +#[derive(Clone)] +pub struct Buffers { + tx: SyncSender, + rx: Arc>>, +} + +impl Buffers { + pub fn new(count: usize, capacity: usize) -> Buffers { + let (tx, rx) = sync_channel(count); + for _ in 0..count { + tx.send(SecretVec::with_capacity(capacity)).expect("room for every buffer"); + } + Buffers { + tx, + rx: Arc::new(Mutex::new(rx)), + } + } + + pub fn take(&self) -> Lease { + let b = match self.rx.lock() { + Ok(r) => r.recv(), + Err(p) => p.into_inner().recv(), + } + .expect("buffer pool alive"); + Lease { + buf: Some(b), + home: self.tx.clone(), + } + } +} + +/// A borrowed buffer, returned (cleared) on drop — including when a job +/// fails or panics. +pub struct Lease { + buf: Option, + home: SyncSender, +} + +impl Lease { + pub fn buf(&mut self) -> &mut SecretVec { + self.buf.as_mut().expect("leased") + } +} + +impl Drop for Lease { + fn drop(&mut self) { + if let Some(mut b) = self.buf.take() { + b.clear(); + let _ = self.home.send(b); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::time::Duration; + + #[test] + fn results_in_submission_order_and_bounded_concurrency() { + let pool = Pool::new(4); + let running = Arc::new(AtomicUsize::new(0)); + let peak = Arc::new(AtomicUsize::new(0)); + let tickets: Vec<_> = (0..64) + .map(|i| { + let (running, peak) = (Arc::clone(&running), Arc::clone(&peak)); + pool.submit(move || { + let now = running.fetch_add(1, Ordering::SeqCst) + 1; + peak.fetch_max(now, Ordering::SeqCst); + std::thread::sleep(Duration::from_millis((64 - i) % 5)); + running.fetch_sub(1, Ordering::SeqCst); + i + }) + }) + .collect(); + let got: Vec<_> = tickets.into_iter().map(|t| t.wait().unwrap()).collect(); + assert_eq!(got, (0..64).collect::>()); + assert!(peak.load(Ordering::SeqCst) <= 4); + } + + #[test] + fn panics_become_errors() { + let pool = Pool::new(2); + let t = pool.submit(|| -> u8 { panic!("boom") }); + assert!(t.wait().is_err()); + assert_eq!(pool.submit(|| 5).wait().unwrap(), 5, "worker survived"); + } + + #[test] + fn dropping_tickets_and_pool_cleans_up() { + for _ in 0..20 { + let pool = Pool::new(3); + for _ in 0..10 { + drop(pool.submit(|| std::thread::sleep(Duration::from_millis(1)))); + } + drop(pool); // joins: no threads outlive it + } + } + + #[test] + fn buffers_are_bounded_and_cleared() { + let b = Buffers::new(2, 8); + let mut l1 = b.take(); + l1.buf().extend_within_capacity(b"secret"); + let _l2 = b.take(); + let b2 = b.clone(); + let waiter = std::thread::spawn(move || { + let mut l = b2.take(); + l.buf().len() + }); + std::thread::sleep(Duration::from_millis(50)); + assert!(!waiter.is_finished(), "a third lease should wait"); + drop(l1); + assert_eq!(waiter.join().unwrap(), 0, "returned buffers are cleared"); + } +} diff --git a/src/secret.rs b/src/secret.rs index 1d8c326..1f9b5cf 100644 --- a/src/secret.rs +++ b/src/secret.rs @@ -99,9 +99,12 @@ impl SecretVec { self.0.resize(n, 0); } - /// Clears the contents for reuse, keeping the allocation. + /// Clears the contents for reuse, keeping the allocation. Bytes past + /// the length are always zero already (shrinking clears them), so only + /// the live part needs overwriting. pub fn clear(&mut self) { - self.0.zeroize(); + self.0.as_mut_slice().zeroize(); + self.0.clear(); } } diff --git a/src/store/mod.rs b/src/store/mod.rs new file mode 100644 index 0000000..82840fe --- /dev/null +++ b/src/store/mod.rs @@ -0,0 +1,795 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! The on-disk vault, compatible with the Go version: +//! +//! ```text +//! DIR/.vault "VAULT 2\n" +//! DIR/records// one encrypted 4096-byte catalogue record per file +//! DIR/objects// a file's body, in one or more encrypted objects +//! ``` +//! +//! `` is the first byte of `` in hex; all 256 shards exist from +//! `init`. Record IDs are random; a file's object IDs are +//! HMAC-SHA-256(seed, index)[:16] for a secret seed in its record, so +//! nothing on disk links a file's objects. An object's plaintext is its own +//! ID, up to 4 MiB − 16 bytes of the file, and random padding up to a +//! power-of-two bucket of 64 KiB to 4 MiB. Every ciphertext's and shard's +//! modification time is pinned to the epoch. +//! +//! Ordering keeps the store consistent across crashes: objects are durable +//! before the record that names them; a new record is durable before an old +//! one it replaces is removed; a record's removal is durable before the +//! objects only it used are deleted; copies share objects. + +pub mod types; + +use crate::crypto::fury; +use crate::crypto::kem::{Identity, Recipient}; +use crate::error::{Error, Kind, Result}; +use crate::pipeline::{Buffers, Lease, OBJECT_BUFFERS, Pool, Ticket}; +use crate::platform::fs as pfs; +use crate::rng::Entropy; +use crate::secret::SecretVec; +use crate::util::Instant; +use std::collections::{BTreeMap, HashSet, VecDeque}; +use std::ffi::OsStr; +use std::io::{Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +pub use types::*; + +pub const MARKER_NAME: &str = ".vault"; +pub const MARKER_CONTENT: &[u8] = b"VAULT 2\n"; +pub const RECORDS: &str = "records"; +pub const OBJECTS: &str = "objects"; +const FILLER_MIN: u64 = 8; +const FILLER_MAX: u64 = 32; +/// Ciphertext payload capacity for one object, plus one byte to detect +/// over-long objects. +const OBJECT_BUF: usize = CHUNK_SIZE as usize + 64 * crate::crypto::stream::TAG_SIZE + 1; + +struct Inner { + root: PathBuf, + id: Identity, + recipient: Recipient, + no_sync: bool, + entropy: Entropy, + /// Test hook: make record writes fail just after their rename. + #[cfg(test)] + fail_after_publish: std::sync::atomic::AtomicBool, +} + +pub struct Store { + inner: Arc, + pool: Arc, + buffers: OnceLock, +} + +/// Every record this identity can open, by name, newest first. +#[derive(Default)] +pub struct Catalogue { + by_name: BTreeMap>, +} + +impl Catalogue { + pub fn current(&self, name: &str) -> Option<&Record> { + self.by_name.get(name).and_then(|v| v.first()) + } + + pub fn contains(&self, name: &str) -> bool { + self.current(name).is_some() + } + + pub fn len(&self) -> usize { + self.by_name.len() + } + + pub fn is_empty(&self) -> bool { + self.by_name.is_empty() + } + + /// Every name, sorted. + pub fn names(&self) -> impl Iterator { + self.by_name.keys().map(String::as_str) + } + + /// Each name with its current record, sorted by name. + pub fn iter(&self) -> impl Iterator { + self.by_name.iter().filter_map(|(n, v)| v.first().map(|r| (n.as_str(), r))) + } + + /// The names at or beneath the folder `prefix`, sorted. + pub fn under(&self, prefix: &str) -> Vec { + self.by_name + .keys() + .filter(|n| *n == prefix || (n.len() > prefix.len() && n.starts_with(prefix) && n.as_bytes()[prefix.len()] == b'/')) + .cloned() + .collect() + } + + fn records(&self, name: &str) -> &[Record] { + self.by_name.get(name).map_or(&[], Vec::as_slice) + } + + fn insert(&mut self, r: Record) { + let v = self.by_name.entry(r.name.as_str().to_string()).or_default(); + v.push(r); + v.sort_by(|a, b| b.added.cmp(&a.added).then(a.id.cmp(&b.id))); + } +} + +/// What `add` did, for the message and for migrate's source check. +#[derive(Debug)] +pub struct Added { + pub replaced: bool, +} + +fn after_publish(e: &Error) -> bool { + e.kind == Kind::AfterPublish +} + +impl Store { + /// Creates a vault at `root` holding at least `filler` bytes of filler. + pub fn init(root: &Path, filler: u64, pool: Arc) -> Result<()> { + if root.join(MARKER_NAME).symlink_metadata().is_ok() { + crate::bail!("{} is already a vault", root.display()); + } + // Every shard exists from the start, so the tree's shape says nothing. + for dir in [RECORDS, OBJECTS] { + for i in 0..256 { + let p = root.join(dir).join(format!("{i:02x}")); + pfs::create_private_dir_all(&p).map_err(|e| Error::io("mkdir", &p, &e))?; + } + } + write_filler(root, filler, Arc::clone(&pool), Entropy::Os)?; + for dir in [RECORDS, OBJECTS] { + for i in 0..256 { + let p = root.join(dir).join(format!("{i:02x}")); + pfs::set_epoch(&p).map_err(|e| Error::io("chtimes", &p, &e))?; + } + let p = root.join(dir); + pfs::set_epoch(&p).map_err(|e| Error::io("chtimes", &p, &e))?; + } + let marker = root.join(MARKER_NAME); + let mut f = pfs::create_new_private(&marker).map_err(|e| Error::io("open", &marker, &e))?; + f.write_all(MARKER_CONTENT) + .and_then(|_| f.sync_all()) + .map_err(|e| Error::io("write", &marker, &e))?; + pfs::sync_dir(root).map_err(|e| Error::io("sync", root, &e))?; + Ok(()) + } + + pub fn open(root: &Path, id: Identity, pool: Arc) -> Result { + match std::fs::read(root.join(MARKER_NAME)) { + Ok(m) if m == MARKER_CONTENT => {} + _ => crate::bail!("{} is not a vault; run vault init", root.display()), + } + Ok(Store::new(root, id, pool, false, Entropy::Os)) + } + + fn new(root: &Path, id: Identity, pool: Arc, no_sync: bool, entropy: Entropy) -> Store { + let recipient = id.recipient(); + let inner = Inner { + root: root.to_path_buf(), + id, + recipient, + no_sync, + entropy, + #[cfg(test)] + fail_after_publish: Default::default(), + }; + Store { + inner: Arc::new(inner), + pool, + buffers: OnceLock::new(), + } + } + + #[cfg(test)] + pub(crate) fn open_with(root: &Path, id: Identity, pool: Arc, entropy: Entropy) -> Store { + Store::new(root, id, pool, false, entropy) + } + + pub fn root(&self) -> &Path { + &self.inner.root + } + + pub fn path(&self, dir: &str, id: &Id) -> PathBuf { + self.inner.path(dir, id) + } + + fn buffers(&self) -> &Buffers { + self.buffers.get_or_init(|| Buffers::new(OBJECT_BUFFERS, OBJECT_BUF)) + } + + /// Decrypts every record this identity can open. Records sealed to a + /// different key are skipped silently: they are simply not ours. + /// Unreadable ones of ours are reported and skipped. + pub fn catalogue(&self) -> Result { + let dir = self.inner.root.join(RECORDS); + let mut ids = Vec::new(); + for shard in sorted_dir(&dir)? { + if !shard.1 { + continue; + } + let sp = dir.join(&shard.0); + for (name, _) in sorted_dir(&sp)? { + if let Some(id) = name.to_str().and_then(Id::parse) { + if id.shard() == shard.0.to_string_lossy() { + ids.push(id); + } + } + } + } + let mut cat = Catalogue::default(); + let window = self.pool.workers() * 2; + let mut pending: VecDeque<(Id, Ticket>)> = VecDeque::new(); + let collect = |id: Id, r: std::result::Result, String>, cat: &mut Catalogue| match r { + Ok(Ok(rec)) => cat.insert(rec), + Ok(Err(ReadRecordError::Foreign)) => {} + Ok(Err(ReadRecordError::Other(e))) | Err(e) => eprintln!("vault: skipping unreadable record {}: {e}", id.hex()), + }; + for id in ids { + if pending.len() == window { + let (i, t) = pending.pop_front().expect("non-empty"); + collect(i, t.wait(), &mut cat); + } + let inner = Arc::clone(&self.inner); + pending.push_back((id, self.pool.submit(move || inner.read_record(id)))); + } + for (i, t) in pending { + collect(i, t.wait(), &mut cat); + } + Ok(cat) + } + + /// Stores the regular file at `src` as `name`, replacing any file of + /// that name. Fails, storing nothing, if the file changes while read. + pub fn add_file(&self, cat: &mut Catalogue, src: &Path, name: &VaultName) -> Result<(Added, SourceStamp)> { + let mut f = std::fs::File::open(src).map_err(|e| Error::io("open", src, &e))?; + let meta = f.metadata().map_err(|e| Error::io("stat", src, &e))?; + if !meta.is_file() || meta.len() > FILE_LIMIT { + crate::bail!("{}: not a regular file of at most 1 TiB", src.display()); + } + let stamp = SourceStamp::of(&meta); + let changed = || Error::new(format!("{} changed while being added", src.display())).with_kind(Kind::SourceShort); + let probe = f.try_clone().map_err(|e| Error::io("open", src, &e))?; + // Before the record is written: the file must still be the one read. + let mut unchanged = || match probe.metadata() { + Ok(m) if SourceStamp::of(&m) == stamp => Ok(()), + _ => Err(changed()), + }; + let mut guard = EndCheck { + inner: &mut f, + size: meta.len(), + read: 0, + }; + let mtime = Instant::from_system(meta.modified().unwrap_or(std::time::UNIX_EPOCH)); + match self.add_checked(cat, &mut guard, meta.len(), mode_of(&meta), mtime, name, &mut unchanged) { + Ok(a) => Ok((a, stamp)), + Err(e) if e.kind == Kind::SourceShort => Err(changed()), + Err(e) => Err(e), + } + } + + /// Stores `size` bytes from `src` as `name`. A source that ends early, + /// or (for an [`EndCheck`]) runs on past `size`, stores nothing. + pub fn add_reader( + &self, + cat: &mut Catalogue, + src: &mut dyn Read, + size: u64, + mode: u32, + mtime: Instant, + name: &VaultName, + ) -> Result { + self.add_checked(cat, src, size, mode, mtime, name, &mut || Ok(())) + } + + #[allow(clippy::too_many_arguments)] + fn add_checked( + &self, + cat: &mut Catalogue, + src: &mut dyn Read, + size: u64, + mode: u32, + mtime: Instant, + name: &VaultName, + before_commit: &mut dyn FnMut() -> Result<()>, + ) -> Result { + if size > FILE_LIMIT { + crate::bail!("{name}: larger than 1 TiB"); + } + let seed = ObjectSeed::random(&mut self.inner.entropy.rng(b"seed")); + if let Err(e) = self.write_objects(src, &seed, size).and_then(|_| before_commit()) { + let _ = self.drop_objects(&seed, size); + return Err(e); + } + let mut idr = self.inner.entropy.rng(seed.object_id(u64::MAX).0.as_slice()); + let rec = Record { + name: name.clone(), + seed, + size, + mode: mode & 0o777, + modified: mtime, + added: Instant::now(), + id: Id::random(&mut idr), + }; + if let Err(e) = self.inner.write_record(&rec) { + // A record already under its name keeps its objects, even if a + // later sync failed. + if !after_publish(&e) { + let _ = self.drop_objects(&rec.seed, size); + } + return Err(e); + } + let replaced = cat.contains(name.as_str()); + self.remove(cat, name.as_str())?; + cat.insert(rec); + Ok(Added { replaced }) + } + + /// Reads `size` bytes into objects derived from `seed`. The caller's + /// thread reads the source in order; encryption, writing and syncing of + /// up to eight objects overlap on the worker pool. + fn write_objects(&self, src: &mut dyn Read, seed: &ObjectSeed, size: u64) -> Result<()> { + let mut pending: VecDeque>> = VecDeque::new(); + let mut first_err: Option = None; + let mut remaining = size; + for i in 0..object_count(size) { + if pending.len() == OBJECT_BUFFERS { + if let Err(e) = pending.pop_front().expect("non-empty").wait().map_err(Error::new).and_then(|r| r) { + first_err = Some(e); + break; + } + } + let id = seed.object_id(i); + let part = remaining.min(CHUNK_DATA) as usize; + remaining -= part as u64; + let mut lease = self.buffers().take(); + let buf = lease.buf(); + buf.extend_within_capacity(&id.0); + buf.resize_within_capacity(ID_SIZE + part); + match crate::crypto::stream::read_full(src, &mut buf.expose_mut()[ID_SIZE..]) { + Ok(n) if n == part => {} + Ok(_) => { + first_err = Some(Error::new("source ended early").with_kind(Kind::SourceShort)); + break; + } + Err(e) => { + first_err = Some(if e.kind() == std::io::ErrorKind::UnexpectedEof { + Error::new("source ended early").with_kind(Kind::SourceShort) + } else { + Error::from(e) + }); + break; + } + } + let inner = Arc::clone(&self.inner); + pending.push_back(self.pool.submit(move || inner.write_object(id, lease))); + } + // Wait for everything in flight, keeping the first error. + for t in pending { + if let Err(e) = t.wait().map_err(Error::new).and_then(|r| r) { + first_err.get_or_insert(e); + } + } + match first_err { + Some(e) => Err(e), + None => Ok(()), + } + } + + /// Deletes every record for `name` and the objects no other name uses. + fn remove(&self, cat: &mut Catalogue, name: &str) -> Result<()> { + let in_use: HashSet = cat + .iter_all() + .filter(|(n, _)| *n != name) + .map(|(_, r)| r.seed.object_id(0)) + .collect(); + let records = cat.by_name.remove(name).unwrap_or_default(); + let mut iter = records.into_iter(); + while let Some(r) = iter.next() { + if let Err(e) = self.inner.remove_record(&r.id) { + // Put back what is still there. + let mut keep: Vec = vec![r]; + keep.extend(iter); + for k in keep { + cat.insert(k); + } + return Err(e); + } + if !in_use.contains(&r.seed.object_id(0)) { + self.drop_objects(&r.seed, r.size)?; + } + } + Ok(()) + } + + /// Deletes every record for `names` and the objects they alone hold, + /// calling `done` after each name. + pub fn remove_all(&self, cat: &mut Catalogue, names: &[String], done: &mut dyn FnMut(&str)) -> Result<()> { + let gone: HashSet<&str> = names.iter().map(String::as_str).collect(); + let in_use: HashSet = cat + .iter_all() + .filter(|(n, _)| !gone.contains(n)) + .map(|(_, r)| r.seed.object_id(0)) + .collect(); + for name in names { + for r in cat.records(name) { + self.inner.remove_record(&r.id)?; + if !in_use.contains(&r.seed.object_id(0)) { + self.drop_objects(&r.seed, r.size)?; + } + } + cat.by_name.remove(name); + done(name); + } + Ok(()) + } + + /// Stores each pair's first name again under its second, sharing its + /// objects; with `mv`, then deletes the first names. New records come + /// first, so an interruption leaves a file under both names, not none. + pub fn copy_all(&self, cat: &mut Catalogue, pairs: &[(String, VaultName)], mv: bool, done: &mut dyn FnMut(&str, &str)) -> Result<()> { + let mut old = Vec::new(); + for (from, to) in pairs { + let src = cat.current(from).ok_or_else(|| Error::new(format!("{from}: not in the vault")))?; + let mut idr = self.inner.entropy.rng(to.as_str().as_bytes()); + let mut r = src.copy_as(to.clone(), Id::random(&mut idr)); + if !mv { + r.added = Instant::now(); + } + self.inner.write_record(&r)?; + cat.by_name.remove(to.as_str()); + cat.insert(r); + old.push(from.clone()); + done(from, to.as_str()); + } + if mv { self.remove_all(cat, &old, &mut |_| {}) } else { Ok(()) } + } + + /// Deletes the objects of a file of `size` bytes, skipping any gone. + fn drop_objects(&self, seed: &ObjectSeed, size: u64) -> Result<()> { + for i in 0..object_count(size) { + match self.inner.remove_file(OBJECTS, &seed.object_id(i), false) { + Err(e) if e.kind == Kind::NotFound => {} + r => r?, + } + } + Ok(()) + } + + /// Decrypts `rec` into `dir/name`. Nothing appears under `name` until + /// every object — ID prefix, data, padding, final tag and end of file — + /// has authenticated, and an existing `name` (even a dangling symlink, + /// even one created meanwhile) is never replaced. + pub fn extract(&self, rec: &Record, dir: &pfs::Dir, name: &OsStr) -> Result<()> { + let out = dir.path().join(name); + if out.symlink_metadata().is_ok() { + crate::bail!("{} already exists", out.display()); + } + let (mut f, tmp) = dir + .create_temp(".vault-get-") + .map_err(|e| Error::io("create temporary file in", dir.path(), &e))?; + let r = self.extract_into(rec, &mut f).and_then(|_| { + f.set_times( + std::fs::FileTimes::new() + .set_accessed(rec.modified.to_system()) + .set_modified(rec.modified.to_system()), + ) + .map_err(|e| Error::io("chtimes", &out, &e))?; + pfs::set_mode(&f, rec.mode).map_err(|e| Error::io("chmod", &out, &e))?; + f.sync_all().map_err(|e| Error::io("sync", &out, &e)) + }); + drop(f); + let r = r.and_then(|_| { + dir.publish(&tmp, name).map_err(|e| { + if e.kind() == std::io::ErrorKind::AlreadyExists { + Error::new(format!("{} already exists", out.display())) + } else { + Error::io("rename", &out, &e) + } + }) + }); + if let Err(e) = r { + let _ = dir.remove(&tmp); + return Err(e); + } + dir.sync().map_err(|e| Error::io("sync", dir.path(), &e))?; + Ok(()) + } + + fn extract_into(&self, rec: &Record, out: &mut std::fs::File) -> Result<()> { + let mut pending: VecDeque<(u64, Ticket>)> = VecDeque::new(); + let count = object_count(rec.size); + let mut write = |i: u64, t: Ticket>| -> Result<()> { + let mut lease = t.wait().map_err(Error::new)??; + let part = rec.part(i) as usize; + out.write_all(&lease.buf().expose()[ID_SIZE..ID_SIZE + part]).map_err(Error::from) + }; + let mut result = Ok(()); + for i in 0..count { + if pending.len() == OBJECT_BUFFERS { + let (j, t) = pending.pop_front().expect("non-empty"); + if let Err(e) = write(j, t) { + result = Err(e); + break; + } + } + let lease = self.buffers().take(); + let inner = Arc::clone(&self.inner); + let id = rec.seed.object_id(i); + let part = rec.part(i); + pending.push_back((i, self.pool.submit(move || inner.read_object(id, part, lease)))); + } + for (j, t) in pending { + if result.is_ok() { + result = write(j, t); + } else { + drop(t.wait()); + } + } + result + } +} + +impl Catalogue { + /// Every record, duplicates included. + fn iter_all(&self) -> impl Iterator { + self.by_name.iter().flat_map(|(n, v)| v.iter().map(move |r| (n.as_str(), r))) + } +} + +enum ReadRecordError { + Foreign, + Other(String), +} + +impl Inner { + fn path(&self, dir: &str, id: &Id) -> PathBuf { + self.root.join(dir).join(id.shard()).join(id.hex()) + } + + /// Encrypts `plaintext` to `dir/` through a temporary file, so a + /// partial ciphertext never appears under a real ID. An error after the + /// rename is marked [`Kind::AfterPublish`]. + fn write_ciphertext(&self, dir: &str, id: &Id, plaintext: &[u8]) -> Result<()> { + let dst = self.path(dir, id); + let shard = dst.parent().expect("sharded path").to_path_buf(); + let (mut f, tmp) = pfs::create_temp(&shard, ".pending-").map_err(|e| Error::io("open", &shard, &e))?; + let mut ct = Vec::with_capacity(plaintext.len() + 32 << 10); + fury::encrypt_to(&mut ct, &self.recipient, plaintext, &mut self.entropy.rng(&id.0)); + let r = (|| -> Result<()> { + f.write_all(&ct).map_err(|e| Error::io("write", &tmp, &e))?; + if !self.no_sync { + f.sync_all().map_err(|e| Error::io("sync", &tmp, &e))?; + } + drop(f); + pfs::set_epoch(&tmp).map_err(|e| Error::io("chtimes", &tmp, &e))?; + std::fs::rename(&tmp, &dst).map_err(|e| Error::io("rename", &dst, &e)) + })(); + if let Err(e) = r { + let _ = std::fs::remove_file(&tmp); + return Err(e); + } + let after = |e: Error| e.with_kind(Kind::AfterPublish); + #[cfg(test)] + if dir == RECORDS && self.fail_after_publish.load(std::sync::atomic::Ordering::SeqCst) { + return Err(after(Error::new("injected sync failure"))); + } + pfs::set_epoch(&shard).map_err(|e| after(Error::io("chtimes", &shard, &e)))?; + if !self.no_sync { + pfs::sync_dir(&shard).map_err(|e| after(Error::io("sync", &shard, &e)))?; + } + Ok(()) + } + + fn write_object(&self, id: Id, mut lease: Lease) -> Result<()> { + let buf = lease.buf(); + let used = buf.len(); + let padded = padded_size(used as u64) as usize; + buf.resize_within_capacity(padded); + self.entropy + .rng(&[b"pad".as_slice(), &id.0].concat()) + .fill(&mut buf.expose_mut()[used..]); + self.write_ciphertext(OBJECTS, &id, buf.expose()) + } + + fn write_record(&self, r: &Record) -> Result<()> { + let json = r.to_json().ok_or("record time outside the years 0 to 9999")?; + if json.len() > RECORD_SIZE - 4 { + crate::bail!("catalogue record too large"); + } + let mut plain = SecretVec::zeroed(RECORD_SIZE); + let p = plain.expose_mut(); + p[..4].copy_from_slice(&(json.len() as u32).to_be_bytes()); + p[4..4 + json.len()].copy_from_slice(&json); + self.entropy + .rng(&[b"recpad".as_slice(), &r.id.0].concat()) + .fill(&mut p[4 + json.len()..]); + self.write_ciphertext(RECORDS, &r.id, plain.expose()) + } + + fn read_record(&self, id: Id) -> std::result::Result { + let p = self.path(RECORDS, &id); + let f = std::fs::File::open(&p).map_err(|e| ReadRecordError::Other(crate::util::io_msg("open", &p, &e)))?; + let plain = match fury::decrypt_bounded(std::io::BufReader::new(f), &self.id, RECORD_SIZE) { + Ok(p) => p, + Err(fury::Error::NoIdentityMatch) => return Err(ReadRecordError::Foreign), + Err(fury::Error::TooLarge) => return Err(ReadRecordError::Other("wrong record size".into())), + Err(e) => return Err(ReadRecordError::Other(e.to_string())), + }; + let b = plain.expose(); + if b.len() != RECORD_SIZE { + return Err(ReadRecordError::Other("wrong record size".into())); + } + let n = u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize; + if n > RECORD_SIZE - 4 { + return Err(ReadRecordError::Other("malformed record".into())); + } + Record::from_json(&b[4..4 + n], id).map_err(ReadRecordError::Other) + } + + /// Decrypts object `id` and checks it leads with its own ID and holds + /// `part` bytes padded to their bucket. Returns the plaintext buffer. + fn read_object(&self, id: Id, part: u64, mut lease: Lease) -> Result { + let p = self.path(OBJECTS, &id); + let f = std::fs::File::open(&p).map_err(|e| Error::io("open", &p, &e))?; + let padded = padded_size(ID_SIZE as u64 + part) as usize; + match fury::decrypt_bounded_into(std::io::BufReader::new(f), &self.id, padded, lease.buf()) { + Ok(()) => {} + Err(fury::Error::TooLarge) => crate::bail!("object size does not match its catalogue record"), + Err(e) => return Err(Error::new(e.to_string())), + } + let b = lease.buf().expose(); + if b.len() < ID_SIZE || b[..ID_SIZE] != id.0 { + crate::bail!("object does not match its catalogue record"); + } + if b.len() != padded { + crate::bail!("object size does not match its catalogue record"); + } + Ok(lease) + } + + /// Removes `dir/` and re-pins its shard's time. For records the + /// shard is synced, so the removal is durable before any object goes. + /// A missing file is reported with [`Kind::NotFound`]. + fn remove_file(&self, dir: &str, id: &Id, sync: bool) -> Result<()> { + let p = self.path(dir, id); + match std::fs::remove_file(&p) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Err(Error::io("remove", &p, &e).with_kind(Kind::NotFound)); + } + Err(e) => return Err(Error::io("remove", &p, &e)), + } + let shard = p.parent().expect("sharded"); + pfs::set_epoch(shard).map_err(|e| Error::io("chtimes", shard, &e))?; + if sync && !self.no_sync { + pfs::sync_dir(shard).map_err(|e| Error::io("sync", shard, &e))?; + } + Ok(()) + } + + fn remove_record(&self, id: &Id) -> Result<()> { + self.remove_file(RECORDS, id, true).map_err(|e| e.with_kind(Kind::Other)) + } +} + +/// Entries of `dir`, sorted by name, each with whether it is a directory +/// (not following symlinks), as Go's `os.ReadDir`. +fn sorted_dir(dir: &Path) -> Result> { + let mut v = Vec::new(); + for e in std::fs::read_dir(dir).map_err(|e| Error::io("open", dir, &e))? { + let e = e.map_err(|e| Error::io("readdir", dir, &e))?; + let is_dir = e.file_type().map(|t| t.is_dir()).unwrap_or(false); + v.push((e.file_name(), is_dir)); + } + v.sort(); + Ok(v) +} + +fn mode_of(m: &std::fs::Metadata) -> u32 { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + m.permissions().mode() & 0o777 + } + #[cfg(not(unix))] + { + // Go reports 0o444 for read-only files and 0o666 otherwise. + if m.permissions().readonly() { 0o444 } else { 0o666 } + } +} + +/// Identity of a source file as read, so migrate deletes only that file. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub struct SourceStamp { + dev: u64, + ino: u64, + size: u64, + mtime: Option, +} + +impl SourceStamp { + pub fn of(m: &std::fs::Metadata) -> SourceStamp { + #[cfg(unix)] + let (dev, ino) = { + use std::os::unix::fs::MetadataExt; + (m.dev(), m.ino()) + }; + #[cfg(not(unix))] + let (dev, ino) = (0, 0); + SourceStamp { + dev, + ino, + size: m.len(), + mtime: m.modified().ok(), + } + } +} + +/// Reads exactly `size` bytes from a file, then insists on end of file, so +/// a file that grew while being added is caught. +struct EndCheck<'a, R: Read> { + inner: &'a mut R, + size: u64, + read: u64, +} + +impl Read for EndCheck<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + let want = (self.size - self.read).min(buf.len() as u64) as usize; + let n = if want == 0 { 0 } else { self.inner.read(&mut buf[..want])? }; + self.read += n as u64; + if self.read == self.size && (n > 0 || want == 0) { + let mut b = [0u8; 1]; + loop { + match self.inner.read(&mut b) { + Ok(0) => break, + Ok(_) => return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "source grew")), + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + } + Ok(n) + } +} + +/// Writes random entries sealed to a throwaway key, through the same code +/// as `add`: from 8 to 32 entries of log-uniformly random sizes, or more +/// until they total `target` bytes. Every vault therefore holds entries +/// nobody can open, so a file hidden from a decoy passphrase looks like +/// filler. Filler needs no per-file durability. +fn write_filler(root: &Path, target: u64, pool: Arc, entropy: Entropy) -> Result<()> { + let mut rng = entropy.rng(b"filler"); + let throwaway = Identity::generate(&mut rng); + let s = Store::new(root, throwaway, pool, true, entropy); + let upper = (target / 8).max(CHUNK_SIZE).min(1 << 30) as f64; + let count = FILLER_MIN + rng.below(FILLER_MAX - FILLER_MIN + 1); + let mut cat = Catalogue::default(); + let (mut i, mut total) = (0u64, 0u64); + while i < count || total < target { + let size = (1024.0 * (rng.f64() * (upper / 1024.0).ln()).exp()) as u64; + let name = VaultName::new(&hex(&rng.array::<8>())).expect("hex is a valid name"); + let mut src = RandomReader(s.inner.entropy.rng(&rng.array::<16>())); + s.add_reader(&mut cat, &mut src, size, 0o600, Instant::now(), &name)?; + total += size; + i += 1; + } + Ok(()) +} + +struct RandomReader(crate::rng::Rng); + +impl Read for RandomReader { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + self.0.fill(buf); + Ok(buf.len()) + } +} + +#[cfg(test)] +mod tests; diff --git a/src/store/tests.rs b/src/store/tests.rs new file mode 100644 index 0000000..2d73daa --- /dev/null +++ b/src/store/tests.rs @@ -0,0 +1,492 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +use super::*; +use crate::crypto::kem::SUITE_ID; +use std::sync::atomic::Ordering; + +pub(crate) struct TempDir(pub PathBuf); + +impl TempDir { + pub fn new(tag: &str) -> TempDir { + let r: [u8; 8] = crate::rng::os().array(); + let p = std::env::temp_dir().join(format!("vault-store-{tag}-{}-{}", std::process::id(), hex(&r))); + std::fs::create_dir_all(&p).unwrap(); + TempDir(p) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + // Read-only test directories must be writable again to delete. + #[cfg(unix)] + for e in walk(&self.0) { + use std::os::unix::fs::PermissionsExt; + if e.is_dir() { + let _ = std::fs::set_permissions(&e, std::fs::Permissions::from_mode(0o700)); + } + } + let _ = std::fs::remove_dir_all(&self.0); + } +} + +fn walk(p: &Path) -> Vec { + let mut out = vec![p.to_path_buf()]; + if let Ok(rd) = std::fs::read_dir(p) { + for e in rd.flatten() { + let path = e.path(); + if e.file_type().map(|t| t.is_dir()).unwrap_or(false) { + out.extend(walk(&path)); + } else { + out.push(path); + } + } + } + out +} + +fn files(p: &Path) -> Vec { + let mut v: Vec<_> = walk(p).into_iter().filter(|p| p.is_file()).collect(); + v.sort(); + v +} + +pub(crate) fn identity(n: u8) -> Identity { + Identity::from_suite_seed(SUITE_ID, &[n; 32]).unwrap() +} + +pub(crate) fn pool() -> Arc { + Arc::new(Pool::new(4)) +} + +/// A fresh vault (filler kept small but present) and a store on it. +fn vault(tag: &str) -> (TempDir, Store) { + let d = TempDir::new(tag); + let root = d.0.join("store"); + Store::init(&root, 0, pool()).unwrap(); + let s = Store::open(&root, identity(1), pool()).unwrap(); + (d, s) +} + +fn name(n: &str) -> VaultName { + VaultName::new(n).unwrap() +} + +fn add_bytes(s: &Store, cat: &mut Catalogue, n: &str, body: &[u8]) -> Added { + s.add_reader(cat, &mut &body[..], body.len() as u64, 0o640, Instant::now(), &name(n)) + .unwrap() +} + +fn get_bytes(s: &Store, rec: &Record, dir: &Path, out: &str) -> Result> { + let d = pfs::Dir::open(dir).unwrap(); + s.extract(rec, &d, OsStr::new(out))?; + Ok(std::fs::read(dir.join(out)).unwrap()) +} + +fn objects(s: &Store) -> Vec { + files(&s.root().join(OBJECTS)) +} + +#[test] +fn lifecycle() { + let (d, s) = vault("life"); + let mut cat = s.catalogue().unwrap(); + assert!(cat.is_empty(), "filler must be invisible"); + let want = vec![b'x'; 70000]; + assert!(!add_bytes(&s, &mut cat, "a/b", &want).replaced); + assert!(add_bytes(&s, &mut cat, "a/b", &want).replaced); + add_bytes(&s, &mut cat, "c", b"see"); + let cat2 = s.catalogue().unwrap(); + assert_eq!(cat2.names().collect::>(), ["a/b", "c"]); + assert_eq!(cat2.records("a/b").len(), 1, "replace removed the old record"); + let r = cat2.current("a/b").unwrap(); + assert_eq!((r.size, r.mode), (70000, 0o640)); + assert_eq!(get_bytes(&s, r, &d.0, "out").unwrap(), want); + let e = get_bytes(&s, r, &d.0, "out").unwrap_err(); + assert!(e.msg.ends_with("already exists"), "{e}"); + + // Swapping one object for another is detected, and nothing appears. + let (dr, cr) = (cat2.current("a/b").unwrap(), cat2.current("c").unwrap()); + std::fs::copy(s.path(OBJECTS, &cr.seed.object_id(0)), s.path(OBJECTS, &dr.seed.object_id(0))).unwrap(); + let e = get_bytes(&s, dr, &d.0, "swapped").unwrap_err(); + assert!(e.msg.contains("does not match"), "{e}"); + assert!(!d.0.join("swapped").exists()); + assert_eq!(std::fs::read_dir(&d.0).unwrap().count(), 2, "temporary file left behind"); +} + +#[test] +fn decoy_isolation_and_filler() { + let (_d, real) = vault("decoy"); + let decoy = Store::open(real.root(), identity(2), pool()).unwrap(); + for (s, n) in [(&real, "secret"), (&decoy, "shopping")] { + let mut cat = s.catalogue().unwrap(); + assert!(cat.is_empty()); + add_bytes(s, &mut cat, n, b"data"); + } + for (s, n) in [(&real, "secret"), (&decoy, "shopping")] { + let cat = s.catalogue().unwrap(); + assert_eq!(cat.names().collect::>(), [n]); + } + let records = files(&real.root().join(RECORDS)); + assert!( + records.len() as u64 >= FILLER_MIN + 2, + "only {} records; filler missing", + records.len() + ); + // Records are all one size, and every object is within the bucket range. + let sizes: HashSet = records.iter().map(|p| p.metadata().unwrap().len()).collect(); + assert_eq!(sizes.len(), 1, "records differ in size: {sizes:?}"); +} + +#[test] +fn filler_target() { + let d = TempDir::new("filler"); + let root = d.0.join("v"); + Store::init(&root, 24 << 20, pool()).unwrap(); + assert!(Store::init(&root, 0, pool()).is_err(), "init twice"); + let objs = files(&root.join(OBJECTS)); + let total: u64 = objs.iter().map(|p| p.metadata().unwrap().len()).sum(); + let largest = objs.iter().map(|p| p.metadata().unwrap().len()).max().unwrap(); + assert!(total >= 24 << 20, "filler objects total {total}"); + assert!(largest <= CHUNK_SIZE + CHUNK_SIZE / 64, "largest object {largest}"); + for i in 0..256 { + for dir in [RECORDS, OBJECTS] { + let p = root.join(dir).join(format!("{i:02x}")); + assert_eq!(p.metadata().unwrap().modified().unwrap(), std::time::UNIX_EPOCH, "{}", p.display()); + } + } + for f in files(&root.join(RECORDS)).iter().chain(&objs) { + assert_eq!(f.metadata().unwrap().modified().unwrap(), std::time::UNIX_EPOCH); + assert!(f.parent().unwrap().file_name().unwrap().to_str().unwrap() == &f.file_name().unwrap().to_str().unwrap()[..2]); + } + assert_eq!(std::fs::read(root.join(MARKER_NAME)).unwrap(), MARKER_CONTENT); +} + +#[test] +fn chunks_and_reordering() { + let (d, s) = vault("chunks"); + let before = objects(&s).len(); + let mut want = vec![0u8; 2 * CHUNK_DATA as usize + 100]; + crate::rng::os().fill(&mut want); + let mut cat = s.catalogue().unwrap(); + add_bytes(&s, &mut cat, "big", &want); + let r = cat.current("big").unwrap(); + let sizes: Vec = (0..object_count(r.size)) + .map(|i| s.path(OBJECTS, &r.seed.object_id(i)).metadata().unwrap().len()) + .collect(); + assert!(sizes.len() == 3 && sizes[0] == sizes[1] && sizes[2] < sizes[0], "{sizes:?}"); + assert_eq!(objects(&s).len() - before, 3); + assert_eq!(get_bytes(&s, r, &d.0, "out").unwrap(), want); + let (a, b) = (s.path(OBJECTS, &r.seed.object_id(0)), s.path(OBJECTS, &r.seed.object_id(1))); + let (ad, bd) = (std::fs::read(&a).unwrap(), std::fs::read(&b).unwrap()); + std::fs::write(&a, &bd).unwrap(); + std::fs::write(&b, &ad).unwrap(); + assert!(get_bytes(&s, r, &d.0, "swapped").is_err()); + assert!(!d.0.join("swapped").exists()); + s.remove_all(&mut cat, &["big".to_string()], &mut |_| {}).unwrap(); + assert_eq!(objects(&s).len(), before, "rm freed the objects"); +} + +#[test] +fn boundaries_round_trip() { + let (d, s) = vault("bounds"); + let mut cat = s.catalogue().unwrap(); + let c = CHUNK_DATA as usize; + for (i, n) in [0, 1, 65535 - 16, 65536 - 16, 65536 - 15, 65536, c - 1, c, c + 1] + .into_iter() + .enumerate() + { + let body: Vec = (0..n).map(|j| (j * 31 + i) as u8).collect(); + add_bytes(&s, &mut cat, &format!("f{i}"), &body); + let r = cat.current(&format!("f{i}")).unwrap(); + assert_eq!(get_bytes(&s, r, &d.0, &format!("o{i}")).unwrap(), body, "size {n}"); + // Each object is padded: one bucket per object, all within range. + for k in 0..object_count(n as u64) { + let len = s.path(OBJECTS, &r.seed.object_id(k)).metadata().unwrap().len(); + assert!( + len > MIN_PADDING && len <= CHUNK_SIZE + (32 << 10), + "object of {len} bytes for size {n}" + ); + } + } +} + +#[test] +fn remove_tree_and_copy_move() { + let (d, s) = vault("tree"); + let before = objects(&s).len(); + let mut cat = s.catalogue().unwrap(); + let mut big = vec![0u8; CHUNK_DATA as usize + 1]; + crate::rng::os().fill(&mut big); + for (n, b) in [("a/x", &big[..]), ("a/b/y", b"y"), ("ab/z", b"z"), ("b/w", b"w")] { + add_bytes(&s, &mut cat, n, b); + } + assert_eq!(objects(&s).len() - before, 5); + assert_eq!(cat.under("a"), ["a/b/y", "a/x"]); + // cp -r a c, then rm -r a: the copies keep the shared objects. + let pairs: Vec<(String, VaultName)> = cat + .under("a") + .into_iter() + .map(|n| (n.clone(), name(&format!("c{}", &n[1..])))) + .collect(); + let mut copied = Vec::new(); + s.copy_all(&mut cat, &pairs, false, &mut |f, t| copied.push((f.to_string(), t.to_string()))) + .unwrap(); + assert_eq!(copied.len(), 2); + assert_eq!(objects(&s).len() - before, 5, "copies share objects"); + let mut done = Vec::new(); + let names = cat.under("a"); + s.remove_all(&mut cat, &names, &mut |n| done.push(n.to_string())).unwrap(); + assert_eq!(done, ["a/b/y", "a/x"]); + assert_eq!(objects(&s).len() - before, 5, "objects still used by the copies stay"); + let fresh = s.catalogue().unwrap(); + assert_eq!(fresh.names().collect::>(), ["ab/z", "b/w", "c/b/y", "c/x"]); + assert_eq!(get_bytes(&s, fresh.current("c/x").unwrap(), &d.0, "x").unwrap(), big); + // mv keeps one copy of each object; rm of the last name frees them. + let mut cat = fresh; + s.copy_all(&mut cat, &[("c/x".into(), name("moved"))], true, &mut |_, _| {}) + .unwrap(); + assert!(!cat.contains("c/x") && cat.contains("moved")); + let names: Vec = cat.names().map(String::from).collect(); + s.remove_all(&mut cat, &names, &mut |_| {}).unwrap(); + assert_eq!(objects(&s).len(), before, "space freed"); +} + +/// The Go oracle's fixture vault opens: real files decrypt to the same +/// bytes and metadata, the decoy sees only its file, the foreign record is +/// skipped silently. +#[test] +fn reads_go_vault() { + let d = TempDir::new("govault"); + let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/mini-vault"); + let root = d.0.join("v"); + for p in walk(&src) { + let rel = p.strip_prefix(&src).unwrap(); + if p.is_dir() { + std::fs::create_dir_all(root.join(rel)).unwrap(); + } else { + std::fs::copy(&p, root.join(rel)).unwrap(); + } + } + for dir in [RECORDS, OBJECTS] { + for i in 0..256 { + std::fs::create_dir_all(root.join(dir).join(format!("{i:02x}"))).unwrap(); + } + } + let key = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/key-with-decoy.pem"); + let real = crate::keyfile::unlock(&key, b"oracle-real passphrase").unwrap(); + let s = Store::open(&root, real, pool()).unwrap(); + let cat = s.catalogue().unwrap(); + assert_eq!(cat.names().collect::>(), ["dir/empty", "dir/ünïcode.bin", "hello.txt"]); + let boundary: Vec = (0..(MIN_PADDING as usize - ID_SIZE)).map(|i| (i * 7) as u8).collect(); + for (n, body, mode, secs, nanos) in [ + ("hello.txt", &b"hello"[..], 0o644, 1700000000, 123456789), + ("dir/empty", b"", 0o600, 1600000000, 0), + ("dir/ünïcode.bin", &boundary[..], 0o755, 1500000000, 999999999), + ] { + let r = cat.current(n).unwrap(); + assert_eq!((r.mode, r.modified), (mode, Instant { secs, nanos }), "{n}"); + let out = format!("out-{}", r.id.hex()); + assert_eq!(get_bytes(&s, r, &d.0, &out).unwrap(), body); + let m = d.0.join(&out).metadata().unwrap(); + assert_eq!(m.modified().unwrap(), Instant { secs, nanos }.to_system()); + #[cfg(unix)] + assert_eq!(std::os::unix::fs::PermissionsExt::mode(&m.permissions()) & 0o777, mode); + } + let decoy = crate::keyfile::unlock(&key, b"oracle-decoy passphrase").unwrap(); + let cat = Store::open(&root, decoy, pool()).unwrap().catalogue().unwrap(); + assert_eq!(cat.names().collect::>(), ["shopping.txt"]); +} + +/// With identical synthetic randomness, one worker and eight produce +/// byte-identical objects. +#[test] +fn sequential_and_parallel_identical() { + let mut outputs = Vec::new(); + for workers in [1, 8] { + let d = TempDir::new("det"); + let root = d.0.join("v"); + Store::init(&root, 0, pool()).unwrap(); + let s = Store::open_with(&root, identity(3), Arc::new(Pool::new(workers)), Entropy::Seeded([7; 32])); + let before: HashSet = objects(&s).into_iter().collect(); + let body: Vec = (0..3 * CHUNK_DATA as usize + 12345).map(|i| (i % 251) as u8).collect(); + let mut cat = s.catalogue().unwrap(); + add_bytes(&s, &mut cat, "f", &body); + let new: Vec<(String, Vec)> = objects(&s) + .into_iter() + .filter(|p| !before.contains(p)) + .map(|p| (p.file_name().unwrap().to_string_lossy().into_owned(), std::fs::read(&p).unwrap())) + .collect(); + assert_eq!(new.len(), 4); + outputs.push(new); + } + assert!(outputs[0] == outputs[1], "sequential and parallel output differ"); +} + +/// A failure while writing objects stores nothing and leaves no objects. +#[cfg(unix)] +#[test] +fn object_write_failure_rolls_back() { + use std::os::unix::fs::PermissionsExt; + let d = TempDir::new("fail"); + let root = d.0.join("v"); + Store::init(&root, 0, pool()).unwrap(); + let entropy = Entropy::Seeded([9; 32]); + let s = Store::open_with(&root, identity(4), pool(), entropy.clone()); + let seed = ObjectSeed::random(&mut entropy.rng(b"seed")); + let blocked = s.path(OBJECTS, &seed.object_id(2)); + let shard = blocked.parent().unwrap().to_path_buf(); + if seed.object_id(0).shard() == seed.object_id(2).shard() || seed.object_id(1).shard() == seed.object_id(2).shard() { + return; // the fixed seed puts two objects in one shard; nothing to test + } + std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o500)).unwrap(); + if std::fs::write(shard.join("probe"), b"").is_ok() { + return; // running as root: permissions do not bite + } + let before = objects(&s); + let mut cat = s.catalogue().unwrap(); + let body = vec![1u8; 3 * CHUNK_DATA as usize]; + assert!( + s.add_reader(&mut cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name("f")) + .is_err() + ); + std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o700)).unwrap(); + assert_eq!(objects(&s), before, "objects of a failed add were left behind"); + assert!(s.catalogue().unwrap().is_empty()); + assert!( + walk(&root) + .iter() + .all(|p| !p.file_name().unwrap().to_string_lossy().starts_with(".pending-")) + ); +} + +/// A short source stores nothing; a longer one than declared is refused. +#[test] +fn short_and_long_sources() { + let (d, s) = vault("short"); + let before = objects(&s); + let mut cat = s.catalogue().unwrap(); + let e = s + .add_reader(&mut cat, &mut &b"abc"[..], 10, 0o600, Instant::now(), &name("f")) + .unwrap_err(); + assert_eq!(e.kind, Kind::SourceShort); + assert_eq!(objects(&s), before); + let p = d.0.join("src"); + std::fs::write(&p, b"0123456789").unwrap(); + s.add_file(&mut cat, &p, &name("ok")).unwrap(); + assert!(s.catalogue().unwrap().contains("ok")); +} + +/// A record whose sync fails after it is in place keeps its objects. +#[test] +fn failure_after_publish_keeps_objects() { + let (d, s) = vault("afterpub"); + let mut cat = s.catalogue().unwrap(); + s.inner.fail_after_publish.store(true, Ordering::SeqCst); + let e = s + .add_reader(&mut cat, &mut &b"precious"[..], 8, 0o600, Instant::now(), &name("f")) + .unwrap_err(); + assert_eq!(e.kind, Kind::AfterPublish); + s.inner.fail_after_publish.store(false, Ordering::SeqCst); + let cat = s.catalogue().unwrap(); + assert_eq!(get_bytes(&s, cat.current("f").unwrap(), &d.0, "out").unwrap(), b"precious"); +} + +/// Tampered, truncated, extended and foreign objects never publish output. +#[test] +fn damaged_objects_never_publish() { + let (d, s) = vault("damage"); + let mut cat = s.catalogue().unwrap(); + let body: Vec = (0..100_000).map(|i| i as u8).collect(); + add_bytes(&s, &mut cat, "f", &body); + let r = cat.current("f").unwrap(); + let obj = s.path(OBJECTS, &r.seed.object_id(0)); + let orig = std::fs::read(&obj).unwrap(); + let foreign = { + let mut v = Vec::new(); + fury::encrypt_to(&mut v, &identity(9).recipient(), &[0u8; 1 << 17], &mut crate::rng::os()); + v + }; + // Same key and ID, but padding to the wrong bucket. + let short_pad = { + let mut p = r.seed.object_id(0).0.to_vec(); + p.extend_from_slice(&body); + let mut v = Vec::new(); + fury::encrypt_to(&mut v, &s.inner.recipient, &p, &mut crate::rng::os()); + v + }; + let damaged: Vec<(&str, Vec)> = vec![ + ("flipped", { + let mut v = orig.clone(); + let n = v.len() - 100; + v[n] ^= 1; + v + }), + ("truncated", orig[..orig.len() - 1].to_vec()), + ("missing final chunk", orig[..orig.len() - 1000].to_vec()), + ("trailing", [&orig[..], b"x"].concat()), + ("empty", Vec::new()), + ("foreign", foreign), + ("short padding", short_pad), + ]; + for (what, bytes) in damaged { + std::fs::write(&obj, &bytes).unwrap(); + let out = format!("out-{}", what.replace(' ', "-")); + assert!(get_bytes(&s, r, &d.0, &out).is_err(), "{what} accepted"); + assert!(!d.0.join(&out).exists(), "{what} published output"); + } + assert_eq!(std::fs::read_dir(&d.0).unwrap().count(), 1, "temporary files left behind"); + std::fs::remove_file(&obj).unwrap(); + assert!(get_bytes(&s, r, &d.0, "missing").is_err()); +} + +#[cfg(unix)] +#[test] +fn extraction_never_replaces_symlinks() { + let (d, s) = vault("symlink"); + let mut cat = s.catalogue().unwrap(); + add_bytes(&s, &mut cat, "f", b"data"); + let r = cat.current("f").unwrap(); + std::os::unix::fs::symlink(d.0.join("target"), d.0.join("dangling")).unwrap(); + assert!(get_bytes(&s, r, &d.0, "dangling").is_err()); + assert!(!d.0.join("target").exists(), "followed a dangling symlink"); + std::fs::write(d.0.join("exists"), b"keep").unwrap(); + assert!(get_bytes(&s, r, &d.0, "exists").is_err()); + assert_eq!(std::fs::read(d.0.join("exists")).unwrap(), b"keep"); +} + +#[test] +fn oversized_record_names() { + let (_d, s) = vault("names"); + let mut cat = s.catalogue().unwrap(); + // 1024 bytes is the limit; '<' is escaped six-fold in JSON, as in Go. + let long = "a".repeat(NAME_LIMIT); + add_bytes(&s, &mut cat, &long, b"x"); + let escaped = "<".repeat(NAME_LIMIT); + let e = s + .add_reader(&mut cat, &mut &b"x"[..], 1, 0o600, Instant::now(), &name(&escaped)) + .unwrap_err(); + assert_eq!(e.msg, "catalogue record too large"); + assert!(VaultName::new(&"a".repeat(NAME_LIMIT + 1)).is_none()); + assert_eq!(s.catalogue().unwrap().names().collect::>(), [long.as_str()]); +} + +#[test] +fn duplicate_records_recover() { + let (_d, s) = vault("dups"); + let mut cat = s.catalogue().unwrap(); + add_bytes(&s, &mut cat, "f", b"old"); + // An interrupted replace leaves two records for one name. + let old = cat.current("f").unwrap(); + let mut dup = old.copy_as(name("f"), Id::random(&mut crate::rng::os())); + dup.added = Instant { + secs: old.added.secs + 10, + nanos: 0, + }; + s.inner.write_record(&dup).unwrap(); + let cat = s.catalogue().unwrap(); + assert_eq!(cat.records("f").len(), 2); + assert_eq!(cat.current("f").unwrap().added, dup.added, "newest first"); + let mut cat = cat; + add_bytes(&s, &mut cat, "f", b"new"); + assert_eq!(s.catalogue().unwrap().records("f").len(), 1, "next write cleans up"); +} diff --git a/src/store/types.rs b/src/store/types.rs new file mode 100644 index 0000000..1f36666 --- /dev/null +++ b/src/store/types.rs @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! Validated values that reach paths, the catalogue and extraction. +//! Each is constructed only through a checking constructor, so raw +//! untrusted strings cannot be used where one of these is expected. + +use crate::secret::SecretArray; +use crate::util::Instant; +use hmac::{KeyInit, Mac}; +use zeroize::Zeroizing; + +pub const NAME_LIMIT: usize = 1024; +pub const FILE_LIMIT: u64 = 1 << 40; +pub const ID_SIZE: usize = 16; +pub const SEED_SIZE: usize = 32; +pub const RECORD_SIZE: usize = 4096; +pub const MIN_PADDING: u64 = 64 << 10; +/// The largest bucket, and so the largest object plaintext. +pub const CHUNK_SIZE: u64 = MIN_PADDING << 6; +/// File bytes per object: an object's plaintext starts with its 16-byte ID. +pub const CHUNK_DATA: u64 = CHUNK_SIZE - ID_SIZE as u64; + +/// A name inside the vault: non-empty UTF-8 of at most 1024 bytes, already +/// clean in the sense of Go's `path.Clean`, relative, not `.`/`..` or under +/// `..`, without backslashes or control characters. +#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct VaultName(String); + +impl VaultName { + pub fn new(name: &str) -> Option { + valid_name(name).then(|| VaultName(name.to_string())) + } + + pub fn from_bytes(b: &[u8]) -> Option { + VaultName::new(std::str::from_utf8(b).ok()?) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + + /// Path components, each a plain non-empty segment. + pub fn components(&self) -> impl Iterator { + self.0.split('/') + } +} + +impl std::fmt::Display for VaultName { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +pub fn valid_name(name: &str) -> bool { + !name.is_empty() + && name.len() <= NAME_LIMIT + && crate::util::clean(name) == name + && !name.starts_with('/') + && name != "." + && name != ".." + && !name.starts_with("../") + && !name.contains('\\') + && !name.chars().any(char::is_control) +} + +/// A 16-byte object or record ID, shown and stored as lowercase hex. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct Id(pub [u8; ID_SIZE]); + +impl Id { + pub fn random(rng: &mut crate::rng::Rng) -> Id { + Id(rng.array()) + } + + /// Only exact lowercase hex of the right length, as Go's `validHex`. + pub fn parse(s: &str) -> Option { + let b = parse_hex(s)?; + Some(Id(b.try_into().ok()?)) + } + + pub fn hex(&self) -> String { + hex(&self.0) + } + + pub fn shard(&self) -> String { + format!("{:02x}", self.0[0]) + } +} + +pub fn hex(b: &[u8]) -> String { + const D: &[u8; 16] = b"0123456789abcdef"; + let mut s = String::with_capacity(b.len() * 2); + for x in b { + s.push(D[(x >> 4) as usize] as char); + s.push(D[(x & 15) as usize] as char); + } + s +} + +pub fn parse_hex(s: &str) -> Option> { + let b = s.as_bytes(); + if b.len() % 2 != 0 { + return None; + } + let v = |c: u8| match c { + b'0'..=b'9' => Some(c - b'0'), + b'a'..=b'f' => Some(c - b'a' + 10), + _ => None, + }; + b.chunks(2).map(|p| Some(v(p[0])? << 4 | v(p[1])?)).collect() +} + +/// The secret per-file seed whose HMAC derives the file's object IDs. +pub struct ObjectSeed(SecretArray); + +impl ObjectSeed { + pub fn random(rng: &mut crate::rng::Rng) -> ObjectSeed { + let mut s = SecretArray::zero(); + rng.fill(s.expose_mut()); + ObjectSeed(s) + } + + pub fn parse(s: &str) -> Option { + let b = Zeroizing::new(parse_hex(s)?); + SecretArray::from_slice(&b).map(ObjectSeed) + } + + pub fn hex(&self) -> Zeroizing { + Zeroizing::new(hex(self.0.expose())) + } + + /// An explicit copy, for a record copied under another name. + pub fn duplicate(&self) -> ObjectSeed { + ObjectSeed(SecretArray::from_slice(self.0.expose()).expect("same size")) + } + + pub fn same_as(&self, other: &ObjectSeed) -> bool { + crate::secret::subtle_eq::ct_eq(self.0.expose(), other.0.expose()) + } + + /// The `i`th object's ID: HMAC-SHA-256(seed, big-endian i)[:16]. + pub fn object_id(&self, i: u64) -> Id { + let mut m = as KeyInit>::new_from_slice(self.0.expose()).expect("any key size"); + m.update(&(i as i64).to_be_bytes()); + let out = m.finalize().into_bytes(); + let mut id = [0u8; ID_SIZE]; + id.copy_from_slice(&out[..ID_SIZE]); + Id(id) + } +} + +/// How many objects hold a file of `size` bytes: at least one. +pub fn object_count(size: u64) -> u64 { + size.div_ceil(CHUNK_DATA).max(1) +} + +/// `size` rounded up to the next power-of-two bucket of at least 64 KiB. +pub fn padded_size(size: u64) -> u64 { + let mut p = MIN_PADDING; + while p < size { + p *= 2; + } + p +} + +/// A decrypted, validated catalogue record. Only the store constructs +/// these, from authenticated plaintext or from its own writes. +pub struct Record { + pub name: VaultName, + pub seed: ObjectSeed, + pub size: u64, + pub mode: u32, + pub modified: Instant, + pub added: Instant, + /// The record file's ID; not stored inside the record. + pub id: Id, +} + +impl Record { + /// The bytes of object `i` that are file data, after its ID prefix. + pub fn part(&self, i: u64) -> u64 { + let before = i * CHUNK_DATA; + (self.size - before).min(CHUNK_DATA) + } + + /// A copy under another name, sharing the objects. + pub fn copy_as(&self, name: VaultName, id: Id) -> Record { + Record { + name, + seed: self.seed.duplicate(), + size: self.size, + mode: self.mode, + modified: self.modified, + added: self.added, + id, + } + } + + /// The record's JSON exactly as Go's `encoding/json` writes it, or None + /// if a timestamp is outside what Go can encode. + pub fn to_json(&self) -> Option>> { + let mut o = Zeroizing::new(Vec::with_capacity(256)); + o.extend_from_slice(b"{\"name\":"); + go_json_string(&mut o, &self.name.0); + o.extend_from_slice(b",\"seed\":\""); + o.extend_from_slice(self.seed.hex().as_bytes()); + o.extend_from_slice(format!("\",\"size\":{},\"mode\":{},\"modified\":\"", self.size, self.mode).as_bytes()); + o.extend_from_slice(self.modified.to_rfc3339()?.as_bytes()); + o.extend_from_slice(b"\",\"added\":\""); + o.extend_from_slice(self.added.to_rfc3339()?.as_bytes()); + o.extend_from_slice(b"\"}"); + Some(o) + } + + /// Parses and validates record JSON. Stricter than Go in one way: field + /// names must match exactly and every field must be present, as Go + /// always writes them. + pub fn from_json(b: &[u8], id: Id) -> Result { + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct Raw { + name: String, + seed: Zeroizing, + size: i64, + mode: u32, + modified: String, + added: String, + } + let r: Raw = serde_json::from_slice(b).map_err(|e| e.to_string())?; + let malformed = || "malformed record".to_string(); + let name = VaultName::new(&r.name).ok_or_else(malformed)?; + let seed = ObjectSeed::parse(&r.seed).ok_or_else(malformed)?; + if r.size < 0 || r.size as u64 > FILE_LIMIT { + return Err(malformed()); + } + let modified = Instant::parse_rfc3339(&r.modified).ok_or_else(|| format!("parsing time {:?}", r.modified))?; + let added = Instant::parse_rfc3339(&r.added).ok_or_else(|| format!("parsing time {:?}", r.added))?; + Ok(Record { + name, + seed, + size: r.size as u64, + mode: r.mode, + modified, + added, + id, + }) + } +} + +/// Go's JSON string encoding with HTML escaping (its default). +fn go_json_string(o: &mut Vec, s: &str) { + o.push(b'"'); + for c in s.chars() { + match c { + '"' => o.extend_from_slice(b"\\\""), + '\\' => o.extend_from_slice(b"\\\\"), + '\n' => o.extend_from_slice(b"\\n"), + '\r' => o.extend_from_slice(b"\\r"), + '\t' => o.extend_from_slice(b"\\t"), + '\u{8}' => o.extend_from_slice(b"\\b"), + '\u{c}' => o.extend_from_slice(b"\\f"), + '<' | '>' | '&' | '\u{2028}' | '\u{2029}' => o.extend_from_slice(format!("\\u{:04x}", c as u32).as_bytes()), + c if (c as u32) < 0x20 => o.extend_from_slice(format!("\\u{:04x}", c as u32).as_bytes()), + c => { + let mut b = [0u8; 4]; + o.extend_from_slice(c.encode_utf8(&mut b).as_bytes()); + } + } + } + o.push(b'"'); +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Value; + + fn v() -> Value { + serde_json::from_str(include_str!("../../tests/vectors/store.json")).unwrap() + } + + #[test] + fn names_match_go() { + for (h, want) in v()["valid_name_hex"].as_object().unwrap() { + let b = parse_hex(h).unwrap(); + assert_eq!( + VaultName::from_bytes(&b).is_some(), + want.as_bool().unwrap(), + "{:?}", + String::from_utf8_lossy(&b) + ); + } + } + + #[test] + fn object_ids_and_sizes_match_go() { + let v = v(); + let o = &v["object_ids"]; + let seed = ObjectSeed::parse(o["seed"].as_str().unwrap()).unwrap(); + for (i, id) in o["ids"].as_array().unwrap().iter().enumerate() { + assert_eq!(seed.object_id(i as u64).hex(), id.as_str().unwrap()); + } + assert_eq!( + seed.object_id(o["big_index_value"].as_u64().unwrap()).hex(), + o["big_index"].as_str().unwrap() + ); + for (n, want) in v["padded_size"].as_object().unwrap() { + assert_eq!(padded_size(n.parse().unwrap()), want.as_u64().unwrap(), "paddedSize({n})"); + } + for (n, want) in v["object_count"].as_object().unwrap() { + assert_eq!(object_count(n.parse().unwrap()), want.as_u64().unwrap(), "objectCount({n})"); + } + } + + #[test] + fn record_json_matches_go() { + for r in v()["records"].as_array().unwrap() { + let json = r["json"].as_str().unwrap().as_bytes(); + let n = r["name"].as_str().unwrap(); + // The name's escaping, whether or not it is a valid vault name. + let mut enc = Vec::new(); + go_json_string(&mut enc, n); + let text = r["json"].as_str().unwrap(); + assert_eq!( + &enc[..], + text["{\"name\":".len()..text.find(",\"seed\"").unwrap()].as_bytes(), + "escaping of {n:?}" + ); + if VaultName::new(n).is_none() { + assert!(Record::from_json(json, Id([0; 16])).is_err()); + continue; + } + let rec = Record::from_json(json, Id([0; 16])).unwrap(); + assert_eq!(rec.name.as_str(), r["name"].as_str().unwrap()); + // Go's UnixNano wraps outside about 1678-2262; so does this. + let ns = |i: Instant| i.secs.wrapping_mul(1_000_000_000).wrapping_add(i.nanos as i64); + assert_eq!(ns(rec.modified), r["modified_unix_nano"].as_i64().unwrap()); + assert_eq!(ns(rec.added), r["added_unix_nano"].as_i64().unwrap()); + assert_eq!(&rec.to_json().unwrap()[..], json, "byte-identical encoding"); + } + } + + #[test] + fn record_json_rejects_bad_records() { + let good = br#"{"name":"a","seed":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","size":1,"mode":420,"modified":"2020-01-01T00:00:00Z","added":"2020-01-01T00:00:00Z"}"#; + assert!(Record::from_json(good, Id([0; 16])).is_ok()); + let s = std::str::from_utf8(good).unwrap(); + for bad in [ + s.replace("\"a\"", "\"../a\""), + s.replace("\"a\"", "\"\""), + s.replace("0001", "0X01"), + s.replace("0001", "0A01"), + s.replace("1e1f", "1e"), + s.replace("\"size\":1", "\"size\":-1"), + s.replace("\"size\":1", "\"size\":1099511627777"), + s.replace("\"size\":1", "\"size\":1.5"), + s.replace("\"mode\":420", "\"mode\":-1"), + s.replace("}", ",\"extra\":1}"), + s.replace("\"name\"", "\"Name\""), + s.replace("2020-01-01T00:00:00Z\",\"added", "yesterday\",\"added"), + s.replace(",\"mode\":420", ""), + format!("{s}trailing"), + ] { + assert!(Record::from_json(bad.as_bytes(), Id([0; 16])).is_err(), "{bad}"); + } + } + + #[test] + fn hex_is_strict() { + assert!(Id::parse("00112233445566778899aabbccddeeff").is_some()); + assert!(Id::parse("00112233445566778899AABBCCDDEEFF").is_none()); + assert!(Id::parse("00112233445566778899aabbccddeef").is_none()); + assert!(Id::parse("00112233445566778899aabbccddeeff00").is_none()); + } +}