Rewrite vault in Rust (format-compatible with 0.5) #1
Generated
+1
@@ -1236,6 +1236,7 @@ version = "1.9.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e13084392c5e4bc371903e2935a5eaeed24905a7511356b883835e18a78f6879"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"zeroize_derive",
|
||||
]
|
||||
|
||||
|
||||
+1
-1
@@ -17,7 +17,7 @@ hmac = "0.13"
|
||||
sha2 = { version = "0.11", default-features = false }
|
||||
sha3 = { version = "0.11", default-features = false }
|
||||
ml-kem = { version = "0.3.2", features = ["zeroize", "hazmat"] }
|
||||
zeroize = { version = "1.9", features = ["derive"] }
|
||||
zeroize = { version = "1.9", features = ["derive", "serde"] }
|
||||
getrandom = "0.4"
|
||||
base64 = "0.23"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
|
||||
+26
-4
@@ -130,6 +130,29 @@ pub fn decrypt_reader<R: Read>(src: R, id: &Identity) -> Result<StreamReader<imp
|
||||
/// reading no more than that much ciphertext. The result is fully
|
||||
/// authenticated, final chunk and end of input included.
|
||||
pub fn decrypt_bounded<R: Read>(src: R, id: &Identity, max_plaintext: usize) -> Result<SecretVec, Error> {
|
||||
let mut buf = SecretVec::with_capacity(max_ciphertext(max_plaintext) + 1);
|
||||
decrypt_bounded_into(src, id, max_plaintext, &mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// The largest STREAM payload holding at most `max_plaintext` bytes.
|
||||
pub fn max_ciphertext(max_plaintext: usize) -> usize {
|
||||
max_plaintext + max_plaintext.div_ceil(stream::CHUNK_SIZE).max(1) * stream::TAG_SIZE
|
||||
}
|
||||
|
||||
/// As [`decrypt_bounded`], into a caller-supplied buffer whose capacity
|
||||
/// must be at least `max_ciphertext(max_plaintext) + 1`. On error the
|
||||
/// buffer is left empty.
|
||||
pub fn decrypt_bounded_into<R: Read>(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> {
|
||||
buf.clear();
|
||||
let r = decrypt_into(src, id, max_plaintext, buf);
|
||||
if r.is_err() {
|
||||
buf.clear();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn decrypt_into<R: Read>(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> {
|
||||
let (h, mut payload) = header::parse(src).map_err(Error::Header)?;
|
||||
let fk = file_key(&h, id)?;
|
||||
let mut nonce = [0u8; NONCE_SIZE];
|
||||
@@ -137,16 +160,15 @@ pub fn decrypt_bounded<R: Read>(src: R, id: &Identity, max_plaintext: usize) ->
|
||||
.read_exact(&mut nonce)
|
||||
.map_err(|e| Error::Io(io::Error::new(e.kind(), format!("failed to read nonce: {e}"))))?;
|
||||
let key = payload_key(fk.expose(), &nonce);
|
||||
let chunks = max_plaintext.div_ceil(stream::CHUNK_SIZE).max(1);
|
||||
let max_ct = max_plaintext + chunks * stream::TAG_SIZE;
|
||||
let mut buf = SecretVec::zeroed(max_ct + 1);
|
||||
let max_ct = max_ciphertext(max_plaintext);
|
||||
buf.resize_within_capacity(max_ct + 1);
|
||||
let n = stream::read_full(&mut payload, buf.expose_mut())?;
|
||||
if n > max_ct {
|
||||
return Err(Error::TooLarge);
|
||||
}
|
||||
let plain = stream::open_all(key.expose(), &mut buf.expose_mut()[..n]).map_err(|e| Error::Io(e.into()))?;
|
||||
buf.resize_within_capacity(plain);
|
||||
Ok(buf)
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -185,7 +185,7 @@ impl<W: Write> Drop for StreamWriter<W> {
|
||||
}
|
||||
|
||||
/// Reads until `buf` is full or the source ends; returns the count.
|
||||
pub fn read_full(src: &mut impl Read, buf: &mut [u8]) -> io::Result<usize> {
|
||||
pub fn read_full<R: Read + ?Sized>(src: &mut R, buf: &mut [u8]) -> io::Result<usize> {
|
||||
let mut n = 0;
|
||||
while n < buf.len() {
|
||||
match src.read(&mut buf[n..]) {
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The command-level error: a lower-case message without a trailing full
|
||||
//! stop, printed as `vault: MESSAGE`.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Error {
|
||||
pub msg: String,
|
||||
pub kind: Kind,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Other,
|
||||
/// A source ended before its declared size.
|
||||
SourceShort,
|
||||
/// A file was written under its final name, but a later durability step
|
||||
/// failed: what it depends on must not be deleted.
|
||||
AfterPublish,
|
||||
/// The thing to remove or read was not there.
|
||||
NotFound,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn new(msg: impl Into<String>) -> Error {
|
||||
Error {
|
||||
msg: msg.into(),
|
||||
kind: Kind::Other,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_kind(mut self, kind: Kind) -> Error {
|
||||
self.kind = kind;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn io(op: &str, path: &std::path::Path, e: &std::io::Error) -> Error {
|
||||
Error::new(crate::util::io_msg(op, path, e))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.msg)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl From<String> for Error {
|
||||
fn from(s: String) -> Self {
|
||||
Error::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Error {
|
||||
fn from(s: &str) -> Self {
|
||||
Error::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Error::new(crate::util::io_reason(&e))
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! bail {
|
||||
($($t:tt)*) => { return Err($crate::error::Error::new(format!($($t)*))) };
|
||||
}
|
||||
@@ -7,8 +7,11 @@
|
||||
#![deny(unsafe_op_in_unsafe_fn)]
|
||||
|
||||
pub mod crypto;
|
||||
pub mod error;
|
||||
pub mod keyfile;
|
||||
pub mod pipeline;
|
||||
pub mod platform;
|
||||
pub mod rng;
|
||||
pub mod secret;
|
||||
pub mod store;
|
||||
pub mod util;
|
||||
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! One bounded worker pool per process, and the bounded buffer pool that
|
||||
//! object I/O borrows from.
|
||||
//!
|
||||
//! The pool runs a fixed number of threads fed from a bounded queue, so
|
||||
//! submitting blocks once the queue is full (backpressure). Coordinators —
|
||||
//! the code that reads a source, writes an output in order, or collects
|
||||
//! results — always run on the caller's thread, never on a worker, so no
|
||||
//! worker ever waits for another and nested waiting cannot deadlock.
|
||||
//! Dropping the pool closes the queue and joins every thread; jobs already
|
||||
//! queued still run (each is bounded file I/O and crypto), and their
|
||||
//! results are dropped if nobody waits for them. A panicking job is turned
|
||||
//! into an error for whoever waits on it and does not take the worker down.
|
||||
|
||||
use crate::secret::SecretVec;
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// The largest worker count accepted from the user.
|
||||
pub const MAX_WORKERS: usize = 256;
|
||||
/// The default cap: past this, the work is bound by storage, not CPU.
|
||||
pub const DEFAULT_MAX_WORKERS: usize = 8;
|
||||
/// Object-sized buffers in flight at once for one file.
|
||||
pub const OBJECT_BUFFERS: usize = 8;
|
||||
|
||||
type Job = Box<dyn FnOnce() + Send + 'static>;
|
||||
|
||||
pub struct Pool {
|
||||
tx: Option<SyncSender<Job>>,
|
||||
threads: Vec<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
pub fn new(workers: usize) -> Pool {
|
||||
assert!((1..=MAX_WORKERS).contains(&workers), "worker count out of range");
|
||||
let (tx, rx) = sync_channel::<Job>(workers * 2);
|
||||
let rx = Arc::new(Mutex::new(rx));
|
||||
let threads = (0..workers)
|
||||
.map(|i| {
|
||||
let rx = Arc::clone(&rx);
|
||||
std::thread::Builder::new()
|
||||
.name(format!("vault-worker-{i}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
// Hold the lock only to take a job, not to run it.
|
||||
let job = match rx.lock() {
|
||||
Ok(r) => r.recv(),
|
||||
Err(p) => p.into_inner().recv(),
|
||||
};
|
||||
match job {
|
||||
Ok(job) => job(),
|
||||
Err(_) => return,
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("spawn worker thread")
|
||||
})
|
||||
.collect();
|
||||
Pool { tx: Some(tx), threads }
|
||||
}
|
||||
|
||||
/// The default worker count: available parallelism, at most 8.
|
||||
pub fn default_workers() -> usize {
|
||||
std::thread::available_parallelism().map_or(1, |n| n.get()).min(DEFAULT_MAX_WORKERS)
|
||||
}
|
||||
|
||||
pub fn workers(&self) -> usize {
|
||||
self.threads.len()
|
||||
}
|
||||
|
||||
/// Queues `f`, blocking while the queue is full.
|
||||
pub fn submit<T: Send + 'static>(&self, f: impl FnOnce() -> T + Send + 'static) -> Ticket<T> {
|
||||
let (rtx, rrx) = sync_channel(1);
|
||||
let job: Job = Box::new(move || {
|
||||
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|_| "internal error: a worker panicked".to_string());
|
||||
let _ = rtx.send(r);
|
||||
});
|
||||
self.tx.as_ref().expect("pool open").send(job).expect("workers alive");
|
||||
Ticket(rrx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Pool {
|
||||
fn drop(&mut self) {
|
||||
drop(self.tx.take());
|
||||
for t in self.threads.drain(..) {
|
||||
let _ = t.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The pending result of a submitted job.
|
||||
pub struct Ticket<T>(Receiver<Result<T, String>>);
|
||||
|
||||
impl<T> Ticket<T> {
|
||||
pub fn wait(self) -> Result<T, String> {
|
||||
self.0.recv().unwrap_or_else(|_| Err("internal error: a worker vanished".into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// A fixed set of reusable, equally sized secret buffers. Taking one blocks
|
||||
/// until one is free; a buffer is cleared when it comes back, and every
|
||||
/// buffer is cleared again when the pool is dropped.
|
||||
#[derive(Clone)]
|
||||
pub struct Buffers {
|
||||
tx: SyncSender<SecretVec>,
|
||||
rx: Arc<Mutex<Receiver<SecretVec>>>,
|
||||
}
|
||||
|
||||
impl Buffers {
|
||||
pub fn new(count: usize, capacity: usize) -> Buffers {
|
||||
let (tx, rx) = sync_channel(count);
|
||||
for _ in 0..count {
|
||||
tx.send(SecretVec::with_capacity(capacity)).expect("room for every buffer");
|
||||
}
|
||||
Buffers {
|
||||
tx,
|
||||
rx: Arc::new(Mutex::new(rx)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take(&self) -> Lease {
|
||||
let b = match self.rx.lock() {
|
||||
Ok(r) => r.recv(),
|
||||
Err(p) => p.into_inner().recv(),
|
||||
}
|
||||
.expect("buffer pool alive");
|
||||
Lease {
|
||||
buf: Some(b),
|
||||
home: self.tx.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A borrowed buffer, returned (cleared) on drop — including when a job
|
||||
/// fails or panics.
|
||||
pub struct Lease {
|
||||
buf: Option<SecretVec>,
|
||||
home: SyncSender<SecretVec>,
|
||||
}
|
||||
|
||||
impl Lease {
|
||||
pub fn buf(&mut self) -> &mut SecretVec {
|
||||
self.buf.as_mut().expect("leased")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Lease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(mut b) = self.buf.take() {
|
||||
b.clear();
|
||||
let _ = self.home.send(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn results_in_submission_order_and_bounded_concurrency() {
|
||||
let pool = Pool::new(4);
|
||||
let running = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let tickets: Vec<_> = (0..64)
|
||||
.map(|i| {
|
||||
let (running, peak) = (Arc::clone(&running), Arc::clone(&peak));
|
||||
pool.submit(move || {
|
||||
let now = running.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
peak.fetch_max(now, Ordering::SeqCst);
|
||||
std::thread::sleep(Duration::from_millis((64 - i) % 5));
|
||||
running.fetch_sub(1, Ordering::SeqCst);
|
||||
i
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let got: Vec<_> = tickets.into_iter().map(|t| t.wait().unwrap()).collect();
|
||||
assert_eq!(got, (0..64).collect::<Vec<_>>());
|
||||
assert!(peak.load(Ordering::SeqCst) <= 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn panics_become_errors() {
|
||||
let pool = Pool::new(2);
|
||||
let t = pool.submit(|| -> u8 { panic!("boom") });
|
||||
assert!(t.wait().is_err());
|
||||
assert_eq!(pool.submit(|| 5).wait().unwrap(), 5, "worker survived");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_tickets_and_pool_cleans_up() {
|
||||
for _ in 0..20 {
|
||||
let pool = Pool::new(3);
|
||||
for _ in 0..10 {
|
||||
drop(pool.submit(|| std::thread::sleep(Duration::from_millis(1))));
|
||||
}
|
||||
drop(pool); // joins: no threads outlive it
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buffers_are_bounded_and_cleared() {
|
||||
let b = Buffers::new(2, 8);
|
||||
let mut l1 = b.take();
|
||||
l1.buf().extend_within_capacity(b"secret");
|
||||
let _l2 = b.take();
|
||||
let b2 = b.clone();
|
||||
let waiter = std::thread::spawn(move || {
|
||||
let mut l = b2.take();
|
||||
l.buf().len()
|
||||
});
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
assert!(!waiter.is_finished(), "a third lease should wait");
|
||||
drop(l1);
|
||||
assert_eq!(waiter.join().unwrap(), 0, "returned buffers are cleared");
|
||||
}
|
||||
}
|
||||
+5
-2
@@ -99,9 +99,12 @@ impl SecretVec {
|
||||
self.0.resize(n, 0);
|
||||
}
|
||||
|
||||
/// Clears the contents for reuse, keeping the allocation.
|
||||
/// Clears the contents for reuse, keeping the allocation. Bytes past
|
||||
/// the length are always zero already (shrinking clears them), so only
|
||||
/// the live part needs overwriting.
|
||||
pub fn clear(&mut self) {
|
||||
self.0.zeroize();
|
||||
self.0.as_mut_slice().zeroize();
|
||||
self.0.clear();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,795 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The on-disk vault, compatible with the Go version:
|
||||
//!
|
||||
//! ```text
|
||||
//! DIR/.vault "VAULT 2\n"
|
||||
//! DIR/records/<xx>/<id> one encrypted 4096-byte catalogue record per file
|
||||
//! DIR/objects/<xx>/<id> a file's body, in one or more encrypted objects
|
||||
//! ```
|
||||
//!
|
||||
//! `<xx>` is the first byte of `<id>` in hex; all 256 shards exist from
|
||||
//! `init`. Record IDs are random; a file's object IDs are
|
||||
//! HMAC-SHA-256(seed, index)[:16] for a secret seed in its record, so
|
||||
//! nothing on disk links a file's objects. An object's plaintext is its own
|
||||
//! ID, up to 4 MiB − 16 bytes of the file, and random padding up to a
|
||||
//! power-of-two bucket of 64 KiB to 4 MiB. Every ciphertext's and shard's
|
||||
//! modification time is pinned to the epoch.
|
||||
//!
|
||||
//! Ordering keeps the store consistent across crashes: objects are durable
|
||||
//! before the record that names them; a new record is durable before an old
|
||||
//! one it replaces is removed; a record's removal is durable before the
|
||||
//! objects only it used are deleted; copies share objects.
|
||||
|
||||
pub mod types;
|
||||
|
||||
use crate::crypto::fury;
|
||||
use crate::crypto::kem::{Identity, Recipient};
|
||||
use crate::error::{Error, Kind, Result};
|
||||
use crate::pipeline::{Buffers, Lease, OBJECT_BUFFERS, Pool, Ticket};
|
||||
use crate::platform::fs as pfs;
|
||||
use crate::rng::Entropy;
|
||||
use crate::secret::SecretVec;
|
||||
use crate::util::Instant;
|
||||
use std::collections::{BTreeMap, HashSet, VecDeque};
|
||||
use std::ffi::OsStr;
|
||||
use std::io::{Read, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
pub use types::*;
|
||||
|
||||
pub const MARKER_NAME: &str = ".vault";
|
||||
pub const MARKER_CONTENT: &[u8] = b"VAULT 2\n";
|
||||
pub const RECORDS: &str = "records";
|
||||
pub const OBJECTS: &str = "objects";
|
||||
const FILLER_MIN: u64 = 8;
|
||||
const FILLER_MAX: u64 = 32;
|
||||
/// Ciphertext payload capacity for one object, plus one byte to detect
|
||||
/// over-long objects.
|
||||
const OBJECT_BUF: usize = CHUNK_SIZE as usize + 64 * crate::crypto::stream::TAG_SIZE + 1;
|
||||
|
||||
struct Inner {
|
||||
root: PathBuf,
|
||||
id: Identity,
|
||||
recipient: Recipient,
|
||||
no_sync: bool,
|
||||
entropy: Entropy,
|
||||
/// Test hook: make record writes fail just after their rename.
|
||||
#[cfg(test)]
|
||||
fail_after_publish: std::sync::atomic::AtomicBool,
|
||||
}
|
||||
|
||||
pub struct Store {
|
||||
inner: Arc<Inner>,
|
||||
pool: Arc<Pool>,
|
||||
buffers: OnceLock<Buffers>,
|
||||
}
|
||||
|
||||
/// Every record this identity can open, by name, newest first.
|
||||
#[derive(Default)]
|
||||
pub struct Catalogue {
|
||||
by_name: BTreeMap<String, Vec<Record>>,
|
||||
}
|
||||
|
||||
impl Catalogue {
|
||||
pub fn current(&self, name: &str) -> Option<&Record> {
|
||||
self.by_name.get(name).and_then(|v| v.first())
|
||||
}
|
||||
|
||||
pub fn contains(&self, name: &str) -> bool {
|
||||
self.current(name).is_some()
|
||||
}
|
||||
|
||||
pub fn len(&self) -> usize {
|
||||
self.by_name.len()
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.by_name.is_empty()
|
||||
}
|
||||
|
||||
/// Every name, sorted.
|
||||
pub fn names(&self) -> impl Iterator<Item = &str> {
|
||||
self.by_name.keys().map(String::as_str)
|
||||
}
|
||||
|
||||
/// Each name with its current record, sorted by name.
|
||||
pub fn iter(&self) -> impl Iterator<Item = (&str, &Record)> {
|
||||
self.by_name.iter().filter_map(|(n, v)| v.first().map(|r| (n.as_str(), r)))
|
||||
}
|
||||
|
||||
/// The names at or beneath the folder `prefix`, sorted.
|
||||
pub fn under(&self, prefix: &str) -> Vec<String> {
|
||||
self.by_name
|
||||
.keys()
|
||||
.filter(|n| *n == prefix || (n.len() > prefix.len() && n.starts_with(prefix) && n.as_bytes()[prefix.len()] == b'/'))
|
||||
.cloned()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn records(&self, name: &str) -> &[Record] {
|
||||
self.by_name.get(name).map_or(&[], Vec::as_slice)
|
||||
}
|
||||
|
||||
fn insert(&mut self, r: Record) {
|
||||
let v = self.by_name.entry(r.name.as_str().to_string()).or_default();
|
||||
v.push(r);
|
||||
v.sort_by(|a, b| b.added.cmp(&a.added).then(a.id.cmp(&b.id)));
|
||||
}
|
||||
}
|
||||
|
||||
/// What `add` did, for the message and for migrate's source check.
|
||||
#[derive(Debug)]
|
||||
pub struct Added {
|
||||
pub replaced: bool,
|
||||
}
|
||||
|
||||
fn after_publish(e: &Error) -> bool {
|
||||
e.kind == Kind::AfterPublish
|
||||
}
|
||||
|
||||
impl Store {
|
||||
/// Creates a vault at `root` holding at least `filler` bytes of filler.
|
||||
pub fn init(root: &Path, filler: u64, pool: Arc<Pool>) -> Result<()> {
|
||||
if root.join(MARKER_NAME).symlink_metadata().is_ok() {
|
||||
crate::bail!("{} is already a vault", root.display());
|
||||
}
|
||||
// Every shard exists from the start, so the tree's shape says nothing.
|
||||
for dir in [RECORDS, OBJECTS] {
|
||||
for i in 0..256 {
|
||||
let p = root.join(dir).join(format!("{i:02x}"));
|
||||
pfs::create_private_dir_all(&p).map_err(|e| Error::io("mkdir", &p, &e))?;
|
||||
}
|
||||
}
|
||||
write_filler(root, filler, Arc::clone(&pool), Entropy::Os)?;
|
||||
for dir in [RECORDS, OBJECTS] {
|
||||
for i in 0..256 {
|
||||
let p = root.join(dir).join(format!("{i:02x}"));
|
||||
pfs::set_epoch(&p).map_err(|e| Error::io("chtimes", &p, &e))?;
|
||||
}
|
||||
let p = root.join(dir);
|
||||
pfs::set_epoch(&p).map_err(|e| Error::io("chtimes", &p, &e))?;
|
||||
}
|
||||
let marker = root.join(MARKER_NAME);
|
||||
let mut f = pfs::create_new_private(&marker).map_err(|e| Error::io("open", &marker, &e))?;
|
||||
f.write_all(MARKER_CONTENT)
|
||||
.and_then(|_| f.sync_all())
|
||||
.map_err(|e| Error::io("write", &marker, &e))?;
|
||||
pfs::sync_dir(root).map_err(|e| Error::io("sync", root, &e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn open(root: &Path, id: Identity, pool: Arc<Pool>) -> Result<Store> {
|
||||
match std::fs::read(root.join(MARKER_NAME)) {
|
||||
Ok(m) if m == MARKER_CONTENT => {}
|
||||
_ => crate::bail!("{} is not a vault; run vault init", root.display()),
|
||||
}
|
||||
Ok(Store::new(root, id, pool, false, Entropy::Os))
|
||||
}
|
||||
|
||||
fn new(root: &Path, id: Identity, pool: Arc<Pool>, no_sync: bool, entropy: Entropy) -> Store {
|
||||
let recipient = id.recipient();
|
||||
let inner = Inner {
|
||||
root: root.to_path_buf(),
|
||||
id,
|
||||
recipient,
|
||||
no_sync,
|
||||
entropy,
|
||||
#[cfg(test)]
|
||||
fail_after_publish: Default::default(),
|
||||
};
|
||||
Store {
|
||||
inner: Arc::new(inner),
|
||||
pool,
|
||||
buffers: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn open_with(root: &Path, id: Identity, pool: Arc<Pool>, entropy: Entropy) -> Store {
|
||||
Store::new(root, id, pool, false, entropy)
|
||||
}
|
||||
|
||||
pub fn root(&self) -> &Path {
|
||||
&self.inner.root
|
||||
}
|
||||
|
||||
pub fn path(&self, dir: &str, id: &Id) -> PathBuf {
|
||||
self.inner.path(dir, id)
|
||||
}
|
||||
|
||||
fn buffers(&self) -> &Buffers {
|
||||
self.buffers.get_or_init(|| Buffers::new(OBJECT_BUFFERS, OBJECT_BUF))
|
||||
}
|
||||
|
||||
/// Decrypts every record this identity can open. Records sealed to a
|
||||
/// different key are skipped silently: they are simply not ours.
|
||||
/// Unreadable ones of ours are reported and skipped.
|
||||
pub fn catalogue(&self) -> Result<Catalogue> {
|
||||
let dir = self.inner.root.join(RECORDS);
|
||||
let mut ids = Vec::new();
|
||||
for shard in sorted_dir(&dir)? {
|
||||
if !shard.1 {
|
||||
continue;
|
||||
}
|
||||
let sp = dir.join(&shard.0);
|
||||
for (name, _) in sorted_dir(&sp)? {
|
||||
if let Some(id) = name.to_str().and_then(Id::parse) {
|
||||
if id.shard() == shard.0.to_string_lossy() {
|
||||
ids.push(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut cat = Catalogue::default();
|
||||
let window = self.pool.workers() * 2;
|
||||
let mut pending: VecDeque<(Id, Ticket<std::result::Result<Record, ReadRecordError>>)> = VecDeque::new();
|
||||
let collect = |id: Id, r: std::result::Result<std::result::Result<Record, ReadRecordError>, String>, cat: &mut Catalogue| match r {
|
||||
Ok(Ok(rec)) => cat.insert(rec),
|
||||
Ok(Err(ReadRecordError::Foreign)) => {}
|
||||
Ok(Err(ReadRecordError::Other(e))) | Err(e) => eprintln!("vault: skipping unreadable record {}: {e}", id.hex()),
|
||||
};
|
||||
for id in ids {
|
||||
if pending.len() == window {
|
||||
let (i, t) = pending.pop_front().expect("non-empty");
|
||||
collect(i, t.wait(), &mut cat);
|
||||
}
|
||||
let inner = Arc::clone(&self.inner);
|
||||
pending.push_back((id, self.pool.submit(move || inner.read_record(id))));
|
||||
}
|
||||
for (i, t) in pending {
|
||||
collect(i, t.wait(), &mut cat);
|
||||
}
|
||||
Ok(cat)
|
||||
}
|
||||
|
||||
/// Stores the regular file at `src` as `name`, replacing any file of
|
||||
/// that name. Fails, storing nothing, if the file changes while read.
|
||||
pub fn add_file(&self, cat: &mut Catalogue, src: &Path, name: &VaultName) -> Result<(Added, SourceStamp)> {
|
||||
let mut f = std::fs::File::open(src).map_err(|e| Error::io("open", src, &e))?;
|
||||
let meta = f.metadata().map_err(|e| Error::io("stat", src, &e))?;
|
||||
if !meta.is_file() || meta.len() > FILE_LIMIT {
|
||||
crate::bail!("{}: not a regular file of at most 1 TiB", src.display());
|
||||
}
|
||||
let stamp = SourceStamp::of(&meta);
|
||||
let changed = || Error::new(format!("{} changed while being added", src.display())).with_kind(Kind::SourceShort);
|
||||
let probe = f.try_clone().map_err(|e| Error::io("open", src, &e))?;
|
||||
// Before the record is written: the file must still be the one read.
|
||||
let mut unchanged = || match probe.metadata() {
|
||||
Ok(m) if SourceStamp::of(&m) == stamp => Ok(()),
|
||||
_ => Err(changed()),
|
||||
};
|
||||
let mut guard = EndCheck {
|
||||
inner: &mut f,
|
||||
size: meta.len(),
|
||||
read: 0,
|
||||
};
|
||||
let mtime = Instant::from_system(meta.modified().unwrap_or(std::time::UNIX_EPOCH));
|
||||
match self.add_checked(cat, &mut guard, meta.len(), mode_of(&meta), mtime, name, &mut unchanged) {
|
||||
Ok(a) => Ok((a, stamp)),
|
||||
Err(e) if e.kind == Kind::SourceShort => Err(changed()),
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stores `size` bytes from `src` as `name`. A source that ends early,
|
||||
/// or (for an [`EndCheck`]) runs on past `size`, stores nothing.
|
||||
pub fn add_reader(
|
||||
&self,
|
||||
cat: &mut Catalogue,
|
||||
src: &mut dyn Read,
|
||||
size: u64,
|
||||
mode: u32,
|
||||
mtime: Instant,
|
||||
name: &VaultName,
|
||||
) -> Result<Added> {
|
||||
self.add_checked(cat, src, size, mode, mtime, name, &mut || Ok(()))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn add_checked(
|
||||
&self,
|
||||
cat: &mut Catalogue,
|
||||
src: &mut dyn Read,
|
||||
size: u64,
|
||||
mode: u32,
|
||||
mtime: Instant,
|
||||
name: &VaultName,
|
||||
before_commit: &mut dyn FnMut() -> Result<()>,
|
||||
) -> Result<Added> {
|
||||
if size > FILE_LIMIT {
|
||||
crate::bail!("{name}: larger than 1 TiB");
|
||||
}
|
||||
let seed = ObjectSeed::random(&mut self.inner.entropy.rng(b"seed"));
|
||||
if let Err(e) = self.write_objects(src, &seed, size).and_then(|_| before_commit()) {
|
||||
let _ = self.drop_objects(&seed, size);
|
||||
return Err(e);
|
||||
}
|
||||
let mut idr = self.inner.entropy.rng(seed.object_id(u64::MAX).0.as_slice());
|
||||
let rec = Record {
|
||||
name: name.clone(),
|
||||
seed,
|
||||
size,
|
||||
mode: mode & 0o777,
|
||||
modified: mtime,
|
||||
added: Instant::now(),
|
||||
id: Id::random(&mut idr),
|
||||
};
|
||||
if let Err(e) = self.inner.write_record(&rec) {
|
||||
// A record already under its name keeps its objects, even if a
|
||||
// later sync failed.
|
||||
if !after_publish(&e) {
|
||||
let _ = self.drop_objects(&rec.seed, size);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
let replaced = cat.contains(name.as_str());
|
||||
self.remove(cat, name.as_str())?;
|
||||
cat.insert(rec);
|
||||
Ok(Added { replaced })
|
||||
}
|
||||
|
||||
/// Reads `size` bytes into objects derived from `seed`. The caller's
|
||||
/// thread reads the source in order; encryption, writing and syncing of
|
||||
/// up to eight objects overlap on the worker pool.
|
||||
fn write_objects(&self, src: &mut dyn Read, seed: &ObjectSeed, size: u64) -> Result<()> {
|
||||
let mut pending: VecDeque<Ticket<Result<()>>> = VecDeque::new();
|
||||
let mut first_err: Option<Error> = None;
|
||||
let mut remaining = size;
|
||||
for i in 0..object_count(size) {
|
||||
if pending.len() == OBJECT_BUFFERS {
|
||||
if let Err(e) = pending.pop_front().expect("non-empty").wait().map_err(Error::new).and_then(|r| r) {
|
||||
first_err = Some(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let id = seed.object_id(i);
|
||||
let part = remaining.min(CHUNK_DATA) as usize;
|
||||
remaining -= part as u64;
|
||||
let mut lease = self.buffers().take();
|
||||
let buf = lease.buf();
|
||||
buf.extend_within_capacity(&id.0);
|
||||
buf.resize_within_capacity(ID_SIZE + part);
|
||||
match crate::crypto::stream::read_full(src, &mut buf.expose_mut()[ID_SIZE..]) {
|
||||
Ok(n) if n == part => {}
|
||||
Ok(_) => {
|
||||
first_err = Some(Error::new("source ended early").with_kind(Kind::SourceShort));
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
first_err = Some(if e.kind() == std::io::ErrorKind::UnexpectedEof {
|
||||
Error::new("source ended early").with_kind(Kind::SourceShort)
|
||||
} else {
|
||||
Error::from(e)
|
||||
});
|
||||
break;
|
||||
}
|
||||
}
|
||||
let inner = Arc::clone(&self.inner);
|
||||
pending.push_back(self.pool.submit(move || inner.write_object(id, lease)));
|
||||
}
|
||||
// Wait for everything in flight, keeping the first error.
|
||||
for t in pending {
|
||||
if let Err(e) = t.wait().map_err(Error::new).and_then(|r| r) {
|
||||
first_err.get_or_insert(e);
|
||||
}
|
||||
}
|
||||
match first_err {
|
||||
Some(e) => Err(e),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Deletes every record for `name` and the objects no other name uses.
|
||||
fn remove(&self, cat: &mut Catalogue, name: &str) -> Result<()> {
|
||||
let in_use: HashSet<Id> = cat
|
||||
.iter_all()
|
||||
.filter(|(n, _)| *n != name)
|
||||
.map(|(_, r)| r.seed.object_id(0))
|
||||
.collect();
|
||||
let records = cat.by_name.remove(name).unwrap_or_default();
|
||||
let mut iter = records.into_iter();
|
||||
while let Some(r) = iter.next() {
|
||||
if let Err(e) = self.inner.remove_record(&r.id) {
|
||||
// Put back what is still there.
|
||||
let mut keep: Vec<Record> = vec![r];
|
||||
keep.extend(iter);
|
||||
for k in keep {
|
||||
cat.insert(k);
|
||||
}
|
||||
return Err(e);
|
||||
}
|
||||
if !in_use.contains(&r.seed.object_id(0)) {
|
||||
self.drop_objects(&r.seed, r.size)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Deletes every record for `names` and the objects they alone hold,
|
||||
/// calling `done` after each name.
|
||||
pub fn remove_all(&self, cat: &mut Catalogue, names: &[String], done: &mut dyn FnMut(&str)) -> Result<()> {
|
||||
let gone: HashSet<&str> = names.iter().map(String::as_str).collect();
|
||||
let in_use: HashSet<Id> = cat
|
||||
.iter_all()
|
||||
.filter(|(n, _)| !gone.contains(n))
|
||||
.map(|(_, r)| r.seed.object_id(0))
|
||||
.collect();
|
||||
for name in names {
|
||||
for r in cat.records(name) {
|
||||
self.inner.remove_record(&r.id)?;
|
||||
if !in_use.contains(&r.seed.object_id(0)) {
|
||||
self.drop_objects(&r.seed, r.size)?;
|
||||
}
|
||||
}
|
||||
cat.by_name.remove(name);
|
||||
done(name);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Stores each pair's first name again under its second, sharing its
|
||||
/// objects; with `mv`, then deletes the first names. New records come
|
||||
/// first, so an interruption leaves a file under both names, not none.
|
||||
pub fn copy_all(&self, cat: &mut Catalogue, pairs: &[(String, VaultName)], mv: bool, done: &mut dyn FnMut(&str, &str)) -> Result<()> {
|
||||
let mut old = Vec::new();
|
||||
for (from, to) in pairs {
|
||||
let src = cat.current(from).ok_or_else(|| Error::new(format!("{from}: not in the vault")))?;
|
||||
let mut idr = self.inner.entropy.rng(to.as_str().as_bytes());
|
||||
let mut r = src.copy_as(to.clone(), Id::random(&mut idr));
|
||||
if !mv {
|
||||
r.added = Instant::now();
|
||||
}
|
||||
self.inner.write_record(&r)?;
|
||||
cat.by_name.remove(to.as_str());
|
||||
cat.insert(r);
|
||||
old.push(from.clone());
|
||||
done(from, to.as_str());
|
||||
}
|
||||
if mv { self.remove_all(cat, &old, &mut |_| {}) } else { Ok(()) }
|
||||
}
|
||||
|
||||
/// Deletes the objects of a file of `size` bytes, skipping any gone.
|
||||
fn drop_objects(&self, seed: &ObjectSeed, size: u64) -> Result<()> {
|
||||
for i in 0..object_count(size) {
|
||||
match self.inner.remove_file(OBJECTS, &seed.object_id(i), false) {
|
||||
Err(e) if e.kind == Kind::NotFound => {}
|
||||
r => r?,
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Decrypts `rec` into `dir/name`. Nothing appears under `name` until
|
||||
/// every object — ID prefix, data, padding, final tag and end of file —
|
||||
/// has authenticated, and an existing `name` (even a dangling symlink,
|
||||
/// even one created meanwhile) is never replaced.
|
||||
pub fn extract(&self, rec: &Record, dir: &pfs::Dir, name: &OsStr) -> Result<()> {
|
||||
let out = dir.path().join(name);
|
||||
if out.symlink_metadata().is_ok() {
|
||||
crate::bail!("{} already exists", out.display());
|
||||
}
|
||||
let (mut f, tmp) = dir
|
||||
.create_temp(".vault-get-")
|
||||
.map_err(|e| Error::io("create temporary file in", dir.path(), &e))?;
|
||||
let r = self.extract_into(rec, &mut f).and_then(|_| {
|
||||
f.set_times(
|
||||
std::fs::FileTimes::new()
|
||||
.set_accessed(rec.modified.to_system())
|
||||
.set_modified(rec.modified.to_system()),
|
||||
)
|
||||
.map_err(|e| Error::io("chtimes", &out, &e))?;
|
||||
pfs::set_mode(&f, rec.mode).map_err(|e| Error::io("chmod", &out, &e))?;
|
||||
f.sync_all().map_err(|e| Error::io("sync", &out, &e))
|
||||
});
|
||||
drop(f);
|
||||
let r = r.and_then(|_| {
|
||||
dir.publish(&tmp, name).map_err(|e| {
|
||||
if e.kind() == std::io::ErrorKind::AlreadyExists {
|
||||
Error::new(format!("{} already exists", out.display()))
|
||||
} else {
|
||||
Error::io("rename", &out, &e)
|
||||
}
|
||||
})
|
||||
});
|
||||
if let Err(e) = r {
|
||||
let _ = dir.remove(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
dir.sync().map_err(|e| Error::io("sync", dir.path(), &e))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_into(&self, rec: &Record, out: &mut std::fs::File) -> Result<()> {
|
||||
let mut pending: VecDeque<(u64, Ticket<Result<Lease>>)> = VecDeque::new();
|
||||
let count = object_count(rec.size);
|
||||
let mut write = |i: u64, t: Ticket<Result<Lease>>| -> Result<()> {
|
||||
let mut lease = t.wait().map_err(Error::new)??;
|
||||
let part = rec.part(i) as usize;
|
||||
out.write_all(&lease.buf().expose()[ID_SIZE..ID_SIZE + part]).map_err(Error::from)
|
||||
};
|
||||
let mut result = Ok(());
|
||||
for i in 0..count {
|
||||
if pending.len() == OBJECT_BUFFERS {
|
||||
let (j, t) = pending.pop_front().expect("non-empty");
|
||||
if let Err(e) = write(j, t) {
|
||||
result = Err(e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
let lease = self.buffers().take();
|
||||
let inner = Arc::clone(&self.inner);
|
||||
let id = rec.seed.object_id(i);
|
||||
let part = rec.part(i);
|
||||
pending.push_back((i, self.pool.submit(move || inner.read_object(id, part, lease))));
|
||||
}
|
||||
for (j, t) in pending {
|
||||
if result.is_ok() {
|
||||
result = write(j, t);
|
||||
} else {
|
||||
drop(t.wait());
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
impl Catalogue {
|
||||
/// Every record, duplicates included.
|
||||
fn iter_all(&self) -> impl Iterator<Item = (&str, &Record)> {
|
||||
self.by_name.iter().flat_map(|(n, v)| v.iter().map(move |r| (n.as_str(), r)))
|
||||
}
|
||||
}
|
||||
|
||||
enum ReadRecordError {
|
||||
Foreign,
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl Inner {
|
||||
fn path(&self, dir: &str, id: &Id) -> PathBuf {
|
||||
self.root.join(dir).join(id.shard()).join(id.hex())
|
||||
}
|
||||
|
||||
/// Encrypts `plaintext` to `dir/<id>` through a temporary file, so a
|
||||
/// partial ciphertext never appears under a real ID. An error after the
|
||||
/// rename is marked [`Kind::AfterPublish`].
|
||||
fn write_ciphertext(&self, dir: &str, id: &Id, plaintext: &[u8]) -> Result<()> {
|
||||
let dst = self.path(dir, id);
|
||||
let shard = dst.parent().expect("sharded path").to_path_buf();
|
||||
let (mut f, tmp) = pfs::create_temp(&shard, ".pending-").map_err(|e| Error::io("open", &shard, &e))?;
|
||||
let mut ct = Vec::with_capacity(plaintext.len() + 32 << 10);
|
||||
fury::encrypt_to(&mut ct, &self.recipient, plaintext, &mut self.entropy.rng(&id.0));
|
||||
let r = (|| -> Result<()> {
|
||||
f.write_all(&ct).map_err(|e| Error::io("write", &tmp, &e))?;
|
||||
if !self.no_sync {
|
||||
f.sync_all().map_err(|e| Error::io("sync", &tmp, &e))?;
|
||||
}
|
||||
drop(f);
|
||||
pfs::set_epoch(&tmp).map_err(|e| Error::io("chtimes", &tmp, &e))?;
|
||||
std::fs::rename(&tmp, &dst).map_err(|e| Error::io("rename", &dst, &e))
|
||||
})();
|
||||
if let Err(e) = r {
|
||||
let _ = std::fs::remove_file(&tmp);
|
||||
return Err(e);
|
||||
}
|
||||
let after = |e: Error| e.with_kind(Kind::AfterPublish);
|
||||
#[cfg(test)]
|
||||
if dir == RECORDS && self.fail_after_publish.load(std::sync::atomic::Ordering::SeqCst) {
|
||||
return Err(after(Error::new("injected sync failure")));
|
||||
}
|
||||
pfs::set_epoch(&shard).map_err(|e| after(Error::io("chtimes", &shard, &e)))?;
|
||||
if !self.no_sync {
|
||||
pfs::sync_dir(&shard).map_err(|e| after(Error::io("sync", &shard, &e)))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_object(&self, id: Id, mut lease: Lease) -> Result<()> {
|
||||
let buf = lease.buf();
|
||||
let used = buf.len();
|
||||
let padded = padded_size(used as u64) as usize;
|
||||
buf.resize_within_capacity(padded);
|
||||
self.entropy
|
||||
.rng(&[b"pad".as_slice(), &id.0].concat())
|
||||
.fill(&mut buf.expose_mut()[used..]);
|
||||
self.write_ciphertext(OBJECTS, &id, buf.expose())
|
||||
}
|
||||
|
||||
fn write_record(&self, r: &Record) -> Result<()> {
|
||||
let json = r.to_json().ok_or("record time outside the years 0 to 9999")?;
|
||||
if json.len() > RECORD_SIZE - 4 {
|
||||
crate::bail!("catalogue record too large");
|
||||
}
|
||||
let mut plain = SecretVec::zeroed(RECORD_SIZE);
|
||||
let p = plain.expose_mut();
|
||||
p[..4].copy_from_slice(&(json.len() as u32).to_be_bytes());
|
||||
p[4..4 + json.len()].copy_from_slice(&json);
|
||||
self.entropy
|
||||
.rng(&[b"recpad".as_slice(), &r.id.0].concat())
|
||||
.fill(&mut p[4 + json.len()..]);
|
||||
self.write_ciphertext(RECORDS, &r.id, plain.expose())
|
||||
}
|
||||
|
||||
fn read_record(&self, id: Id) -> std::result::Result<Record, ReadRecordError> {
|
||||
let p = self.path(RECORDS, &id);
|
||||
let f = std::fs::File::open(&p).map_err(|e| ReadRecordError::Other(crate::util::io_msg("open", &p, &e)))?;
|
||||
let plain = match fury::decrypt_bounded(std::io::BufReader::new(f), &self.id, RECORD_SIZE) {
|
||||
Ok(p) => p,
|
||||
Err(fury::Error::NoIdentityMatch) => return Err(ReadRecordError::Foreign),
|
||||
Err(fury::Error::TooLarge) => return Err(ReadRecordError::Other("wrong record size".into())),
|
||||
Err(e) => return Err(ReadRecordError::Other(e.to_string())),
|
||||
};
|
||||
let b = plain.expose();
|
||||
if b.len() != RECORD_SIZE {
|
||||
return Err(ReadRecordError::Other("wrong record size".into()));
|
||||
}
|
||||
let n = u32::from_be_bytes([b[0], b[1], b[2], b[3]]) as usize;
|
||||
if n > RECORD_SIZE - 4 {
|
||||
return Err(ReadRecordError::Other("malformed record".into()));
|
||||
}
|
||||
Record::from_json(&b[4..4 + n], id).map_err(ReadRecordError::Other)
|
||||
}
|
||||
|
||||
/// Decrypts object `id` and checks it leads with its own ID and holds
|
||||
/// `part` bytes padded to their bucket. Returns the plaintext buffer.
|
||||
fn read_object(&self, id: Id, part: u64, mut lease: Lease) -> Result<Lease> {
|
||||
let p = self.path(OBJECTS, &id);
|
||||
let f = std::fs::File::open(&p).map_err(|e| Error::io("open", &p, &e))?;
|
||||
let padded = padded_size(ID_SIZE as u64 + part) as usize;
|
||||
match fury::decrypt_bounded_into(std::io::BufReader::new(f), &self.id, padded, lease.buf()) {
|
||||
Ok(()) => {}
|
||||
Err(fury::Error::TooLarge) => crate::bail!("object size does not match its catalogue record"),
|
||||
Err(e) => return Err(Error::new(e.to_string())),
|
||||
}
|
||||
let b = lease.buf().expose();
|
||||
if b.len() < ID_SIZE || b[..ID_SIZE] != id.0 {
|
||||
crate::bail!("object does not match its catalogue record");
|
||||
}
|
||||
if b.len() != padded {
|
||||
crate::bail!("object size does not match its catalogue record");
|
||||
}
|
||||
Ok(lease)
|
||||
}
|
||||
|
||||
/// Removes `dir/<id>` and re-pins its shard's time. For records the
|
||||
/// shard is synced, so the removal is durable before any object goes.
|
||||
/// A missing file is reported with [`Kind::NotFound`].
|
||||
fn remove_file(&self, dir: &str, id: &Id, sync: bool) -> Result<()> {
|
||||
let p = self.path(dir, id);
|
||||
match std::fs::remove_file(&p) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
return Err(Error::io("remove", &p, &e).with_kind(Kind::NotFound));
|
||||
}
|
||||
Err(e) => return Err(Error::io("remove", &p, &e)),
|
||||
}
|
||||
let shard = p.parent().expect("sharded");
|
||||
pfs::set_epoch(shard).map_err(|e| Error::io("chtimes", shard, &e))?;
|
||||
if sync && !self.no_sync {
|
||||
pfs::sync_dir(shard).map_err(|e| Error::io("sync", shard, &e))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_record(&self, id: &Id) -> Result<()> {
|
||||
self.remove_file(RECORDS, id, true).map_err(|e| e.with_kind(Kind::Other))
|
||||
}
|
||||
}
|
||||
|
||||
/// Entries of `dir`, sorted by name, each with whether it is a directory
|
||||
/// (not following symlinks), as Go's `os.ReadDir`.
|
||||
fn sorted_dir(dir: &Path) -> Result<Vec<(std::ffi::OsString, bool)>> {
|
||||
let mut v = Vec::new();
|
||||
for e in std::fs::read_dir(dir).map_err(|e| Error::io("open", dir, &e))? {
|
||||
let e = e.map_err(|e| Error::io("readdir", dir, &e))?;
|
||||
let is_dir = e.file_type().map(|t| t.is_dir()).unwrap_or(false);
|
||||
v.push((e.file_name(), is_dir));
|
||||
}
|
||||
v.sort();
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
fn mode_of(m: &std::fs::Metadata) -> u32 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
m.permissions().mode() & 0o777
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
// Go reports 0o444 for read-only files and 0o666 otherwise.
|
||||
if m.permissions().readonly() { 0o444 } else { 0o666 }
|
||||
}
|
||||
}
|
||||
|
||||
/// Identity of a source file as read, so migrate deletes only that file.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
|
||||
pub struct SourceStamp {
|
||||
dev: u64,
|
||||
ino: u64,
|
||||
size: u64,
|
||||
mtime: Option<std::time::SystemTime>,
|
||||
}
|
||||
|
||||
impl SourceStamp {
|
||||
pub fn of(m: &std::fs::Metadata) -> SourceStamp {
|
||||
#[cfg(unix)]
|
||||
let (dev, ino) = {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
(m.dev(), m.ino())
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let (dev, ino) = (0, 0);
|
||||
SourceStamp {
|
||||
dev,
|
||||
ino,
|
||||
size: m.len(),
|
||||
mtime: m.modified().ok(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads exactly `size` bytes from a file, then insists on end of file, so
|
||||
/// a file that grew while being added is caught.
|
||||
struct EndCheck<'a, R: Read> {
|
||||
inner: &'a mut R,
|
||||
size: u64,
|
||||
read: u64,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for EndCheck<'_, R> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let want = (self.size - self.read).min(buf.len() as u64) as usize;
|
||||
let n = if want == 0 { 0 } else { self.inner.read(&mut buf[..want])? };
|
||||
self.read += n as u64;
|
||||
if self.read == self.size && (n > 0 || want == 0) {
|
||||
let mut b = [0u8; 1];
|
||||
loop {
|
||||
match self.inner.read(&mut b) {
|
||||
Ok(0) => break,
|
||||
Ok(_) => return Err(std::io::Error::new(std::io::ErrorKind::UnexpectedEof, "source grew")),
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Writes random entries sealed to a throwaway key, through the same code
|
||||
/// as `add`: from 8 to 32 entries of log-uniformly random sizes, or more
|
||||
/// until they total `target` bytes. Every vault therefore holds entries
|
||||
/// nobody can open, so a file hidden from a decoy passphrase looks like
|
||||
/// filler. Filler needs no per-file durability.
|
||||
fn write_filler(root: &Path, target: u64, pool: Arc<Pool>, entropy: Entropy) -> Result<()> {
|
||||
let mut rng = entropy.rng(b"filler");
|
||||
let throwaway = Identity::generate(&mut rng);
|
||||
let s = Store::new(root, throwaway, pool, true, entropy);
|
||||
let upper = (target / 8).max(CHUNK_SIZE).min(1 << 30) as f64;
|
||||
let count = FILLER_MIN + rng.below(FILLER_MAX - FILLER_MIN + 1);
|
||||
let mut cat = Catalogue::default();
|
||||
let (mut i, mut total) = (0u64, 0u64);
|
||||
while i < count || total < target {
|
||||
let size = (1024.0 * (rng.f64() * (upper / 1024.0).ln()).exp()) as u64;
|
||||
let name = VaultName::new(&hex(&rng.array::<8>())).expect("hex is a valid name");
|
||||
let mut src = RandomReader(s.inner.entropy.rng(&rng.array::<16>()));
|
||||
s.add_reader(&mut cat, &mut src, size, 0o600, Instant::now(), &name)?;
|
||||
total += size;
|
||||
i += 1;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct RandomReader(crate::rng::Rng);
|
||||
|
||||
impl Read for RandomReader {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
self.0.fill(buf);
|
||||
Ok(buf.len())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,492 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
use super::*;
|
||||
use crate::crypto::kem::SUITE_ID;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
pub(crate) struct TempDir(pub PathBuf);
|
||||
|
||||
impl TempDir {
|
||||
pub fn new(tag: &str) -> TempDir {
|
||||
let r: [u8; 8] = crate::rng::os().array();
|
||||
let p = std::env::temp_dir().join(format!("vault-store-{tag}-{}-{}", std::process::id(), hex(&r)));
|
||||
std::fs::create_dir_all(&p).unwrap();
|
||||
TempDir(p)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TempDir {
|
||||
fn drop(&mut self) {
|
||||
// Read-only test directories must be writable again to delete.
|
||||
#[cfg(unix)]
|
||||
for e in walk(&self.0) {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if e.is_dir() {
|
||||
let _ = std::fs::set_permissions(&e, std::fs::Permissions::from_mode(0o700));
|
||||
}
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&self.0);
|
||||
}
|
||||
}
|
||||
|
||||
fn walk(p: &Path) -> Vec<PathBuf> {
|
||||
let mut out = vec![p.to_path_buf()];
|
||||
if let Ok(rd) = std::fs::read_dir(p) {
|
||||
for e in rd.flatten() {
|
||||
let path = e.path();
|
||||
if e.file_type().map(|t| t.is_dir()).unwrap_or(false) {
|
||||
out.extend(walk(&path));
|
||||
} else {
|
||||
out.push(path);
|
||||
}
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
fn files(p: &Path) -> Vec<PathBuf> {
|
||||
let mut v: Vec<_> = walk(p).into_iter().filter(|p| p.is_file()).collect();
|
||||
v.sort();
|
||||
v
|
||||
}
|
||||
|
||||
pub(crate) fn identity(n: u8) -> Identity {
|
||||
Identity::from_suite_seed(SUITE_ID, &[n; 32]).unwrap()
|
||||
}
|
||||
|
||||
pub(crate) fn pool() -> Arc<Pool> {
|
||||
Arc::new(Pool::new(4))
|
||||
}
|
||||
|
||||
/// A fresh vault (filler kept small but present) and a store on it.
|
||||
fn vault(tag: &str) -> (TempDir, Store) {
|
||||
let d = TempDir::new(tag);
|
||||
let root = d.0.join("store");
|
||||
Store::init(&root, 0, pool()).unwrap();
|
||||
let s = Store::open(&root, identity(1), pool()).unwrap();
|
||||
(d, s)
|
||||
}
|
||||
|
||||
fn name(n: &str) -> VaultName {
|
||||
VaultName::new(n).unwrap()
|
||||
}
|
||||
|
||||
fn add_bytes(s: &Store, cat: &mut Catalogue, n: &str, body: &[u8]) -> Added {
|
||||
s.add_reader(cat, &mut &body[..], body.len() as u64, 0o640, Instant::now(), &name(n))
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn get_bytes(s: &Store, rec: &Record, dir: &Path, out: &str) -> Result<Vec<u8>> {
|
||||
let d = pfs::Dir::open(dir).unwrap();
|
||||
s.extract(rec, &d, OsStr::new(out))?;
|
||||
Ok(std::fs::read(dir.join(out)).unwrap())
|
||||
}
|
||||
|
||||
fn objects(s: &Store) -> Vec<PathBuf> {
|
||||
files(&s.root().join(OBJECTS))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle() {
|
||||
let (d, s) = vault("life");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
assert!(cat.is_empty(), "filler must be invisible");
|
||||
let want = vec![b'x'; 70000];
|
||||
assert!(!add_bytes(&s, &mut cat, "a/b", &want).replaced);
|
||||
assert!(add_bytes(&s, &mut cat, "a/b", &want).replaced);
|
||||
add_bytes(&s, &mut cat, "c", b"see");
|
||||
let cat2 = s.catalogue().unwrap();
|
||||
assert_eq!(cat2.names().collect::<Vec<_>>(), ["a/b", "c"]);
|
||||
assert_eq!(cat2.records("a/b").len(), 1, "replace removed the old record");
|
||||
let r = cat2.current("a/b").unwrap();
|
||||
assert_eq!((r.size, r.mode), (70000, 0o640));
|
||||
assert_eq!(get_bytes(&s, r, &d.0, "out").unwrap(), want);
|
||||
let e = get_bytes(&s, r, &d.0, "out").unwrap_err();
|
||||
assert!(e.msg.ends_with("already exists"), "{e}");
|
||||
|
||||
// Swapping one object for another is detected, and nothing appears.
|
||||
let (dr, cr) = (cat2.current("a/b").unwrap(), cat2.current("c").unwrap());
|
||||
std::fs::copy(s.path(OBJECTS, &cr.seed.object_id(0)), s.path(OBJECTS, &dr.seed.object_id(0))).unwrap();
|
||||
let e = get_bytes(&s, dr, &d.0, "swapped").unwrap_err();
|
||||
assert!(e.msg.contains("does not match"), "{e}");
|
||||
assert!(!d.0.join("swapped").exists());
|
||||
assert_eq!(std::fs::read_dir(&d.0).unwrap().count(), 2, "temporary file left behind");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decoy_isolation_and_filler() {
|
||||
let (_d, real) = vault("decoy");
|
||||
let decoy = Store::open(real.root(), identity(2), pool()).unwrap();
|
||||
for (s, n) in [(&real, "secret"), (&decoy, "shopping")] {
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
assert!(cat.is_empty());
|
||||
add_bytes(s, &mut cat, n, b"data");
|
||||
}
|
||||
for (s, n) in [(&real, "secret"), (&decoy, "shopping")] {
|
||||
let cat = s.catalogue().unwrap();
|
||||
assert_eq!(cat.names().collect::<Vec<_>>(), [n]);
|
||||
}
|
||||
let records = files(&real.root().join(RECORDS));
|
||||
assert!(
|
||||
records.len() as u64 >= FILLER_MIN + 2,
|
||||
"only {} records; filler missing",
|
||||
records.len()
|
||||
);
|
||||
// Records are all one size, and every object is within the bucket range.
|
||||
let sizes: HashSet<u64> = records.iter().map(|p| p.metadata().unwrap().len()).collect();
|
||||
assert_eq!(sizes.len(), 1, "records differ in size: {sizes:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn filler_target() {
|
||||
let d = TempDir::new("filler");
|
||||
let root = d.0.join("v");
|
||||
Store::init(&root, 24 << 20, pool()).unwrap();
|
||||
assert!(Store::init(&root, 0, pool()).is_err(), "init twice");
|
||||
let objs = files(&root.join(OBJECTS));
|
||||
let total: u64 = objs.iter().map(|p| p.metadata().unwrap().len()).sum();
|
||||
let largest = objs.iter().map(|p| p.metadata().unwrap().len()).max().unwrap();
|
||||
assert!(total >= 24 << 20, "filler objects total {total}");
|
||||
assert!(largest <= CHUNK_SIZE + CHUNK_SIZE / 64, "largest object {largest}");
|
||||
for i in 0..256 {
|
||||
for dir in [RECORDS, OBJECTS] {
|
||||
let p = root.join(dir).join(format!("{i:02x}"));
|
||||
assert_eq!(p.metadata().unwrap().modified().unwrap(), std::time::UNIX_EPOCH, "{}", p.display());
|
||||
}
|
||||
}
|
||||
for f in files(&root.join(RECORDS)).iter().chain(&objs) {
|
||||
assert_eq!(f.metadata().unwrap().modified().unwrap(), std::time::UNIX_EPOCH);
|
||||
assert!(f.parent().unwrap().file_name().unwrap().to_str().unwrap() == &f.file_name().unwrap().to_str().unwrap()[..2]);
|
||||
}
|
||||
assert_eq!(std::fs::read(root.join(MARKER_NAME)).unwrap(), MARKER_CONTENT);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunks_and_reordering() {
|
||||
let (d, s) = vault("chunks");
|
||||
let before = objects(&s).len();
|
||||
let mut want = vec![0u8; 2 * CHUNK_DATA as usize + 100];
|
||||
crate::rng::os().fill(&mut want);
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
add_bytes(&s, &mut cat, "big", &want);
|
||||
let r = cat.current("big").unwrap();
|
||||
let sizes: Vec<u64> = (0..object_count(r.size))
|
||||
.map(|i| s.path(OBJECTS, &r.seed.object_id(i)).metadata().unwrap().len())
|
||||
.collect();
|
||||
assert!(sizes.len() == 3 && sizes[0] == sizes[1] && sizes[2] < sizes[0], "{sizes:?}");
|
||||
assert_eq!(objects(&s).len() - before, 3);
|
||||
assert_eq!(get_bytes(&s, r, &d.0, "out").unwrap(), want);
|
||||
let (a, b) = (s.path(OBJECTS, &r.seed.object_id(0)), s.path(OBJECTS, &r.seed.object_id(1)));
|
||||
let (ad, bd) = (std::fs::read(&a).unwrap(), std::fs::read(&b).unwrap());
|
||||
std::fs::write(&a, &bd).unwrap();
|
||||
std::fs::write(&b, &ad).unwrap();
|
||||
assert!(get_bytes(&s, r, &d.0, "swapped").is_err());
|
||||
assert!(!d.0.join("swapped").exists());
|
||||
s.remove_all(&mut cat, &["big".to_string()], &mut |_| {}).unwrap();
|
||||
assert_eq!(objects(&s).len(), before, "rm freed the objects");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn boundaries_round_trip() {
|
||||
let (d, s) = vault("bounds");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let c = CHUNK_DATA as usize;
|
||||
for (i, n) in [0, 1, 65535 - 16, 65536 - 16, 65536 - 15, 65536, c - 1, c, c + 1]
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
{
|
||||
let body: Vec<u8> = (0..n).map(|j| (j * 31 + i) as u8).collect();
|
||||
add_bytes(&s, &mut cat, &format!("f{i}"), &body);
|
||||
let r = cat.current(&format!("f{i}")).unwrap();
|
||||
assert_eq!(get_bytes(&s, r, &d.0, &format!("o{i}")).unwrap(), body, "size {n}");
|
||||
// Each object is padded: one bucket per object, all within range.
|
||||
for k in 0..object_count(n as u64) {
|
||||
let len = s.path(OBJECTS, &r.seed.object_id(k)).metadata().unwrap().len();
|
||||
assert!(
|
||||
len > MIN_PADDING && len <= CHUNK_SIZE + (32 << 10),
|
||||
"object of {len} bytes for size {n}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remove_tree_and_copy_move() {
|
||||
let (d, s) = vault("tree");
|
||||
let before = objects(&s).len();
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let mut big = vec![0u8; CHUNK_DATA as usize + 1];
|
||||
crate::rng::os().fill(&mut big);
|
||||
for (n, b) in [("a/x", &big[..]), ("a/b/y", b"y"), ("ab/z", b"z"), ("b/w", b"w")] {
|
||||
add_bytes(&s, &mut cat, n, b);
|
||||
}
|
||||
assert_eq!(objects(&s).len() - before, 5);
|
||||
assert_eq!(cat.under("a"), ["a/b/y", "a/x"]);
|
||||
// cp -r a c, then rm -r a: the copies keep the shared objects.
|
||||
let pairs: Vec<(String, VaultName)> = cat
|
||||
.under("a")
|
||||
.into_iter()
|
||||
.map(|n| (n.clone(), name(&format!("c{}", &n[1..]))))
|
||||
.collect();
|
||||
let mut copied = Vec::new();
|
||||
s.copy_all(&mut cat, &pairs, false, &mut |f, t| copied.push((f.to_string(), t.to_string())))
|
||||
.unwrap();
|
||||
assert_eq!(copied.len(), 2);
|
||||
assert_eq!(objects(&s).len() - before, 5, "copies share objects");
|
||||
let mut done = Vec::new();
|
||||
let names = cat.under("a");
|
||||
s.remove_all(&mut cat, &names, &mut |n| done.push(n.to_string())).unwrap();
|
||||
assert_eq!(done, ["a/b/y", "a/x"]);
|
||||
assert_eq!(objects(&s).len() - before, 5, "objects still used by the copies stay");
|
||||
let fresh = s.catalogue().unwrap();
|
||||
assert_eq!(fresh.names().collect::<Vec<_>>(), ["ab/z", "b/w", "c/b/y", "c/x"]);
|
||||
assert_eq!(get_bytes(&s, fresh.current("c/x").unwrap(), &d.0, "x").unwrap(), big);
|
||||
// mv keeps one copy of each object; rm of the last name frees them.
|
||||
let mut cat = fresh;
|
||||
s.copy_all(&mut cat, &[("c/x".into(), name("moved"))], true, &mut |_, _| {})
|
||||
.unwrap();
|
||||
assert!(!cat.contains("c/x") && cat.contains("moved"));
|
||||
let names: Vec<String> = cat.names().map(String::from).collect();
|
||||
s.remove_all(&mut cat, &names, &mut |_| {}).unwrap();
|
||||
assert_eq!(objects(&s).len(), before, "space freed");
|
||||
}
|
||||
|
||||
/// The Go oracle's fixture vault opens: real files decrypt to the same
|
||||
/// bytes and metadata, the decoy sees only its file, the foreign record is
|
||||
/// skipped silently.
|
||||
#[test]
|
||||
fn reads_go_vault() {
|
||||
let d = TempDir::new("govault");
|
||||
let src = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/mini-vault");
|
||||
let root = d.0.join("v");
|
||||
for p in walk(&src) {
|
||||
let rel = p.strip_prefix(&src).unwrap();
|
||||
if p.is_dir() {
|
||||
std::fs::create_dir_all(root.join(rel)).unwrap();
|
||||
} else {
|
||||
std::fs::copy(&p, root.join(rel)).unwrap();
|
||||
}
|
||||
}
|
||||
for dir in [RECORDS, OBJECTS] {
|
||||
for i in 0..256 {
|
||||
std::fs::create_dir_all(root.join(dir).join(format!("{i:02x}"))).unwrap();
|
||||
}
|
||||
}
|
||||
let key = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/vectors/key-with-decoy.pem");
|
||||
let real = crate::keyfile::unlock(&key, b"oracle-real passphrase").unwrap();
|
||||
let s = Store::open(&root, real, pool()).unwrap();
|
||||
let cat = s.catalogue().unwrap();
|
||||
assert_eq!(cat.names().collect::<Vec<_>>(), ["dir/empty", "dir/ünïcode.bin", "hello.txt"]);
|
||||
let boundary: Vec<u8> = (0..(MIN_PADDING as usize - ID_SIZE)).map(|i| (i * 7) as u8).collect();
|
||||
for (n, body, mode, secs, nanos) in [
|
||||
("hello.txt", &b"hello"[..], 0o644, 1700000000, 123456789),
|
||||
("dir/empty", b"", 0o600, 1600000000, 0),
|
||||
("dir/ünïcode.bin", &boundary[..], 0o755, 1500000000, 999999999),
|
||||
] {
|
||||
let r = cat.current(n).unwrap();
|
||||
assert_eq!((r.mode, r.modified), (mode, Instant { secs, nanos }), "{n}");
|
||||
let out = format!("out-{}", r.id.hex());
|
||||
assert_eq!(get_bytes(&s, r, &d.0, &out).unwrap(), body);
|
||||
let m = d.0.join(&out).metadata().unwrap();
|
||||
assert_eq!(m.modified().unwrap(), Instant { secs, nanos }.to_system());
|
||||
#[cfg(unix)]
|
||||
assert_eq!(std::os::unix::fs::PermissionsExt::mode(&m.permissions()) & 0o777, mode);
|
||||
}
|
||||
let decoy = crate::keyfile::unlock(&key, b"oracle-decoy passphrase").unwrap();
|
||||
let cat = Store::open(&root, decoy, pool()).unwrap().catalogue().unwrap();
|
||||
assert_eq!(cat.names().collect::<Vec<_>>(), ["shopping.txt"]);
|
||||
}
|
||||
|
||||
/// With identical synthetic randomness, one worker and eight produce
|
||||
/// byte-identical objects.
|
||||
#[test]
|
||||
fn sequential_and_parallel_identical() {
|
||||
let mut outputs = Vec::new();
|
||||
for workers in [1, 8] {
|
||||
let d = TempDir::new("det");
|
||||
let root = d.0.join("v");
|
||||
Store::init(&root, 0, pool()).unwrap();
|
||||
let s = Store::open_with(&root, identity(3), Arc::new(Pool::new(workers)), Entropy::Seeded([7; 32]));
|
||||
let before: HashSet<PathBuf> = objects(&s).into_iter().collect();
|
||||
let body: Vec<u8> = (0..3 * CHUNK_DATA as usize + 12345).map(|i| (i % 251) as u8).collect();
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
add_bytes(&s, &mut cat, "f", &body);
|
||||
let new: Vec<(String, Vec<u8>)> = objects(&s)
|
||||
.into_iter()
|
||||
.filter(|p| !before.contains(p))
|
||||
.map(|p| (p.file_name().unwrap().to_string_lossy().into_owned(), std::fs::read(&p).unwrap()))
|
||||
.collect();
|
||||
assert_eq!(new.len(), 4);
|
||||
outputs.push(new);
|
||||
}
|
||||
assert!(outputs[0] == outputs[1], "sequential and parallel output differ");
|
||||
}
|
||||
|
||||
/// A failure while writing objects stores nothing and leaves no objects.
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn object_write_failure_rolls_back() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let d = TempDir::new("fail");
|
||||
let root = d.0.join("v");
|
||||
Store::init(&root, 0, pool()).unwrap();
|
||||
let entropy = Entropy::Seeded([9; 32]);
|
||||
let s = Store::open_with(&root, identity(4), pool(), entropy.clone());
|
||||
let seed = ObjectSeed::random(&mut entropy.rng(b"seed"));
|
||||
let blocked = s.path(OBJECTS, &seed.object_id(2));
|
||||
let shard = blocked.parent().unwrap().to_path_buf();
|
||||
if seed.object_id(0).shard() == seed.object_id(2).shard() || seed.object_id(1).shard() == seed.object_id(2).shard() {
|
||||
return; // the fixed seed puts two objects in one shard; nothing to test
|
||||
}
|
||||
std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o500)).unwrap();
|
||||
if std::fs::write(shard.join("probe"), b"").is_ok() {
|
||||
return; // running as root: permissions do not bite
|
||||
}
|
||||
let before = objects(&s);
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let body = vec![1u8; 3 * CHUNK_DATA as usize];
|
||||
assert!(
|
||||
s.add_reader(&mut cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name("f"))
|
||||
.is_err()
|
||||
);
|
||||
std::fs::set_permissions(&shard, std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
assert_eq!(objects(&s), before, "objects of a failed add were left behind");
|
||||
assert!(s.catalogue().unwrap().is_empty());
|
||||
assert!(
|
||||
walk(&root)
|
||||
.iter()
|
||||
.all(|p| !p.file_name().unwrap().to_string_lossy().starts_with(".pending-"))
|
||||
);
|
||||
}
|
||||
|
||||
/// A short source stores nothing; a longer one than declared is refused.
|
||||
#[test]
|
||||
fn short_and_long_sources() {
|
||||
let (d, s) = vault("short");
|
||||
let before = objects(&s);
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let e = s
|
||||
.add_reader(&mut cat, &mut &b"abc"[..], 10, 0o600, Instant::now(), &name("f"))
|
||||
.unwrap_err();
|
||||
assert_eq!(e.kind, Kind::SourceShort);
|
||||
assert_eq!(objects(&s), before);
|
||||
let p = d.0.join("src");
|
||||
std::fs::write(&p, b"0123456789").unwrap();
|
||||
s.add_file(&mut cat, &p, &name("ok")).unwrap();
|
||||
assert!(s.catalogue().unwrap().contains("ok"));
|
||||
}
|
||||
|
||||
/// A record whose sync fails after it is in place keeps its objects.
|
||||
#[test]
|
||||
fn failure_after_publish_keeps_objects() {
|
||||
let (d, s) = vault("afterpub");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
s.inner.fail_after_publish.store(true, Ordering::SeqCst);
|
||||
let e = s
|
||||
.add_reader(&mut cat, &mut &b"precious"[..], 8, 0o600, Instant::now(), &name("f"))
|
||||
.unwrap_err();
|
||||
assert_eq!(e.kind, Kind::AfterPublish);
|
||||
s.inner.fail_after_publish.store(false, Ordering::SeqCst);
|
||||
let cat = s.catalogue().unwrap();
|
||||
assert_eq!(get_bytes(&s, cat.current("f").unwrap(), &d.0, "out").unwrap(), b"precious");
|
||||
}
|
||||
|
||||
/// Tampered, truncated, extended and foreign objects never publish output.
|
||||
#[test]
|
||||
fn damaged_objects_never_publish() {
|
||||
let (d, s) = vault("damage");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let body: Vec<u8> = (0..100_000).map(|i| i as u8).collect();
|
||||
add_bytes(&s, &mut cat, "f", &body);
|
||||
let r = cat.current("f").unwrap();
|
||||
let obj = s.path(OBJECTS, &r.seed.object_id(0));
|
||||
let orig = std::fs::read(&obj).unwrap();
|
||||
let foreign = {
|
||||
let mut v = Vec::new();
|
||||
fury::encrypt_to(&mut v, &identity(9).recipient(), &[0u8; 1 << 17], &mut crate::rng::os());
|
||||
v
|
||||
};
|
||||
// Same key and ID, but padding to the wrong bucket.
|
||||
let short_pad = {
|
||||
let mut p = r.seed.object_id(0).0.to_vec();
|
||||
p.extend_from_slice(&body);
|
||||
let mut v = Vec::new();
|
||||
fury::encrypt_to(&mut v, &s.inner.recipient, &p, &mut crate::rng::os());
|
||||
v
|
||||
};
|
||||
let damaged: Vec<(&str, Vec<u8>)> = vec![
|
||||
("flipped", {
|
||||
let mut v = orig.clone();
|
||||
let n = v.len() - 100;
|
||||
v[n] ^= 1;
|
||||
v
|
||||
}),
|
||||
("truncated", orig[..orig.len() - 1].to_vec()),
|
||||
("missing final chunk", orig[..orig.len() - 1000].to_vec()),
|
||||
("trailing", [&orig[..], b"x"].concat()),
|
||||
("empty", Vec::new()),
|
||||
("foreign", foreign),
|
||||
("short padding", short_pad),
|
||||
];
|
||||
for (what, bytes) in damaged {
|
||||
std::fs::write(&obj, &bytes).unwrap();
|
||||
let out = format!("out-{}", what.replace(' ', "-"));
|
||||
assert!(get_bytes(&s, r, &d.0, &out).is_err(), "{what} accepted");
|
||||
assert!(!d.0.join(&out).exists(), "{what} published output");
|
||||
}
|
||||
assert_eq!(std::fs::read_dir(&d.0).unwrap().count(), 1, "temporary files left behind");
|
||||
std::fs::remove_file(&obj).unwrap();
|
||||
assert!(get_bytes(&s, r, &d.0, "missing").is_err());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn extraction_never_replaces_symlinks() {
|
||||
let (d, s) = vault("symlink");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
add_bytes(&s, &mut cat, "f", b"data");
|
||||
let r = cat.current("f").unwrap();
|
||||
std::os::unix::fs::symlink(d.0.join("target"), d.0.join("dangling")).unwrap();
|
||||
assert!(get_bytes(&s, r, &d.0, "dangling").is_err());
|
||||
assert!(!d.0.join("target").exists(), "followed a dangling symlink");
|
||||
std::fs::write(d.0.join("exists"), b"keep").unwrap();
|
||||
assert!(get_bytes(&s, r, &d.0, "exists").is_err());
|
||||
assert_eq!(std::fs::read(d.0.join("exists")).unwrap(), b"keep");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_record_names() {
|
||||
let (_d, s) = vault("names");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
// 1024 bytes is the limit; '<' is escaped six-fold in JSON, as in Go.
|
||||
let long = "a".repeat(NAME_LIMIT);
|
||||
add_bytes(&s, &mut cat, &long, b"x");
|
||||
let escaped = "<".repeat(NAME_LIMIT);
|
||||
let e = s
|
||||
.add_reader(&mut cat, &mut &b"x"[..], 1, 0o600, Instant::now(), &name(&escaped))
|
||||
.unwrap_err();
|
||||
assert_eq!(e.msg, "catalogue record too large");
|
||||
assert!(VaultName::new(&"a".repeat(NAME_LIMIT + 1)).is_none());
|
||||
assert_eq!(s.catalogue().unwrap().names().collect::<Vec<_>>(), [long.as_str()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn duplicate_records_recover() {
|
||||
let (_d, s) = vault("dups");
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
add_bytes(&s, &mut cat, "f", b"old");
|
||||
// An interrupted replace leaves two records for one name.
|
||||
let old = cat.current("f").unwrap();
|
||||
let mut dup = old.copy_as(name("f"), Id::random(&mut crate::rng::os()));
|
||||
dup.added = Instant {
|
||||
secs: old.added.secs + 10,
|
||||
nanos: 0,
|
||||
};
|
||||
s.inner.write_record(&dup).unwrap();
|
||||
let cat = s.catalogue().unwrap();
|
||||
assert_eq!(cat.records("f").len(), 2);
|
||||
assert_eq!(cat.current("f").unwrap().added, dup.added, "newest first");
|
||||
let mut cat = cat;
|
||||
add_bytes(&s, &mut cat, "f", b"new");
|
||||
assert_eq!(s.catalogue().unwrap().records("f").len(), 1, "next write cleans up");
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Validated values that reach paths, the catalogue and extraction.
|
||||
//! Each is constructed only through a checking constructor, so raw
|
||||
//! untrusted strings cannot be used where one of these is expected.
|
||||
|
||||
use crate::secret::SecretArray;
|
||||
use crate::util::Instant;
|
||||
use hmac::{KeyInit, Mac};
|
||||
use zeroize::Zeroizing;
|
||||
|
||||
pub const NAME_LIMIT: usize = 1024;
|
||||
pub const FILE_LIMIT: u64 = 1 << 40;
|
||||
pub const ID_SIZE: usize = 16;
|
||||
pub const SEED_SIZE: usize = 32;
|
||||
pub const RECORD_SIZE: usize = 4096;
|
||||
pub const MIN_PADDING: u64 = 64 << 10;
|
||||
/// The largest bucket, and so the largest object plaintext.
|
||||
pub const CHUNK_SIZE: u64 = MIN_PADDING << 6;
|
||||
/// File bytes per object: an object's plaintext starts with its 16-byte ID.
|
||||
pub const CHUNK_DATA: u64 = CHUNK_SIZE - ID_SIZE as u64;
|
||||
|
||||
/// A name inside the vault: non-empty UTF-8 of at most 1024 bytes, already
|
||||
/// clean in the sense of Go's `path.Clean`, relative, not `.`/`..` or under
|
||||
/// `..`, without backslashes or control characters.
|
||||
#[derive(Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
pub struct VaultName(String);
|
||||
|
||||
impl VaultName {
|
||||
pub fn new(name: &str) -> Option<VaultName> {
|
||||
valid_name(name).then(|| VaultName(name.to_string()))
|
||||
}
|
||||
|
||||
pub fn from_bytes(b: &[u8]) -> Option<VaultName> {
|
||||
VaultName::new(std::str::from_utf8(b).ok()?)
|
||||
}
|
||||
|
||||
pub fn as_str(&self) -> &str {
|
||||
&self.0
|
||||
}
|
||||
|
||||
/// Path components, each a plain non-empty segment.
|
||||
pub fn components(&self) -> impl Iterator<Item = &str> {
|
||||
self.0.split('/')
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for VaultName {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.0)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn valid_name(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= NAME_LIMIT
|
||||
&& crate::util::clean(name) == name
|
||||
&& !name.starts_with('/')
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& !name.starts_with("../")
|
||||
&& !name.contains('\\')
|
||||
&& !name.chars().any(char::is_control)
|
||||
}
|
||||
|
||||
/// A 16-byte object or record ID, shown and stored as lowercase hex.
|
||||
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)]
|
||||
pub struct Id(pub [u8; ID_SIZE]);
|
||||
|
||||
impl Id {
|
||||
pub fn random(rng: &mut crate::rng::Rng) -> Id {
|
||||
Id(rng.array())
|
||||
}
|
||||
|
||||
/// Only exact lowercase hex of the right length, as Go's `validHex`.
|
||||
pub fn parse(s: &str) -> Option<Id> {
|
||||
let b = parse_hex(s)?;
|
||||
Some(Id(b.try_into().ok()?))
|
||||
}
|
||||
|
||||
pub fn hex(&self) -> String {
|
||||
hex(&self.0)
|
||||
}
|
||||
|
||||
pub fn shard(&self) -> String {
|
||||
format!("{:02x}", self.0[0])
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hex(b: &[u8]) -> String {
|
||||
const D: &[u8; 16] = b"0123456789abcdef";
|
||||
let mut s = String::with_capacity(b.len() * 2);
|
||||
for x in b {
|
||||
s.push(D[(x >> 4) as usize] as char);
|
||||
s.push(D[(x & 15) as usize] as char);
|
||||
}
|
||||
s
|
||||
}
|
||||
|
||||
pub fn parse_hex(s: &str) -> Option<Vec<u8>> {
|
||||
let b = s.as_bytes();
|
||||
if b.len() % 2 != 0 {
|
||||
return None;
|
||||
}
|
||||
let v = |c: u8| match c {
|
||||
b'0'..=b'9' => Some(c - b'0'),
|
||||
b'a'..=b'f' => Some(c - b'a' + 10),
|
||||
_ => None,
|
||||
};
|
||||
b.chunks(2).map(|p| Some(v(p[0])? << 4 | v(p[1])?)).collect()
|
||||
}
|
||||
|
||||
/// The secret per-file seed whose HMAC derives the file's object IDs.
|
||||
pub struct ObjectSeed(SecretArray<SEED_SIZE>);
|
||||
|
||||
impl ObjectSeed {
|
||||
pub fn random(rng: &mut crate::rng::Rng) -> ObjectSeed {
|
||||
let mut s = SecretArray::zero();
|
||||
rng.fill(s.expose_mut());
|
||||
ObjectSeed(s)
|
||||
}
|
||||
|
||||
pub fn parse(s: &str) -> Option<ObjectSeed> {
|
||||
let b = Zeroizing::new(parse_hex(s)?);
|
||||
SecretArray::from_slice(&b).map(ObjectSeed)
|
||||
}
|
||||
|
||||
pub fn hex(&self) -> Zeroizing<String> {
|
||||
Zeroizing::new(hex(self.0.expose()))
|
||||
}
|
||||
|
||||
/// An explicit copy, for a record copied under another name.
|
||||
pub fn duplicate(&self) -> ObjectSeed {
|
||||
ObjectSeed(SecretArray::from_slice(self.0.expose()).expect("same size"))
|
||||
}
|
||||
|
||||
pub fn same_as(&self, other: &ObjectSeed) -> bool {
|
||||
crate::secret::subtle_eq::ct_eq(self.0.expose(), other.0.expose())
|
||||
}
|
||||
|
||||
/// The `i`th object's ID: HMAC-SHA-256(seed, big-endian i)[:16].
|
||||
pub fn object_id(&self, i: u64) -> Id {
|
||||
let mut m = <hmac::Hmac<sha2::Sha256> as KeyInit>::new_from_slice(self.0.expose()).expect("any key size");
|
||||
m.update(&(i as i64).to_be_bytes());
|
||||
let out = m.finalize().into_bytes();
|
||||
let mut id = [0u8; ID_SIZE];
|
||||
id.copy_from_slice(&out[..ID_SIZE]);
|
||||
Id(id)
|
||||
}
|
||||
}
|
||||
|
||||
/// How many objects hold a file of `size` bytes: at least one.
|
||||
pub fn object_count(size: u64) -> u64 {
|
||||
size.div_ceil(CHUNK_DATA).max(1)
|
||||
}
|
||||
|
||||
/// `size` rounded up to the next power-of-two bucket of at least 64 KiB.
|
||||
pub fn padded_size(size: u64) -> u64 {
|
||||
let mut p = MIN_PADDING;
|
||||
while p < size {
|
||||
p *= 2;
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// A decrypted, validated catalogue record. Only the store constructs
|
||||
/// these, from authenticated plaintext or from its own writes.
|
||||
pub struct Record {
|
||||
pub name: VaultName,
|
||||
pub seed: ObjectSeed,
|
||||
pub size: u64,
|
||||
pub mode: u32,
|
||||
pub modified: Instant,
|
||||
pub added: Instant,
|
||||
/// The record file's ID; not stored inside the record.
|
||||
pub id: Id,
|
||||
}
|
||||
|
||||
impl Record {
|
||||
/// The bytes of object `i` that are file data, after its ID prefix.
|
||||
pub fn part(&self, i: u64) -> u64 {
|
||||
let before = i * CHUNK_DATA;
|
||||
(self.size - before).min(CHUNK_DATA)
|
||||
}
|
||||
|
||||
/// A copy under another name, sharing the objects.
|
||||
pub fn copy_as(&self, name: VaultName, id: Id) -> Record {
|
||||
Record {
|
||||
name,
|
||||
seed: self.seed.duplicate(),
|
||||
size: self.size,
|
||||
mode: self.mode,
|
||||
modified: self.modified,
|
||||
added: self.added,
|
||||
id,
|
||||
}
|
||||
}
|
||||
|
||||
/// The record's JSON exactly as Go's `encoding/json` writes it, or None
|
||||
/// if a timestamp is outside what Go can encode.
|
||||
pub fn to_json(&self) -> Option<Zeroizing<Vec<u8>>> {
|
||||
let mut o = Zeroizing::new(Vec::with_capacity(256));
|
||||
o.extend_from_slice(b"{\"name\":");
|
||||
go_json_string(&mut o, &self.name.0);
|
||||
o.extend_from_slice(b",\"seed\":\"");
|
||||
o.extend_from_slice(self.seed.hex().as_bytes());
|
||||
o.extend_from_slice(format!("\",\"size\":{},\"mode\":{},\"modified\":\"", self.size, self.mode).as_bytes());
|
||||
o.extend_from_slice(self.modified.to_rfc3339()?.as_bytes());
|
||||
o.extend_from_slice(b"\",\"added\":\"");
|
||||
o.extend_from_slice(self.added.to_rfc3339()?.as_bytes());
|
||||
o.extend_from_slice(b"\"}");
|
||||
Some(o)
|
||||
}
|
||||
|
||||
/// Parses and validates record JSON. Stricter than Go in one way: field
|
||||
/// names must match exactly and every field must be present, as Go
|
||||
/// always writes them.
|
||||
pub fn from_json(b: &[u8], id: Id) -> Result<Record, String> {
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Raw {
|
||||
name: String,
|
||||
seed: Zeroizing<String>,
|
||||
size: i64,
|
||||
mode: u32,
|
||||
modified: String,
|
||||
added: String,
|
||||
}
|
||||
let r: Raw = serde_json::from_slice(b).map_err(|e| e.to_string())?;
|
||||
let malformed = || "malformed record".to_string();
|
||||
let name = VaultName::new(&r.name).ok_or_else(malformed)?;
|
||||
let seed = ObjectSeed::parse(&r.seed).ok_or_else(malformed)?;
|
||||
if r.size < 0 || r.size as u64 > FILE_LIMIT {
|
||||
return Err(malformed());
|
||||
}
|
||||
let modified = Instant::parse_rfc3339(&r.modified).ok_or_else(|| format!("parsing time {:?}", r.modified))?;
|
||||
let added = Instant::parse_rfc3339(&r.added).ok_or_else(|| format!("parsing time {:?}", r.added))?;
|
||||
Ok(Record {
|
||||
name,
|
||||
seed,
|
||||
size: r.size as u64,
|
||||
mode: r.mode,
|
||||
modified,
|
||||
added,
|
||||
id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's JSON string encoding with HTML escaping (its default).
|
||||
fn go_json_string(o: &mut Vec<u8>, s: &str) {
|
||||
o.push(b'"');
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'"' => o.extend_from_slice(b"\\\""),
|
||||
'\\' => o.extend_from_slice(b"\\\\"),
|
||||
'\n' => o.extend_from_slice(b"\\n"),
|
||||
'\r' => o.extend_from_slice(b"\\r"),
|
||||
'\t' => o.extend_from_slice(b"\\t"),
|
||||
'\u{8}' => o.extend_from_slice(b"\\b"),
|
||||
'\u{c}' => o.extend_from_slice(b"\\f"),
|
||||
'<' | '>' | '&' | '\u{2028}' | '\u{2029}' => o.extend_from_slice(format!("\\u{:04x}", c as u32).as_bytes()),
|
||||
c if (c as u32) < 0x20 => o.extend_from_slice(format!("\\u{:04x}", c as u32).as_bytes()),
|
||||
c => {
|
||||
let mut b = [0u8; 4];
|
||||
o.extend_from_slice(c.encode_utf8(&mut b).as_bytes());
|
||||
}
|
||||
}
|
||||
}
|
||||
o.push(b'"');
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::Value;
|
||||
|
||||
fn v() -> Value {
|
||||
serde_json::from_str(include_str!("../../tests/vectors/store.json")).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_match_go() {
|
||||
for (h, want) in v()["valid_name_hex"].as_object().unwrap() {
|
||||
let b = parse_hex(h).unwrap();
|
||||
assert_eq!(
|
||||
VaultName::from_bytes(&b).is_some(),
|
||||
want.as_bool().unwrap(),
|
||||
"{:?}",
|
||||
String::from_utf8_lossy(&b)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_ids_and_sizes_match_go() {
|
||||
let v = v();
|
||||
let o = &v["object_ids"];
|
||||
let seed = ObjectSeed::parse(o["seed"].as_str().unwrap()).unwrap();
|
||||
for (i, id) in o["ids"].as_array().unwrap().iter().enumerate() {
|
||||
assert_eq!(seed.object_id(i as u64).hex(), id.as_str().unwrap());
|
||||
}
|
||||
assert_eq!(
|
||||
seed.object_id(o["big_index_value"].as_u64().unwrap()).hex(),
|
||||
o["big_index"].as_str().unwrap()
|
||||
);
|
||||
for (n, want) in v["padded_size"].as_object().unwrap() {
|
||||
assert_eq!(padded_size(n.parse().unwrap()), want.as_u64().unwrap(), "paddedSize({n})");
|
||||
}
|
||||
for (n, want) in v["object_count"].as_object().unwrap() {
|
||||
assert_eq!(object_count(n.parse().unwrap()), want.as_u64().unwrap(), "objectCount({n})");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_json_matches_go() {
|
||||
for r in v()["records"].as_array().unwrap() {
|
||||
let json = r["json"].as_str().unwrap().as_bytes();
|
||||
let n = r["name"].as_str().unwrap();
|
||||
// The name's escaping, whether or not it is a valid vault name.
|
||||
let mut enc = Vec::new();
|
||||
go_json_string(&mut enc, n);
|
||||
let text = r["json"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
&enc[..],
|
||||
text["{\"name\":".len()..text.find(",\"seed\"").unwrap()].as_bytes(),
|
||||
"escaping of {n:?}"
|
||||
);
|
||||
if VaultName::new(n).is_none() {
|
||||
assert!(Record::from_json(json, Id([0; 16])).is_err());
|
||||
continue;
|
||||
}
|
||||
let rec = Record::from_json(json, Id([0; 16])).unwrap();
|
||||
assert_eq!(rec.name.as_str(), r["name"].as_str().unwrap());
|
||||
// Go's UnixNano wraps outside about 1678-2262; so does this.
|
||||
let ns = |i: Instant| i.secs.wrapping_mul(1_000_000_000).wrapping_add(i.nanos as i64);
|
||||
assert_eq!(ns(rec.modified), r["modified_unix_nano"].as_i64().unwrap());
|
||||
assert_eq!(ns(rec.added), r["added_unix_nano"].as_i64().unwrap());
|
||||
assert_eq!(&rec.to_json().unwrap()[..], json, "byte-identical encoding");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_json_rejects_bad_records() {
|
||||
let good = br#"{"name":"a","seed":"000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f","size":1,"mode":420,"modified":"2020-01-01T00:00:00Z","added":"2020-01-01T00:00:00Z"}"#;
|
||||
assert!(Record::from_json(good, Id([0; 16])).is_ok());
|
||||
let s = std::str::from_utf8(good).unwrap();
|
||||
for bad in [
|
||||
s.replace("\"a\"", "\"../a\""),
|
||||
s.replace("\"a\"", "\"\""),
|
||||
s.replace("0001", "0X01"),
|
||||
s.replace("0001", "0A01"),
|
||||
s.replace("1e1f", "1e"),
|
||||
s.replace("\"size\":1", "\"size\":-1"),
|
||||
s.replace("\"size\":1", "\"size\":1099511627777"),
|
||||
s.replace("\"size\":1", "\"size\":1.5"),
|
||||
s.replace("\"mode\":420", "\"mode\":-1"),
|
||||
s.replace("}", ",\"extra\":1}"),
|
||||
s.replace("\"name\"", "\"Name\""),
|
||||
s.replace("2020-01-01T00:00:00Z\",\"added", "yesterday\",\"added"),
|
||||
s.replace(",\"mode\":420", ""),
|
||||
format!("{s}trailing"),
|
||||
] {
|
||||
assert!(Record::from_json(bad.as_bytes(), Id([0; 16])).is_err(), "{bad}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hex_is_strict() {
|
||||
assert!(Id::parse("00112233445566778899aabbccddeeff").is_some());
|
||||
assert!(Id::parse("00112233445566778899AABBCCDDEEFF").is_none());
|
||||
assert!(Id::parse("00112233445566778899aabbccddeef").is_none());
|
||||
assert!(Id::parse("00112233445566778899aabbccddeeff00").is_none());
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user