Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8aee1f0f04 | ||
|
|
06cf7355dc | ||
|
|
31dbdc7009 | ||
|
|
0877cc7447 | ||
|
|
438180b6fb | ||
|
|
25496ddb97 | ||
|
|
7506c1dbef | ||
|
|
91d49aeecf | ||
|
|
25f50ed73b | ||
|
|
53626a7de7 | ||
|
|
afdfbc5ebd | ||
|
|
53968f8822 | ||
|
|
cddde5ca52 | ||
|
|
a98931779a | ||
|
|
aedc605a82 | ||
|
|
24ebede9bb | ||
|
|
b97c690ef7 | ||
|
|
a6610449f2 | ||
|
|
c324f6a306 | ||
|
|
69f4b90777 | ||
|
|
4c1507c463 | ||
|
|
ec204f0ea2 | ||
|
|
0d45ecea59 | ||
|
|
730877823e | ||
|
|
1403be8b1c | ||
|
|
fc5a856487 | ||
|
|
55af82f95f |
@@ -0,0 +1,111 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Oracle-backed interoperability tests must run, not skip, in CI.
|
||||
VAULT_REQUIRE_ORACLE: "1"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- run: cargo fmt --check
|
||||
- run: cargo clippy --locked --all-targets --all-features -- -D warnings
|
||||
|
||||
test:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, macos-15-intel, windows-2025]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
# The Go toolchain builds the test-only oracle; vault itself needs no Go.
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: "1.26.x"
|
||||
cache-dependency-path: oracle/go/go.sum
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- name: test
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ $RUNNER_OS == Windows ]]; then unset VAULT_REQUIRE_ORACLE; fi
|
||||
cargo test --locked --all-targets --all-features
|
||||
- run: cargo test --locked --doc
|
||||
- run: cargo build --locked --release
|
||||
- name: oracle self-test
|
||||
if: runner.os == 'Linux'
|
||||
working-directory: oracle/go
|
||||
run: go test ./...
|
||||
|
||||
cross:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: "1.26.x"
|
||||
cache-dependency-path: oracle/go/go.sum
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- name: zig and cargo-zigbuild
|
||||
run: |
|
||||
curl -sSLo zig.tar.xz https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz
|
||||
echo "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 zig.tar.xz" | sha256sum -c
|
||||
mkdir -p "$HOME/zig" && tar -xJf zig.tar.xz -C "$HOME/zig" --strip-components=1
|
||||
echo "$HOME/zig" >>"$GITHUB_PATH"
|
||||
cargo install cargo-zigbuild --version 0.23.4 --locked
|
||||
cargo install cross --version 0.2.5 --locked
|
||||
- name: targets
|
||||
run: |
|
||||
rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl i686-unknown-linux-musl \
|
||||
arm-unknown-linux-musleabihf armv7-unknown-linux-musleabihf riscv64gc-unknown-linux-musl \
|
||||
loongarch64-unknown-linux-musl powerpc64le-unknown-linux-musl \
|
||||
s390x-unknown-linux-gnu x86_64-pc-windows-gnu i686-pc-windows-gnu aarch64-pc-windows-gnullvm \
|
||||
x86_64-unknown-freebsd i686-unknown-freebsd
|
||||
- name: build
|
||||
run: |
|
||||
# Everything but macOS, which the test job builds on a Mac.
|
||||
TARGETS="linux/amd64 linux/arm64 linux/386 linux/armv6 linux/armv7 linux/riscv64 linux/loong64 \
|
||||
linux/ppc64le linux/s390x windows/amd64 windows/386 windows/arm64 freebsd/amd64 freebsd/386" ./build.sh
|
||||
- name: run builds under qemu, cross-checked with the Go oracle
|
||||
run: |
|
||||
sudo apt-get update -q && sudo apt-get install -yq qemu-user libc6-s390x-cross
|
||||
(cd oracle/go && go build -o ../../target/vault-go .)
|
||||
for t in "arm64 qemu-aarch64" "armv7 qemu-arm" "riscv64 qemu-riscv64" "ppc64le qemu-ppc64le" \
|
||||
"s390x qemu-s390x -L /usr/s390x-linux-gnu"; do
|
||||
EMU="${t#* }" scripts/smoke.sh "dist/vault-linux-${t%% *}" target/vault-go
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist/
|
||||
|
||||
supply-chain:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- run: cargo install cargo-deny --version 0.20.2 --locked
|
||||
- run: cargo deny --locked check
|
||||
|
||||
fuzz:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup toolchain install nightly-2026-10-01 --profile minimal
|
||||
- run: cargo +nightly-2026-10-01 install cargo-fuzz --version 0.13.2 --locked
|
||||
- name: a short run of every target
|
||||
working-directory: fuzz
|
||||
run: |
|
||||
for t in $(cargo +nightly-2026-10-01 fuzz list); do
|
||||
cargo +nightly-2026-10-01 fuzz run "$t" -- -max_total_time=60
|
||||
done
|
||||
@@ -6,3 +6,8 @@
|
||||
/docs/guide/version.tex
|
||||
/docs/guide/vault-guide.pdf
|
||||
/docs/guide/*.log
|
||||
/target
|
||||
fuzz/target
|
||||
fuzz/corpus
|
||||
fuzz/artifacts
|
||||
fuzz/coverage
|
||||
|
||||
@@ -4,59 +4,93 @@ This file provides guidance to Claude Code when working with this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
vault is a post-quantum encrypted file vault: one `package main` at the repo
|
||||
root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
|
||||
vault is a post-quantum encrypted file vault: one Rust package (`Cargo.toml`
|
||||
at the root, toolchain pinned in `rust-toolchain.toml`) that builds a single
|
||||
`vault` binary. The library (`src/lib.rs`) exists so tests and benchmarks can
|
||||
reach the same code; `src/main.rs` only calls `cli::main`.
|
||||
|
||||
- `main.go` - CLI: `genkey`, `init`, `add` (file, directory
|
||||
or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`.
|
||||
- `keyfile.go` - key file with two passphrase slots (Argon2id +
|
||||
XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second
|
||||
slot is either a decoy identity or random bytes; slot order is random.
|
||||
- `store.go` - on-disk vault: `records/` holds fixed-size encrypted catalogue
|
||||
records, `objects/` holds file bodies cut into 4 MiB chunks (last one
|
||||
bucket-padded) whose IDs are HMAC'd from a per-record seed. Both are
|
||||
sharded into 256 `xx/` subdirectories by ID prefix. Records a key cannot
|
||||
open are skipped, not treated as errors.
|
||||
- `lock*.go` - vault lock on `DIR/.lock`: fcntl on unix, LockFileEx on
|
||||
Windows, an exclusive lock file elsewhere. Writers lock exclusively,
|
||||
readers shared, taken after the passphrase prompt.
|
||||
- `docs/guide` - LaTeX user guide (`build-pdf.sh` via Tectonic; `shots.py`
|
||||
re-shoots screenshots from the real binary). Keep it in step with the CLI.
|
||||
- `build.sh` - cross-compiles every `go tool dist list` target except
|
||||
iOS and android/386, amd64, arm (cgo-only).
|
||||
- `internal/pqcrypt` - file encryption (header + STREAM ChaCha20-Poly1305),
|
||||
suite registry, hybrid FrodoKEM-640 + ML-KEM-768 default suite.
|
||||
- `internal/format`, `internal/stream`, `internal/term` - header encoding,
|
||||
chunked payload encryption, terminal passphrase prompts.
|
||||
- `src/cli.rs` - commands and Go-`flag`-compatible parsing: `genkey`,
|
||||
`init`, `add` (file, directory or URL), `migrate`, `mirror`, `absorb`,
|
||||
`ls`/`list`, `get` (`-r`), `mv`/`cp` (`transfers` resolves names like
|
||||
mv(1)), `rm` (`-r`, asks unless `-f`), `unlock`/`lock`, hidden `__agent`.
|
||||
- `src/agent.rs` - the unlock agent: `unlock` re-executes the binary as
|
||||
`__agent` (passphrase variables stripped, secret over a pipe); it hardens
|
||||
itself (`platform::memory`) before reading the key, serves it on a Unix
|
||||
socket in a 0700 per-user dir until a deadline (fixed or `--idle`).
|
||||
Wire-compatible with the Go agent. Not available on Windows.
|
||||
- `src/keyfile.rs` - key file: two Argon2id/XChaCha20-Poly1305 slots, decoy
|
||||
or random bytes in the second, random order, both always tried.
|
||||
- `src/crypto/` - `frodo.rs` (FrodoKEM-640-SHAKE ported from CIRCL v1.6.4),
|
||||
`kem.rs` (hybrid suite 0x0001 with RustCrypto ml-kem, `Identity`),
|
||||
`header.rs` (Fury header format), `fury.rs` (file encryption), `stream.rs`
|
||||
(STREAM, including random access).
|
||||
- `src/store/` - on-disk vault: `records/`, `objects/` sharded by ID prefix,
|
||||
catalogue, add/remove/copy/extract, filler; `types.rs` holds the validated
|
||||
types (`VaultName`, `Id`, `ObjectSeed`, `Record`) and record JSON.
|
||||
- `src/pipeline.rs` - the one bounded worker pool and the eight reusable
|
||||
object buffers; coordinators run on the caller's thread.
|
||||
- `src/platform/` - all `unsafe` and OS specifics: `fs.rs` (no-clobber
|
||||
publish, `Dir` confined extraction, epoch times, dir sync), `lock.rs`
|
||||
(fcntl / LockFileEx / lock file), `memory.rs` (mlock, undumpable),
|
||||
`term.rs` (prompts).
|
||||
- `src/net/` - `add URL` and `mirror` with limits; `url.rs` (Go `net/url`
|
||||
semantics), `html.rs` (tokenizer and link rewriting).
|
||||
- `src/absorb.rs` - tar (optionally gzipped) from stdin, streamed in.
|
||||
- `src/secret.rs`, `src/rng.rs`, `src/util.rs`, `src/error.rs` - secret
|
||||
buffers, randomness (seeded only under `cfg(test)`), Go-compatible
|
||||
parsing/printing helpers, the error type.
|
||||
- `oracle/go` - the frozen Go implementation (test-only reference, see
|
||||
`oracle/README.md`); `oracle/gen-vectors.sh` writes `tests/vectors`.
|
||||
- `tests/` - CLI and interoperability tests against the real binaries;
|
||||
`fuzz/` - cargo-fuzz targets; `benches/micro.rs` - micro-benchmarks.
|
||||
- `docs/` - man pages, `COMPATIBILITY.md`, `MIGRATION.md`, `PLATFORMS.md`,
|
||||
`BENCHMARKS.md`, and the LaTeX guide (`docs/guide`, `build-pdf.sh` via
|
||||
Tectonic; `shots.py` re-shoots screenshots from the real binary). Keep
|
||||
them in step with the CLI.
|
||||
- `build.sh` - cross-builds every supported target (see `PLATFORMS.md`).
|
||||
- `packaging/package.sh` builds and inspects dixon-vault packages locally;
|
||||
`packaging/repo.sh [TAG]` signs and publishes them to repo.dixon.cx.
|
||||
|
||||
## Commands
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go build .
|
||||
./build.sh # all platforms, into ./dist
|
||||
cargo test --locked # needs go on PATH for interop tests
|
||||
cargo clippy --locked --all-targets --all-features -- -D warnings
|
||||
cargo fmt --check
|
||||
cargo build --release --locked
|
||||
./build.sh # all buildable targets, into ./dist
|
||||
TARGETS="linux/amd64" ./build.sh # a subset
|
||||
cargo deny --locked check # advisories and licences
|
||||
(cd fuzz && cargo +nightly fuzz run header)
|
||||
```
|
||||
|
||||
End-to-end check:
|
||||
|
||||
```sh
|
||||
export VAULT_PASSPHRASE=test VAULT_KEY=$PWD/k VAULT_DIR=$PWD/s
|
||||
./vault genkey && ./vault init && ./vault add README.md docs && ./vault list
|
||||
target/release/vault genkey && target/release/vault init && target/release/vault add README.md docs && target/release/vault list
|
||||
```
|
||||
|
||||
## Invariants
|
||||
|
||||
- Never write a seed or private key unencrypted.
|
||||
- Keep suite IDs, stanza tags, HKDF info strings and `fileKeySize` (32) stable.
|
||||
- The on-disk formats are frozen and shared with the Go version; check any
|
||||
change against `docs/COMPATIBILITY.md` and the oracle vectors.
|
||||
- Never write a seed or private key unencrypted (the agent gets it by pipe).
|
||||
- Keep suite IDs, stanza tags, HKDF info strings and the 32-byte file key stable.
|
||||
- Every ciphertext's and shard directory's mtime is pinned to the epoch;
|
||||
records stay one fixed size; objects stay at most 4 MiB and bucket-padded.
|
||||
These hide metadata from the storage host.
|
||||
- An object's plaintext starts with its own ID, binding it to its record
|
||||
and its position in the file.
|
||||
- `get` publishes output only after every object has authenticated.
|
||||
- `get` publishes output only after every object has authenticated, with a
|
||||
no-replace primitive; never fall back to check-then-rename.
|
||||
- Decoy deniability: both key slots stay the same size and are both always
|
||||
tried; `init` always writes filler; foreign records are skipped silently.
|
||||
- Every command that changes the vault holds the exclusive lock.
|
||||
- Keep dependencies pure Go and CGO_ENABLED=0 builds working.
|
||||
- Every command that changes the vault holds the exclusive lock (POSIX
|
||||
fcntl locks, compatible with Go's).
|
||||
- Crash-safe order: objects before records, new records before removing old
|
||||
ones, record removal (synced) before object removal.
|
||||
- Secrets live in `SecretArray`/`SecretVec`/`LockedBuf`: no Clone, Debug or
|
||||
serialisation. Production randomness is the OS only.
|
||||
- `unsafe` only in `src/platform`, each block with a SAFETY comment.
|
||||
- Diagnostics go to standard error, lower-case, no trailing full stops.
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
[package]
|
||||
name = "vault"
|
||||
version = "0.6.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.99"
|
||||
license = "GPL-3.0-or-later"
|
||||
description = "Post-quantum encrypted file vault with a decoy passphrase"
|
||||
homepage = "https://dixon.cx/vault"
|
||||
repository = "https://github.com/andydixon/vault"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
argon2 = { version = "0.6", default-features = false, features = ["alloc", "zeroize", "parallel"] }
|
||||
chacha20poly1305 = { version = "0.11", default-features = false, features = ["alloc", "zeroize"] }
|
||||
hkdf = "0.13"
|
||||
hmac = { version = "0.13", features = ["zeroize"] }
|
||||
sha2 = { version = "0.11", default-features = false, features = ["zeroize"] }
|
||||
sha3 = { version = "0.11", default-features = false, features = ["zeroize"] }
|
||||
ml-kem = { version = "0.3.2", features = ["zeroize", "hazmat"] }
|
||||
zeroize = { version = "1.9", features = ["derive", "serde"] }
|
||||
getrandom = "0.4"
|
||||
base64 = "0.23"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
jiff = "0.2"
|
||||
tar = { version = "0.4.46", default-features = false }
|
||||
flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] }
|
||||
ureq = { version = "3.4", default-features = false, features = ["rustls", "gzip"] }
|
||||
|
||||
[features]
|
||||
default = ["system-roots"]
|
||||
# Verify TLS certificates against the operating system's trust store. Off,
|
||||
# the bundled Mozilla roots are used (for cross-built macOS binaries).
|
||||
system-roots = ["ureq/platform-verifier"]
|
||||
|
||||
[dev-dependencies]
|
||||
aes = "0.9"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
rustix = { version = "1.1.5", features = ["fs", "process", "mm", "termios", "event", "net", "stdio", "param"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_IO", "Win32_System_Memory", "Win32_Security"] }
|
||||
|
||||
# Tests run the real KEMs and Argon2id; unoptimised they are far too slow.
|
||||
[profile.dev]
|
||||
opt-level = 1
|
||||
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1
|
||||
lto = true
|
||||
strip = true
|
||||
|
||||
[[bench]]
|
||||
name = "micro"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "aead_probe"
|
||||
harness = false
|
||||
@@ -11,18 +11,48 @@ whatever the file's size. Object names derive from a secret seed in the
|
||||
record, so nothing on disk links a file's pieces. Records and objects are
|
||||
spread over 256 subfolders (`records/3f/3fa9…`) to keep directories small.
|
||||
|
||||
## Install
|
||||
|
||||
On Debian, Ubuntu, Fedora, RHEL, openSUSE and Arch, install the
|
||||
`dixon-vault` package from the signed repository at repo.dixon.cx; see
|
||||
https://dixon.cx/repo for the two or three commands. (The package isn't
|
||||
called `vault` because that name is HashiCorp Vault's.) Every platform also
|
||||
has a single binary at https://download.dixon.cx/vault/.
|
||||
|
||||
Releases are packaged and published with `packaging/repo.sh [TAG]`.
|
||||
|
||||
## Build
|
||||
|
||||
vault is written in Rust (the toolchain is pinned in `rust-toolchain.toml`;
|
||||
`rustup` installs it on first use). Releases up to 0.5 were Go; the Go code
|
||||
lives on, frozen, in `oracle/go` as the test reference. Nothing at run time
|
||||
needs Go.
|
||||
|
||||
```sh
|
||||
go build . # produces ./vault
|
||||
go test ./...
|
||||
./build.sh # every OS and CPU architecture, into ./dist
|
||||
cargo build --release --locked # produces target/release/vault
|
||||
cargo test --locked # unit, CLI and Go-interoperability tests
|
||||
./build.sh # every target this host can build, into ./dist
|
||||
TARGETS="linux/amd64 windows/amd64" ./build.sh
|
||||
```
|
||||
|
||||
`build.sh` cross-compiles static binaries for every target in
|
||||
`go tool dist list` except iOS and 32-bit or x86 Android, which need cgo. It builds 32-bit ARM Linux for
|
||||
both armv6 and armv7, and writes `dist/SHA256SUMS`. Set `TARGETS` to build a
|
||||
subset, for example `TARGETS="linux/amd64 darwin/arm64" ./build.sh`.
|
||||
`build.sh` cross-compiles with `cargo-zigbuild` and `zig` (Linux, statically
|
||||
linked with musl, and Windows), on a Mac for macOS, and with `cross` and
|
||||
Docker for FreeBSD. The release covers Linux, Windows, macOS and FreeBSD. It names binaries as before
|
||||
(`vault-linux-arm64`, `vault-windows-amd64.exe`, …), builds 32-bit ARM Linux
|
||||
for both armv6 and armv7, and writes `dist/SHA256SUMS`. `docs/PLATFORMS.md`
|
||||
lists every target, how it is built and tested, and the Go targets Rust
|
||||
cannot build.
|
||||
|
||||
The interoperability tests build the Go oracle when `go` is installed and
|
||||
are skipped (and say so) otherwise; set `VAULT_REQUIRE_ORACLE=1` to make a
|
||||
missing oracle an error, as CI does. `oracle/gen-vectors.sh` regenerates the
|
||||
Go test vectors. Packages are built and inspected locally, unsigned, with
|
||||
`packaging/package.sh`, and published with `packaging/repo.sh [TAG]`.
|
||||
|
||||
Further reading: `SECURITY.md` (what is and is not protected, memory and
|
||||
filesystem guarantees), `docs/COMPATIBILITY.md` (the formats),
|
||||
`docs/MIGRATION.md` (what changed from the Go version) and
|
||||
`docs/BENCHMARKS.md`.
|
||||
|
||||
## User guide
|
||||
|
||||
@@ -48,29 +78,55 @@ vault add ./photos # every regular file under photos/
|
||||
vault add https://example.com/a.pdf # download and store a URL
|
||||
vault migrate ./photos # like add, then delete the originals
|
||||
vault mirror https://example.com/ # crawl a site into example.com/, links rewritten
|
||||
vault list # everything
|
||||
tar -cz photos | vault absorb backup # every file in a tar (or .tar.gz) under backup/
|
||||
vault absorb backup < photos.tar.gz # same, from an existing archive
|
||||
vault ls # the top level: folders (ending in /) and files
|
||||
vault ls work # one level down, inside work/
|
||||
vault list # every file, all the way down, full names
|
||||
vault list work # everything under work/
|
||||
vault get work/2026/report.pdf # writes ./report.pdf
|
||||
vault get report.pdf -o copy.pdf
|
||||
vault get -r work # a whole folder, into ./work
|
||||
vault mv report.pdf old/report.pdf
|
||||
vault mv report.pdf old/ # the same: into a folder, like mv(1)
|
||||
vault mv work archive/work # folders move too
|
||||
vault cp -r work work-backup # instant; copies share their encrypted data
|
||||
vault rm old/report.pdf
|
||||
vault rm -r old # a whole folder; lists it and asks first
|
||||
vault rm -rf old # the same, without asking
|
||||
vault unlock # remember the key for 5 minutes (--for 30m)
|
||||
vault unlock --idle # ... or until 5 minutes go by unused
|
||||
vault lock # forget it now
|
||||
```
|
||||
|
||||
Commands that change the vault take an exclusive lock on `DIR/.lock`, and
|
||||
commands that read it take a shared one. A second process waits, and says
|
||||
so, until the first finishes. The operating system releases the lock if a
|
||||
process dies. Plan 9 and WebAssembly have no file locks, so there the lock
|
||||
file itself is the lock. After a crash on those platforms you remove it by
|
||||
hand; the error message names it.
|
||||
process dies; the locks are the same ones the Go version used, so old and
|
||||
new versions exclude each other.
|
||||
|
||||
Adding a name that already exists replaces it. `get` refuses to overwrite an
|
||||
existing file and writes nothing until the whole object has authenticated.
|
||||
Adding a name that already exists replaces it. `get` never replaces an
|
||||
existing file (or symlink) and writes nothing until the whole file has
|
||||
authenticated. `add URL` and `mirror` have size and time limits; see
|
||||
`vault(1)`.
|
||||
|
||||
`--key PATH` and `--vault DIR` work with every command. They default to
|
||||
`$VAULT_KEY` and `$VAULT_DIR`, then to `~/.vault/key` and `~/.vault/store`.
|
||||
The passphrase is read from the terminal, or from `$VAULT_PASSPHRASE` for
|
||||
scripts.
|
||||
|
||||
`vault unlock` asks once, then starts a helper process, like `ssh-agent`,
|
||||
that holds the unlocked key in memory until a hard deadline (5 minutes by
|
||||
default; with `--idle`, 5 minutes since a command last used it) or
|
||||
`vault lock`. Until then commands skip the prompt, unless
|
||||
`$VAULT_PASSPHRASE` is set, which always wins. The key reaches the helper
|
||||
over a pipe and never touches the disk; the helper's memory is locked
|
||||
against swapping, it cannot be core-dumped (or, on Linux, debugged), and it
|
||||
answers only on a socket in a directory private to your user. While it is
|
||||
unlocked, anything running as you can use the vault, decoy or not, so lock
|
||||
before handing the machine over. If the helper cannot protect its memory it
|
||||
refuses the key. `unlock` is not available on Windows.
|
||||
|
||||
## Decoy passphrase
|
||||
|
||||
`vault genkey` offers an optional second passphrase. In scripts, set
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
# Security notes
|
||||
|
||||
What vault protects, how, and where the protection stops. See
|
||||
`docs/COMPATIBILITY.md` for the exact formats.
|
||||
|
||||
## What is protected
|
||||
|
||||
- **Contents, names, sizes, modes and times** of stored files, against
|
||||
anyone who reads the vault directory without the passphrase: everything is
|
||||
inside Fury files sealed to a hybrid FrodoKEM-640 + ML-KEM-768 key, so the
|
||||
file key stays secret unless both KEMs are broken.
|
||||
- **Structure**: records are one fixed size; objects are at most 4 MiB and
|
||||
padded to power-of-two buckets; object names derive from a secret per-file
|
||||
seed, so nothing on disk links a file's objects; every ciphertext and shard
|
||||
directory has its timestamps pinned to the epoch; all 256 shards exist from
|
||||
the start; `init` adds filler sealed to a discarded key.
|
||||
- **Integrity**: every byte that is acted on is authenticated (header MAC,
|
||||
STREAM tags with a final-chunk flag, no trailing data); an object starts
|
||||
with its own ID, so objects cannot be swapped, reordered or replayed under
|
||||
another record; `get` publishes output only after the whole file, padding
|
||||
and end of file included, has authenticated.
|
||||
- **Deniability**: the key file always has two equal slots, one possibly
|
||||
random; both are always tried; decoy and real files are indistinguishable
|
||||
from filler to the other passphrase.
|
||||
|
||||
## What is not protected
|
||||
|
||||
- The number and total size of ciphertexts, and when the directory changes,
|
||||
are visible to whoever stores it.
|
||||
- A passphrase is only as strong as it is: Argon2id (3 passes, 64 MiB, 4
|
||||
lanes) slows guessing, it does not prevent it.
|
||||
- Anyone who can run code as you while the vault is unlocked (or watch you
|
||||
type the passphrase) can read everything.
|
||||
- Nothing here hides that you use vault.
|
||||
|
||||
## Secrets in memory
|
||||
|
||||
Passphrases, seeds, private keys, key-encryption keys and plaintext buffers
|
||||
are held in owned types (`SecretArray`, `SecretVec`, `LockedBuf`) that are
|
||||
not `Clone`, `Copy`, `Debug` or serialisable, have fixed allocations that
|
||||
never reallocate, and are cleared with `zeroize` when dropped — on success,
|
||||
on error and while unwinding from a panic (release builds unwind).
|
||||
|
||||
What cannot be cleared, so is not claimed:
|
||||
|
||||
- copies the compiler leaves in registers or on the stack;
|
||||
- copies inside third-party key objects: `chacha20poly1305` cipher states
|
||||
(the key schedule), `ml-kem`'s `Seed` passed by value (the decapsulation
|
||||
key itself is zeroized on drop with the crate's `zeroize` feature),
|
||||
`hmac`/`hkdf` internal states, `argon2` internals other than its working
|
||||
memory (which vault allocates and clears itself);
|
||||
- the passphrase in `$VAULT_PASSPHRASE` (it is in the process environment
|
||||
for the process lifetime) and in the terminal driver's buffers;
|
||||
- anything after `abort`, `kill -9`, a power cut, or once swapped out;
|
||||
- file data in the OS page cache, and plaintext temporary files on disk
|
||||
(below).
|
||||
|
||||
None of this is erasure from physical memory.
|
||||
|
||||
## The agent
|
||||
|
||||
`vault unlock` re-executes the binary as `vault __agent` with
|
||||
`VAULT_PASSPHRASE` and `VAULT_DECOY_PASSPHRASE` removed from its environment,
|
||||
no secret in its arguments, its working directory `/` and standard error
|
||||
closed. It hands over the 34-byte suite ID and seed through a pipe. Before
|
||||
reading it (straight from the pipe into the locked page, with no
|
||||
intermediate buffer), the agent:
|
||||
|
||||
1. sets the core-dump size limit to zero;
|
||||
2. on Linux, sets `PR_SET_DUMPABLE` to 0 (no core dumps, no same-user
|
||||
`ptrace`, `/proc/PID/mem` or `/proc/PID/environ`) and checks it took;
|
||||
3. maps a private page, `mlock`s it and, on Linux, marks it
|
||||
`MADV_DONTDUMP`, all checked.
|
||||
|
||||
Any failure is reported to `unlock` and the agent exits without the key;
|
||||
there is no fallback to unprotected memory. On other Unix systems (macOS,
|
||||
the BSDs, illumos) steps 1 and 3 apply; there is no portable equivalent of
|
||||
`PR_SET_DUMPABLE`, so another process of the same user may be able to read
|
||||
the agent's memory with a debugger.
|
||||
|
||||
The socket lives in a directory that must be owned by you with mode exactly
|
||||
0700 (refused otherwise); the socket is 0600, checked before readiness. Each
|
||||
request has a two-second deadline; the expiry deadline is enforced with a
|
||||
timed `poll`, so it does not depend on another client arriving, and a key
|
||||
is never handed out after expiry. On exit the agent clears the page and
|
||||
removes the socket only if it is still its own. `unlock` waits at most ten
|
||||
seconds for the agent to report and kills it otherwise.
|
||||
|
||||
Windows: the agent is not supported (`unlock` says so); every command asks
|
||||
for the passphrase.
|
||||
|
||||
## Filesystem safety
|
||||
|
||||
- Output is published with an atomic no-replace primitive
|
||||
(`renameat2(RENAME_NOREPLACE)`, `renameatx_np(RENAME_EXCL)`, `link`+`unlink`,
|
||||
or `MoveFileExW` without replace), so an existing file, symlink (dangling
|
||||
or not) or one created concurrently is never replaced; only vault's own
|
||||
temporary file is removed. There is no check-then-rename fallback.
|
||||
- The temporary output file is mode 0600 until the content has fully
|
||||
authenticated; only then are the recorded mode and times applied.
|
||||
- `get -r` opens each directory below the chosen destination with
|
||||
`O_NOFOLLOW|O_DIRECTORY` relative to its parent, so a symlink planted
|
||||
inside the destination cannot redirect files elsewhere. On Windows, which
|
||||
has no `openat`, each component is checked not to be a link or junction
|
||||
just before use; a concurrent swap between check and use is not excluded.
|
||||
- Writes follow crash-safe order with explicit `fsync` of files and
|
||||
directories (Unix): objects before their record; a new record before the
|
||||
removal of the one it replaces; a record's removal before its objects'.
|
||||
A failure after a record is already in place never deletes its objects.
|
||||
Windows has no directory fsync through the APIs used; NTFS journals
|
||||
metadata.
|
||||
- `migrate` checks an original is still the file it stored (device, inode,
|
||||
size, time; on Windows only size and time) and then deletes it by name; a
|
||||
replacement in the instant between the check and the deletion is not
|
||||
excluded. Paths inside the vault directory are refused outright.
|
||||
- A filesystem with neither `RENAME_NOREPLACE` nor hard links (some FUSE
|
||||
mounts, FAT) cannot publish without risking a replace, so `get` fails
|
||||
there with a message saying so rather than fall back.
|
||||
- The vault lock uses the same POSIX `fcntl` locks as the Go version, so
|
||||
mixed Go and Rust processes exclude each other. Only a read-only
|
||||
filesystem lets a reader proceed without the lock.
|
||||
|
||||
## Plaintext on disk
|
||||
|
||||
- `vault add URL` and `vault mirror` stream non-HTML downloads to a 0600
|
||||
temporary file in `$TMPDIR` before storing them, because the size must be
|
||||
known first. On Unix the file is unlinked as soon as it is created, so it
|
||||
has no name and disappears even if vault is killed; on other systems it
|
||||
is a named file removed on every normal path, including errors. HTML and
|
||||
CSS are rewritten in memory. `vault absorb` stores tar
|
||||
entries straight from the stream with no temporary file.
|
||||
- `vault get` writes plaintext where you ask; `vault migrate` deletes the
|
||||
originals with `unlink`, which is not secure deletion.
|
||||
|
||||
## Network
|
||||
|
||||
Downloads verify TLS certificates (rustls with the operating system's trust
|
||||
store; macOS binaries cross-built on Linux use the bundled Mozilla roots
|
||||
instead, see `docs/PLATFORMS.md`; verification is never disabled), follow at most 10 redirects (http and https only; for
|
||||
`mirror`, only to the same host), and are bounded by connect, response and
|
||||
whole-request deadlines and a byte budget counted after decompression. See
|
||||
`vault(1)` for the defaults and flags.
|
||||
|
||||
## Cryptographic implementations
|
||||
|
||||
FrodoKEM-640-SHAKE is a port of CIRCL v1.6.4's implementation (constant-time
|
||||
in the same places: no secret-dependent branches or indices; the
|
||||
re-encryption check selects the rejection key with a mask). It reproduces
|
||||
CIRCL's NIST KAT digest. ML-KEM-768 (`ml-kem`), ChaCha20-Poly1305 and
|
||||
XChaCha20-Poly1305 (`chacha20poly1305`), Argon2id (`argon2`), HKDF, HMAC,
|
||||
SHA-2 and SHA-3 are RustCrypto crates; TLS is rustls with `ring`. A Rust
|
||||
rewrite is not stronger cryptography: the algorithms and parameters are
|
||||
unchanged.
|
||||
|
||||
## Reporting
|
||||
|
||||
Please report vulnerabilities privately to andy@dixon.cx.
|
||||
@@ -0,0 +1,25 @@
|
||||
use chacha20poly1305::aead::{AeadInOut, KeyInit};
|
||||
use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce};
|
||||
use std::time::Instant;
|
||||
fn main() {
|
||||
let c = ChaCha20Poly1305::new(&Key::from([1u8; 32]));
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
let n = 1024; // 64 MiB
|
||||
let t = Instant::now();
|
||||
for i in 0..n {
|
||||
let mut nb = [0u8; 12];
|
||||
nb[3..11].copy_from_slice(&(i as u64).to_be_bytes());
|
||||
std::hint::black_box(c.encrypt_inout_detached(&Nonce::from(nb), &[], buf.as_mut_slice().into()).unwrap());
|
||||
}
|
||||
let s = t.elapsed().as_secs_f64();
|
||||
println!("raw seal: {:.0} MB/s", (n * 65536) as f64 / s / 1e6);
|
||||
let key = [1u8; 32];
|
||||
let data = vec![0u8; 64 << 20];
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::with_capacity(data.len() + (1 << 20));
|
||||
vault::crypto::stream::seal_all(&key, &data, &mut out);
|
||||
println!("seal_all: {:.0} MB/s", data.len() as f64 / t.elapsed().as_secs_f64() / 1e6);
|
||||
let t = Instant::now();
|
||||
let n = vault::crypto::stream::open_all(&key, &mut out).unwrap();
|
||||
println!("open_all: {:.0} MB/s ({n})", data.len() as f64 / t.elapsed().as_secs_f64() / 1e6);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The Rust side of scripts/bench.sh: the same operations as the Go
|
||||
//! oracle's Benchmark* functions, timed as the median of repeated runs.
|
||||
//! Run with `cargo bench --bench micro`.
|
||||
|
||||
use std::hint::black_box;
|
||||
use std::io::{Read, Write};
|
||||
use std::time::{Duration, Instant};
|
||||
use vault::crypto::{fury, kem, stream};
|
||||
|
||||
fn pattern(n: usize, salt: u8) -> Vec<u8> {
|
||||
(0..n).map(|i| (i.wrapping_mul(7) as u8) ^ salt).collect()
|
||||
}
|
||||
|
||||
/// Runs `f` until about two seconds have passed (at least 5 times) and
|
||||
/// prints the median and the spread.
|
||||
fn bench(name: &str, bytes: usize, mut f: impl FnMut()) {
|
||||
f(); // warm up
|
||||
let mut times = Vec::new();
|
||||
let start = Instant::now();
|
||||
while times.len() < 5 || (start.elapsed() < Duration::from_secs(2) && times.len() < 10_000) {
|
||||
let t = Instant::now();
|
||||
f();
|
||||
times.push(t.elapsed());
|
||||
}
|
||||
times.sort();
|
||||
let med = times[times.len() / 2];
|
||||
let (lo, hi) = (times[times.len() / 10], times[times.len() * 9 / 10]);
|
||||
let rate = if bytes > 0 {
|
||||
format!(" {:.0} MB/s", bytes as f64 / med.as_secs_f64() / 1e6)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(
|
||||
"{name:<24} median {:>10.3} ms (p10 {:.3}, p90 {:.3}, n={}){rate}",
|
||||
med.as_secs_f64() * 1e3,
|
||||
lo.as_secs_f64() * 1e3,
|
||||
hi.as_secs_f64() * 1e3,
|
||||
times.len()
|
||||
);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let seed = pattern(32, 1);
|
||||
bench("derive", 0, || {
|
||||
black_box(kem::Identity::from_suite_seed(kem::SUITE_ID, &seed).unwrap());
|
||||
});
|
||||
let id = kem::Identity::from_suite_seed(kem::SUITE_ID, &seed).unwrap();
|
||||
let r = id.recipient();
|
||||
let fk: [u8; 32] = pattern(32, 2).try_into().unwrap();
|
||||
bench("wrap", 0, || {
|
||||
black_box(r.wrap(&fk, &mut vault::rng::os()));
|
||||
});
|
||||
let st = r.wrap(&fk, &mut vault::rng::os());
|
||||
bench("unwrap", 0, || {
|
||||
black_box(id.unwrap(std::slice::from_ref(&st)).unwrap());
|
||||
});
|
||||
let pt = pattern(4 << 20, 3);
|
||||
bench("object 4MiB", pt.len(), || {
|
||||
let mut ct = Vec::new();
|
||||
fury::encrypt_to(&mut ct, &r, &pt, &mut vault::rng::os());
|
||||
black_box(fury::decrypt_bounded(&ct[..], &id, pt.len()).unwrap());
|
||||
});
|
||||
let key = [0u8; 32];
|
||||
let big = vec![0u8; 64 << 20];
|
||||
bench("stream 64MiB", big.len(), || {
|
||||
let mut w = stream::StreamWriter::new(&key, Vec::with_capacity(big.len() + (1 << 16)));
|
||||
w.write_all(&big).unwrap();
|
||||
let ct = w.finish().unwrap();
|
||||
let mut out = Vec::with_capacity(big.len());
|
||||
stream::StreamReader::new(&key, &ct[..]).read_to_end(&mut out).unwrap();
|
||||
black_box(out);
|
||||
});
|
||||
let body = vault::keyfile::new_body(b"bench", b"", &mut vault::rng::os());
|
||||
bench("unlock", 0, || {
|
||||
black_box(vault::keyfile::unlock_body(&body, b"bench").unwrap());
|
||||
});
|
||||
}
|
||||
@@ -1,48 +1,145 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build vault for every OS and CPU architecture Go can target without cgo.
|
||||
# Build vault for the supported operating systems and CPU architectures.
|
||||
#
|
||||
# ./build.sh # everything, into ./dist
|
||||
# ./build.sh # every target this host can build, into ./dist
|
||||
# TARGETS="linux/amd64 darwin/arm64" ./build.sh
|
||||
# OUT=/tmp/release ./build.sh
|
||||
# OUT=/tmp/release JOBS=4 ./build.sh
|
||||
#
|
||||
# Binaries are static, stripped and reproducible (CGO_ENABLED=0, -trimpath).
|
||||
# iOS and most Android targets are skipped (they need cgo); see below.
|
||||
# 32-bit ARM Linux is built twice: armv6 (Raspberry Pi Zero/1) and armv7.
|
||||
# SHA256SUMS is written next to the binaries.
|
||||
# Binaries keep the Go-era names, vault-OS-ARCH[.exe], and are stripped
|
||||
# release builds from the pinned toolchain (rust-toolchain.toml) and the
|
||||
# locked dependencies (Cargo.lock), with build paths remapped so the output
|
||||
# does not depend on where the tree lives. Linux binaries are static (musl)
|
||||
# except s390x, which links glibc dynamically; static linking is
|
||||
# checked. SHA256SUMS is written next to the binaries.
|
||||
#
|
||||
# How each target is built, and what the host needs:
|
||||
# zig cargo-zigbuild and zig on PATH (Linux, Windows GNU ABI)
|
||||
# mac a Mac with Xcode, or zig (macOS; zig builds trust the
|
||||
# bundled Mozilla roots, not the
|
||||
# keychain)
|
||||
# cross `cross` and Docker (FreeBSD)
|
||||
# A requested target the host cannot build is reported, the others are still
|
||||
# built, and the script exits non-zero. See docs/PLATFORMS.md for the Go
|
||||
# targets with no Rust equivalent; asking for one is an error.
|
||||
#
|
||||
# JOBS builds several targets at once, each in its own target directory
|
||||
# under target/build-sh (default 1: one at a time, cargo using every CPU).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
root=$PWD
|
||||
|
||||
table() {
|
||||
# os/arch Rust target method
|
||||
cat <<-'EOF'
|
||||
linux/amd64 x86_64-unknown-linux-musl zig
|
||||
linux/arm64 aarch64-unknown-linux-musl zig
|
||||
linux/386 i686-unknown-linux-musl zig
|
||||
linux/armv6 arm-unknown-linux-musleabihf zig
|
||||
linux/armv7 armv7-unknown-linux-musleabihf zig
|
||||
linux/riscv64 riscv64gc-unknown-linux-musl zig
|
||||
linux/loong64 loongarch64-unknown-linux-musl zig
|
||||
linux/ppc64le powerpc64le-unknown-linux-musl zig
|
||||
linux/s390x s390x-unknown-linux-gnu zig
|
||||
windows/amd64 x86_64-pc-windows-gnu zig
|
||||
windows/386 i686-pc-windows-gnu zig
|
||||
windows/arm64 aarch64-pc-windows-gnullvm zig
|
||||
darwin/amd64 x86_64-apple-darwin mac
|
||||
darwin/arm64 aarch64-apple-darwin mac
|
||||
freebsd/amd64 x86_64-unknown-freebsd cross
|
||||
freebsd/386 i686-unknown-freebsd cross
|
||||
EOF
|
||||
}
|
||||
|
||||
# Go-era targets no longer released (the release covers Linux, Windows,
|
||||
# macOS and FreeBSD). See docs/PLATFORMS.md for why each was dropped.
|
||||
unsupported="aix/ppc64 android/arm64 dragonfly/amd64 freebsd/arm freebsd/arm64 freebsd/riscv64
|
||||
illumos/amd64 js/wasm linux/mips linux/mipsle linux/mips64 linux/mips64le linux/ppc64
|
||||
netbsd/386 netbsd/amd64 netbsd/arm netbsd/arm64 openbsd/386 openbsd/amd64 openbsd/arm
|
||||
openbsd/arm64 openbsd/ppc64 openbsd/riscv64 plan9/386 plan9/amd64 plan9/arm solaris/amd64 wasip1/wasm"
|
||||
|
||||
out=${OUT:-dist}
|
||||
# iOS and 32-bit/x86 Android need cgo and a vendor SDK. android/arm64 builds
|
||||
# a static binary that runs in Termux.
|
||||
targets=${TARGETS:-$(go tool dist list | grep -vE '^ios/|^android/(386|amd64|arm)$')}
|
||||
jobs=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}
|
||||
jobs=${JOBS:-1}
|
||||
targets=${TARGETS:-$(table | awk '{print $1}')}
|
||||
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
# Never clean anything but our own output, and never in a directory that
|
||||
# matters: OUT must be a dedicated directory, not /, $HOME or the tree.
|
||||
mkdir -p -- "$out"
|
||||
out=$(cd -- "$out" && pwd -P)
|
||||
case $out in
|
||||
/ | "$(cd ~ && pwd -P)" | "$root") echo "refusing OUT=$out: use a dedicated directory" >&2; exit 2 ;;
|
||||
esac
|
||||
rm -f -- "$out"/vault-* "$out"/SHA256SUMS
|
||||
|
||||
export CARGO_TARGET_DIR_BASE=$root/target/build-sh
|
||||
export RUSTFLAGS="${RUSTFLAGS:-} --remap-path-prefix=$root=vault --remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=cargo"
|
||||
export OUT_DIR=$out JOBS_N=$jobs HOST_OS=$(uname -s) root
|
||||
logs=$root/target/build-sh-logs
|
||||
mkdir -p -- "$logs"
|
||||
export logs
|
||||
|
||||
build_one() {
|
||||
local os=${1%/*} arch=${1#*/} variants=("") name ext=""
|
||||
[[ $os == windows ]] && ext=.exe
|
||||
[[ $arch == wasm ]] && ext=.wasm
|
||||
[[ $os/$arch == linux/arm ]] && variants=(6 7)
|
||||
for v in "${variants[@]}"; do
|
||||
name=vault-$os-$arch${v:+v$v}$ext
|
||||
if ! GOOS=$os GOARCH=$arch GOARM=${v:-} CGO_ENABLED=0 \
|
||||
go build -trimpath -buildvcs=false -ldflags='-s -w -buildid=' -o "$OUT_DIR/$name" . 2>"$OUT_DIR/$name.log"; then
|
||||
echo "FAILED $name" >&2
|
||||
cat "$OUT_DIR/$name.log" >&2
|
||||
local want=$1 row os triple method name ext="" tdir bin
|
||||
row=$(table | awk -v t="$want" '$1 == t')
|
||||
if [[ -z $row ]]; then
|
||||
if grep -qw -- "$want" <<<"$unsupported"; then
|
||||
echo "UNSUPPORTED $want: not released; see docs/PLATFORMS.md" >&2
|
||||
else
|
||||
echo "UNKNOWN $want" >&2
|
||||
fi
|
||||
return 1
|
||||
fi
|
||||
rm "$OUT_DIR/$name.log"
|
||||
echo "built $name"
|
||||
done
|
||||
read -r _ triple method <<<"$row"
|
||||
os=${want%/*}
|
||||
[[ $os == windows ]] && ext=.exe
|
||||
name=vault-${want/\//-}$ext
|
||||
tdir=$root/target
|
||||
[[ $JOBS_N != 1 ]] && tdir=$CARGO_TARGET_DIR_BASE/$triple
|
||||
case $method in
|
||||
zig)
|
||||
command -v cargo-zigbuild >/dev/null && command -v zig >/dev/null ||
|
||||
{ echo "SKIPPED $want: needs cargo-zigbuild and zig" >&2; return 1; }
|
||||
cmd=(cargo zigbuild) ;;
|
||||
mac)
|
||||
if [[ $HOST_OS == Darwin ]]; then
|
||||
cmd=(cargo build)
|
||||
elif command -v cargo-zigbuild >/dev/null && command -v zig >/dev/null; then
|
||||
# Without Apple's SDK there is no Security framework to link, so
|
||||
# the cross-built binary trusts the bundled Mozilla roots rather
|
||||
# than the macOS keychain.
|
||||
cmd=(cargo zigbuild --no-default-features)
|
||||
else
|
||||
echo "SKIPPED $want: needs a Mac, or cargo-zigbuild and zig" >&2; return 1
|
||||
fi ;;
|
||||
cross)
|
||||
command -v cross >/dev/null && command -v docker >/dev/null ||
|
||||
{ echo "SKIPPED $want: needs cross and docker" >&2; return 1; }
|
||||
cmd=(cross build) ;;
|
||||
esac
|
||||
local flags=$RUSTFLAGS
|
||||
# zig's musl getrandom() never reaches the kernel on riscv64 (it spins);
|
||||
# make the getrandom crate issue the system call itself.
|
||||
[[ $triple == riscv64*-musl ]] && flags+=' --cfg getrandom_backend="linux_raw"'
|
||||
if ! RUSTFLAGS=$flags CARGO_TARGET_DIR=$tdir "${cmd[@]}" --release --locked --target "$triple" >"$logs/$name.log" 2>&1; then
|
||||
echo "FAILED $want ($triple)" >&2
|
||||
tail -20 "$logs/$name.log" >&2
|
||||
return 1
|
||||
fi
|
||||
bin=$tdir/$triple/release/vault$ext
|
||||
cp -- "$bin" "$OUT_DIR/$name"
|
||||
if [[ $triple == *-musl* ]] && command -v file >/dev/null; then
|
||||
file -L "$OUT_DIR/$name" | grep -q 'static' || { echo "FAILED $want: not statically linked" >&2; return 1; }
|
||||
fi
|
||||
echo "built $name ($triple)"
|
||||
}
|
||||
export -f build_one
|
||||
export OUT_DIR=$out
|
||||
export -f build_one table
|
||||
export unsupported
|
||||
|
||||
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}'
|
||||
status=0
|
||||
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}' || status=1
|
||||
|
||||
cd "$out"
|
||||
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
|
||||
echo "$(ls vault-* | wc -l | tr -d ' ') binaries in $out"
|
||||
cd -- "$out"
|
||||
if ls vault-* >/dev/null 2>&1; then
|
||||
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
|
||||
fi
|
||||
echo "$(ls vault-* 2>/dev/null | wc -l | tr -d ' ') binaries in $out"
|
||||
exit $status
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# cargo-deny policy: run `cargo deny --locked check`.
|
||||
|
||||
[graph]
|
||||
all-features = true
|
||||
|
||||
[advisories]
|
||||
# Every advisory is an error unless listed here with a reason.
|
||||
ignore = []
|
||||
|
||||
[licenses]
|
||||
# Licences compatible with distributing vault under GPL-3.0-or-later.
|
||||
allow = [
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-3-Clause",
|
||||
"CDLA-Permissive-2.0",
|
||||
"GPL-3.0-or-later",
|
||||
"ISC",
|
||||
"MIT",
|
||||
"Unicode-3.0",
|
||||
"Unlicense",
|
||||
"Zlib",
|
||||
]
|
||||
confidence-threshold = 0.9
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
wildcards = "deny"
|
||||
|
||||
[sources]
|
||||
unknown-registry = "deny"
|
||||
unknown-git = "deny"
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
@@ -0,0 +1,85 @@
|
||||
# Benchmarks: Go 0.5 against Rust 0.6
|
||||
|
||||
Measured on 2026-10-08 on one machine: Intel Core i7-7700 (4 cores, 8
|
||||
threads), 31 GiB RAM, ext4 on a two-disk RAID 1 (`/dev/md2`), Linux 7.0.
|
||||
Both binaries were built from this tree: the Go oracle (`oracle/go`, Go
|
||||
1.27.1, `CGO_ENABLED=0`) and the Rust release build (Rust 1.99.0, LTO).
|
||||
|
||||
**Limitation:** the machine was shared. Other users' Docker builds kept the
|
||||
load average between 4 and 12 throughout, so absolute times are noisy.
|
||||
Go and Rust were run alternately under the same conditions, and every
|
||||
figure is a median with its spread, but small differences mean nothing.
|
||||
Page caches were warm (dropping them needs root), so cold-cache numbers
|
||||
were not measured.
|
||||
|
||||
## Primitives
|
||||
|
||||
`cargo bench --bench micro` and
|
||||
`(cd oracle/go && go test -run XXX -bench Oracle -benchtime 2s ./ ./internal/pqcrypt ./internal/stream)`,
|
||||
run alternately twice (load 4.4 to 5.4). Medians in milliseconds; the Rust
|
||||
harness's p10 to p90 range is in brackets.
|
||||
|
||||
| Operation | Go | Rust |
|
||||
| --- | --- | --- |
|
||||
| Derive hybrid identity from seed | 5.6 | 3.0 (2.9 to 4.6) |
|
||||
| Wrap file key (both encapsulations) | 7.4 to 8.0 | 3.1 (3.0 to 4.8) |
|
||||
| Unwrap file key (both decapsulations) | 7.6 to 8.3 | 3.1 (3.0 to 3.6) |
|
||||
| Encrypt and decrypt one 4 MiB object | 19.9 to 20.1 | 19.0 to 19.5 |
|
||||
| STREAM seal and open, 64 MiB, streaming API | 89 to 92 | 187 to 188 |
|
||||
| Unlock key file (Argon2id for both slots) | 121 to 125 | 129 (125 to 147) |
|
||||
|
||||
- The FrodoKEM port is about twice as fast as CIRCL's Go code, mostly from
|
||||
multiplying A row by row instead of walking it by column.
|
||||
- ChaCha20-Poly1305 is slower: RustCrypto reaches about 1.2 GB/s on this CPU
|
||||
(`cargo bench --bench aead_probe`), Go's assembly about twice that. The
|
||||
64 MiB row uses the streaming reader and writer, which copy through
|
||||
64 KiB buffers. The vault itself uses the in-memory path (`seal_all`
|
||||
0.85 GB/s, `open_all` 1.4 GB/s), and per 4 MiB object the two are
|
||||
level because the KEM work is cheaper.
|
||||
- Unlock takes 180 ms with Argon2id lanes computed in turn. Computing the
|
||||
four lanes in parallel (argon2's `parallel` feature) brings it to 129 ms,
|
||||
close to Go. The parameters are unchanged.
|
||||
|
||||
## End to end
|
||||
|
||||
`RUNS=3 BIG_MB=1024 SMALL=500 scripts/bench.sh` (load 8 to 13). Every step
|
||||
includes unlocking the key. The 1 GiB file and the 500 small files (1 to
|
||||
16 KiB each) are each added to a fresh vault. The reads use one vault
|
||||
holding both, plus filler. Times are medians of 3 (min to max).
|
||||
|
||||
| Step | Go | Rust |
|
||||
| --- | --- | --- |
|
||||
| add 1 GiB file | 41.7 s (32.0 to 50.3) | 29.6 s (28.5 to 31.0) |
|
||||
| add 500 small files | 123.9 s (113.3 to 156.2) | 79.0 s (77.3 to 133.3) |
|
||||
| get 1 GiB file | 9.8 s (8.8 to 10.0) | 16.2 s (15.5 to 16.7) |
|
||||
| list (about 530 records) | 4.95 s (4.89 to 5.20) | 0.74 s (0.71 to 0.77) |
|
||||
| get one small file | 5.24 s (4.88 to 6.00) | 0.74 s (0.70 to 0.81) |
|
||||
| Peak RSS | 137 to 140 MiB | 132 MiB |
|
||||
| Peak threads | 18 | 17 |
|
||||
| Binary size | 11.5 MB | 3.5 MB (static musl: 3.4 MB) |
|
||||
|
||||
- **Catalogue reads** (`list`, and every command that looks a name up) are
|
||||
about 6 times faster. Records are decrypted in parallel on the worker pool
|
||||
(bounded at twice the worker count in flight), and each FrodoKEM
|
||||
decapsulation is cheaper.
|
||||
- **Adds** are faster. Both implementations fsync every object, its shard
|
||||
directory and every record. Rust overlaps up to eight objects' encryption
|
||||
and syncs on the pool.
|
||||
- **`get` of a large file is slower, on purpose.** Rust fsyncs the output
|
||||
file before giving it its name, so a crash just after `get` cannot leave
|
||||
the name pointing at an empty or partial file. Go did not. On an idle
|
||||
rerun (load about 8, output on the same disk), Go took 3.6 s and Rust
|
||||
15.5 s. Of Rust's time, 13.3 s was the single `fsync` of 1 GiB of
|
||||
freshly written data (measured with `strace -T`). The other 2 s or so is
|
||||
unlock, decryption and writing. This is the cost of the
|
||||
durability barrier on this RAID 1 array, and it scales with the disk's
|
||||
write speed.
|
||||
- Peak memory is dominated by Argon2id: 64 MiB for each of the two slots,
|
||||
computed at the same time in both implementations. Rust's object pipeline
|
||||
adds at most eight 4 MiB plaintext buffers and their ciphertexts. Peak
|
||||
threads: Rust has 8 workers, 8 Argon2 threads while unlocking, and the
|
||||
main thread.
|
||||
|
||||
Reproduce with `scripts/bench.sh` (it builds both binaries, creates
|
||||
synthetic data in `target/bench`, and prints the tables' numbers) and the
|
||||
two micro-benchmark commands above.
|
||||
@@ -0,0 +1,187 @@
|
||||
# Compatibility specification
|
||||
|
||||
This is the on-disk and on-wire contract between the Go implementation
|
||||
(frozen in `oracle/go`, commit `4c1507c`) and the Rust implementation. It is
|
||||
derived from the Go sources and pinned by vectors the Go oracle generated
|
||||
(`tests/vectors`, regenerated with `oracle/gen-vectors.sh`). The Rust port is
|
||||
an implementation migration, not a format migration: nothing here changes,
|
||||
no data is rekeyed or converted, and no new suite becomes the default.
|
||||
|
||||
Each section names the tests that hold the Rust code to it. "Go" below always
|
||||
means the oracle.
|
||||
|
||||
## 1. Key file
|
||||
|
||||
| Item | Value |
|
||||
| --- | --- |
|
||||
| Encoding | one PEM block, type `VAULT KEY`, standard base64 with padding, 64 columns, as Go's `pem.Encode` |
|
||||
| Body | exactly 198 bytes: an 18-byte header and two 90-byte slots |
|
||||
| Header | `VAULTKEY1`, Argon2id time `3` (u32 BE), memory `65536` KiB (u32 BE), lanes `4` (u8) |
|
||||
| Slot | 16-byte salt, 24-byte XChaCha20-Poly1305 nonce, 34 bytes sealed (u16 BE suite ID, 32-byte master seed), 16-byte tag |
|
||||
| Slot key | Argon2id v1.3 (passphrase bytes as given, no normalisation, salt, t=3, m=64 MiB, p=4, 32 bytes) |
|
||||
| Associated data | the 18-byte header |
|
||||
| Second slot | a decoy identity's slot, or 90 random bytes; which slot is real is random |
|
||||
| Unlock | Argon2id runs for both slots every time (Rust runs them concurrently); the first slot that opens wins; "incorrect passphrase" if neither |
|
||||
|
||||
Accepted framing: optional text before the block (Go's `pem.Decode`), only
|
||||
whitespace after it, header lines inside the block ignored. Files over 64 KiB
|
||||
are refused outright (Go would read them, then reject them).
|
||||
|
||||
Tests: `keyfile::tests::*` (Argon2id vectors, Go-written key files with and
|
||||
without decoy open to the Go-recorded seeds, byte-identical PEM re-encoding,
|
||||
malformed framing, NFC/NFD passphrases differ).
|
||||
|
||||
## 2. Hybrid suite `0x0001`, tag `frodo640shake+mlkem768`
|
||||
|
||||
- Master seed: 32 bytes. Expansion:
|
||||
`HKDF-SHA-256(ikm = seed, salt = none, info = "fury/v1 seed-expand frodo640shake+mlkem768")`,
|
||||
read as 48 bytes of FrodoKEM seed then 64 bytes of ML-KEM seed.
|
||||
- FrodoKEM-640-SHAKE exactly as CIRCL v1.6.4 `kem/frodo/frodo640shake`
|
||||
(the round-3 FrodoKEM, not the ISO/eFrodoKEM variants): key seed
|
||||
`s(16) || seedSE(16) || z(16)`; public key 9616 bytes (`seedA || pack(B)`);
|
||||
private key 19888 bytes (`s || pk || S as u16 LE || H(pk)`); ciphertext 9720
|
||||
bytes; 16-byte shared secret; 16-byte encapsulation seed `mu`; implicit
|
||||
rejection returns `SHAKE128(ct || s)`.
|
||||
- ML-KEM-768 (FIPS 203) with seed `d || z`; encapsulation key 1184 bytes,
|
||||
expanded decapsulation key 2400 bytes, ciphertext 1088 bytes, 32-byte
|
||||
shared secret, implicit rejection.
|
||||
- Public key: FrodoKEM public key `||` ML-KEM encapsulation key (10800 bytes).
|
||||
Private key (for comparison only): FrodoKEM private `||` ML-KEM expanded.
|
||||
- Wrap: fresh, independent randomness for both encapsulations every time.
|
||||
`KEK = HKDF-SHA-256(ikm = ss1 || ss2, salt = none, info = "fury/v1 kek frodo640shake+mlkem768" || ct1 || ct2)`;
|
||||
the 32-byte file key is sealed with ChaCha20-Poly1305 under the KEK and an
|
||||
all-zero 12-byte nonce. The zero nonce is safe only because each KEK is new
|
||||
and used once; the Rust code keeps that sealing function private to the
|
||||
suite.
|
||||
- Stanza: type `frodo640shake+mlkem768`, two arguments
|
||||
`base64(ct1)`, `base64(ct2)`, body = 48-byte sealed file key.
|
||||
- Unwrap: a stanza of another type is "not mine"; wrong argument count,
|
||||
bad base64 or wrong ciphertext lengths are fatal for the file; an AEAD
|
||||
failure (wrong key) is "not mine". The first stanza that opens wins.
|
||||
|
||||
Rust implementation: FrodoKEM is a line-by-line port of CIRCL
|
||||
(`src/crypto/frodo.rs`) because no maintained Rust crate implements CIRCL's
|
||||
variant (RustCrypto `frodo-kem` 0.1 implements the salted ISO version with
|
||||
9752-byte ciphertexts). ML-KEM is RustCrypto `ml-kem` 0.3.2.
|
||||
|
||||
Tests: `crypto::frodo::tests::nist_kat_digest` (reproduces the SHA-256
|
||||
CIRCL checks against the FrodoKEM reference KAT file),
|
||||
`crypto::kem::tests::{frodo_matches_circl, mlkem_matches_circl,
|
||||
hybrid_derivation_matches_go, wrap_matches_go_and_unwraps,
|
||||
fresh_wraps_differ_and_foreign_keys_do_not_match}`.
|
||||
|
||||
## 3. Fury header
|
||||
|
||||
```text
|
||||
fury-encryption.org/v1\n
|
||||
-> TYPE ARG ARG\n per stanza; TYPE and ARGs are 1+ bytes of 0x21-0x7e
|
||||
BODY, 64 base64 columns per line, ending with a line shorter than 64
|
||||
--- MAC\n MAC = base64 of 32 bytes
|
||||
```
|
||||
|
||||
- Base64: standard alphabet, no padding, non-zero trailing bits rejected,
|
||||
CR or LF anywhere in an encoded value rejected.
|
||||
- A body whose encoding fills its last line (the 48-byte wrapped key encodes
|
||||
to exactly 64 characters) is followed by an empty line; serialisation is
|
||||
canonical and Rust reproduces Go's bytes exactly.
|
||||
- Limits: 1 MiB per stanza (all its lines), 16 MiB per header; a line must
|
||||
end in LF; three bytes of look-ahead decide between a stanza and the
|
||||
footer; the payload starts immediately after the footer's LF, with any
|
||||
read-ahead returned first.
|
||||
- MAC: `HMAC-SHA-256(key = HKDF-SHA-256(file key, salt none, info "header"), message = header up to and including "---")`,
|
||||
compared in constant time.
|
||||
- Payload: 16-byte nonce, then STREAM keyed by
|
||||
`HKDF-SHA-256(file key, salt = nonce, info "payload")`.
|
||||
|
||||
Tests: `crypto::fury::tests::{header_vectors_match_go,
|
||||
header_parser_agrees_with_go, decrypts_go_files}` (the parser's accept or
|
||||
reject verdict, and the payload bytes left after the header, match Go on
|
||||
every case in `crypto.json`), `crypto::header::tests::*`,
|
||||
`tests/interop_crypto.rs` (Go decrypts files Rust wrote, and rejects
|
||||
truncated, extended and corrupted ones).
|
||||
|
||||
## 4. STREAM
|
||||
|
||||
- 64 KiB plaintext chunks; ChaCha20-Poly1305; 16-byte tag per chunk.
|
||||
- Nonce: 12 bytes, bytes 3..11 the big-endian chunk index, last byte `1`
|
||||
on the final chunk only, `0` otherwise.
|
||||
- Only the final chunk may be short; it may be empty only when it is the
|
||||
only chunk (empty plaintext). A plaintext that is an exact multiple of
|
||||
64 KiB ends with a full-length final chunk.
|
||||
- Rejected: no final chunk (truncation), an empty final chunk after
|
||||
others, bytes after the final chunk, any chunk failing authentication,
|
||||
a payload length no STREAM encoding can have.
|
||||
- The index never wraps; Rust refuses to go past 2^64 − 1 chunks (Go's
|
||||
counter is 88 bits; neither limit is reachable in practice).
|
||||
- Random access (`StreamReaderAt`) authenticates the final chunk when
|
||||
constructed, then any chunk on demand.
|
||||
|
||||
Tests: `crypto::stream::tests::*` (Go vectors for sizes around every
|
||||
boundary, any write/read step sizes, malformations rejected identically by
|
||||
the streaming, in-memory and random-access readers, concurrency of the
|
||||
random-access reader, counter overflow).
|
||||
|
||||
## 5. Store
|
||||
|
||||
- `DIR/.vault` contains exactly `VAULT 2\n`.
|
||||
- `DIR/records/xx/ID` and `DIR/objects/xx/ID`: `ID` is 16 bytes as
|
||||
lowercase hex, `xx` its first byte; all 256 shards of both exist after
|
||||
`init`. Entries whose names are not exactly that form, or are in the
|
||||
wrong shard, are ignored.
|
||||
- Every ciphertext, every shard and both top directories have access and
|
||||
modification times pinned to the Unix epoch; removals re-pin the shard.
|
||||
- Record: a Fury file whose plaintext is exactly 4096 bytes: a u32 BE JSON
|
||||
length (≤ 4092), the JSON, random padding. JSON is Go's `encoding/json`
|
||||
output for `{"name","seed","size","mode","modified","added"}`: HTML
|
||||
characters and U+2028/2029 escaped, times RFC 3339 with up to nine
|
||||
fractional digits, trailing zeros removed, `Z`. Rust writes it
|
||||
byte-identically.
|
||||
- Validation on read: name valid (below), seed 64 lowercase hex characters,
|
||||
size 0..=1 TiB, both times valid RFC 3339; unknown fields refused.
|
||||
- Names: non-empty UTF-8, ≤ 1024 bytes, already clean (Go `path.Clean`),
|
||||
relative, not `.`/`..`/`../…`, no backslash, no control characters.
|
||||
- Object IDs: `HMAC-SHA-256(key = seed, msg = index as big-endian i64)[:16]`.
|
||||
- Objects: plaintext = own ID (16 bytes), then up to `4 MiB − 16` file
|
||||
bytes, then random padding to the next power of two from 64 KiB to 4 MiB.
|
||||
An empty file still has one object. These sizes are plaintext; the
|
||||
ciphertext adds the header (~14.6 KB) and 16 bytes per 64 KiB.
|
||||
- Reading an object checks: it opens, its first 16 bytes are its ID, its
|
||||
length is exactly the padded size for its part, and the whole STREAM
|
||||
(final chunk, end of file) authenticates.
|
||||
- Records the key cannot open (no stanza matches) are skipped silently;
|
||||
ones that open but are malformed are reported and skipped.
|
||||
- Duplicates (an interrupted replace) are ordered newest `added` first; the
|
||||
next write of that name removes them all.
|
||||
- Filler: `init` writes 8–32 entries (or more, to reach `--filler`) of
|
||||
log-uniform sizes from 1 KiB to `clamp(target/8, 4 MiB, 1 GiB)`, sealed to
|
||||
a throwaway key, through the same code as `add`.
|
||||
|
||||
Tests: `store::types::tests::*` (names, object IDs, padded sizes, object
|
||||
counts, record JSON byte-identical to Go), `store::tests::reads_go_vault`
|
||||
(the oracle's fixture vault: real files, decoy file, foreign record),
|
||||
`store::tests::*`, `tests/cli.rs` lifecycle tests in both directions.
|
||||
|
||||
## 6. Locks
|
||||
|
||||
`DIR/.lock`, created mode 0600. Unix: POSIX `fcntl` record locks over the
|
||||
whole file (`F_SETLK` then `F_SETLKW`), shared for readers, exclusive for
|
||||
writers, released by the kernel when the process dies. Windows:
|
||||
`LockFileEx` on byte 0, length 1. Other systems: an `O_EXCL` lock file.
|
||||
Tests: `tests/cli.rs::lock_*` (Rust waits for Go, Go waits for Rust, both
|
||||
proceed after the holder is killed).
|
||||
|
||||
## 7. Agent
|
||||
|
||||
Socket `$XDG_RUNTIME_DIR` (else `$TMPDIR`, else `/tmp`) `/vault-UID/agent-`
|
||||
followed by the first 8 bytes of SHA-256 of the key file's absolute path
|
||||
(Go `filepath.Abs`, including its use of `$PWD`) in hex. The directory must
|
||||
be the user's own with mode exactly 0700. One request per connection: `g`
|
||||
→ 34 bytes (u16 BE suite ID, seed); `q` → the agent wipes the key and
|
||||
closes. Two-second deadline per request. Tests:
|
||||
`tests/cli.rs::agent_*` (Rust client with Go agent and the reverse).
|
||||
|
||||
## 8. What is deliberately not identical
|
||||
|
||||
See `docs/MIGRATION.md` for the full list of intentional behaviour changes.
|
||||
None of them alters a byte that is written to the vault or the key file
|
||||
beyond what the Go format already allows (fresh randomness, timestamps).
|
||||
@@ -0,0 +1,103 @@
|
||||
# Migrating from the Go implementation
|
||||
|
||||
vault 0.6 is a rewrite in Rust of the Go vault (0.5 and earlier). It is an
|
||||
implementation migration only: key files, vaults, the lock and the unlock
|
||||
agent are the same formats, byte for byte where the format is
|
||||
deterministic (see `COMPATIBILITY.md`). There is nothing to convert. Go and
|
||||
Rust builds can be used on the same vault, even at the same time: they
|
||||
honour each other's locks, and either can use an agent the other started.
|
||||
|
||||
To go back, install 0.5: everything 0.6 writes, 0.5 reads.
|
||||
|
||||
## Intentional behaviour changes
|
||||
|
||||
Safety and correctness:
|
||||
|
||||
- **`get` never replaces anything.** Output is published with an atomic
|
||||
no-replace rename or link, so a file, a symlink (even a dangling one) or a
|
||||
file created while `get` runs is left alone. Go checked first and then
|
||||
renamed, which a concurrent creator could race.
|
||||
- **`get -r` does not follow symlinks inside the output directory.** Each
|
||||
directory below the chosen one is opened without following links; a link
|
||||
or non-directory in the way stops the run. (Windows: checked just before
|
||||
use, see `SECURITY.md`.)
|
||||
- **Extracted files are synced** to disk before they appear under their
|
||||
name, and their directory afterwards.
|
||||
- **`get` needs an atomic no-replace primitive.** On a filesystem with
|
||||
neither `RENAME_NOREPLACE` nor hard links (some FUSE mounts, FAT) it fails
|
||||
with a message saying so, where Go would rename.
|
||||
- **`add`/`migrate` refuse paths inside the vault directory** (Go refused
|
||||
only the directory itself; `migrate store/records` would have deleted the
|
||||
vault's records).
|
||||
- **`absorb` refuses GNU sparse files** (Go expanded them), and honours PAX
|
||||
size records as Go does.
|
||||
- **A source file that changes while being added is not stored.** If it
|
||||
grows, shrinks, is modified or replaced during `add`/`migrate`, the
|
||||
command fails and nothing is stored. Go stored whatever it read (for a
|
||||
file that grew, a truncated copy).
|
||||
- **`migrate` deletes an original only if it is still the file stored**
|
||||
(same inode, size and time), after the record is durable.
|
||||
- **Deletes are durable in order.** Removing a file syncs the removal of its
|
||||
record before deleting its objects, so a crash cannot leave a record
|
||||
pointing at deleted objects. A failed sync after a record is already in
|
||||
place no longer deletes its objects.
|
||||
- **Lock errors are reported.** Go let a reader continue without the lock on
|
||||
any error; Rust does so only on a read-only filesystem.
|
||||
- **Stricter records.** A record must have every field, with exactly the
|
||||
field names Go writes, and nothing after the JSON. Go also accepted
|
||||
differently-cased names, missing fields (as zero) and trailing bytes;
|
||||
Go never writes such records, so no existing vault is affected.
|
||||
- **Key files over 64 KiB** are rejected without reading them whole.
|
||||
|
||||
Agent (`unlock`):
|
||||
|
||||
- The agent **refuses the key if it cannot protect it**: locking its memory,
|
||||
disabling core dumps and (Linux) becoming undumpable must all succeed. Go
|
||||
ignored failures there.
|
||||
- Its environment is stripped of `VAULT_PASSPHRASE` and
|
||||
`VAULT_DECOY_PASSPHRASE`; it runs in `/` with standard error closed.
|
||||
- Expiry no longer waits for the request in progress beyond its deadline,
|
||||
and a key is never handed out after expiry.
|
||||
- `unlock` waits at most ten seconds for the agent to start and kills it on
|
||||
failure; the agent removes its socket only if it is still its own.
|
||||
- **Windows: `unlock` is not available** (Go's agent there had no memory
|
||||
protection). Every command asks for the passphrase.
|
||||
|
||||
Network (`add URL`, `mirror`):
|
||||
|
||||
- Explicit limits, all adjustable: connect 30 s, response headers 60 s,
|
||||
whole request 2 h (`add`) or 5 min (`mirror`, per request), download size
|
||||
16 GiB (`add`) or 4 GiB in total (`mirror`), 10 redirects, 10 000 mirror
|
||||
pages, 16 MiB per HTML or CSS page. Bytes are counted after decompression.
|
||||
New flags `--max-bytes`, `--timeout`, `--max-pages`. Go had no limits.
|
||||
- Redirects go to http or https only; a `mirror` redirect to another host is
|
||||
skipped (Go followed it).
|
||||
- `mirror` resolves links against the final address after redirects and the
|
||||
page's `<base href>`; it parses HTML instead of matching a pattern, so it
|
||||
rewrites only `href`, `src`, `action` and `poster` on real tags, plus
|
||||
`url(...)` in CSS, `<style>` and `style=`, keeping every other byte and
|
||||
the original quoting. Text in scripts and comments is no longer touched.
|
||||
- `mirror` never lets two different URLs in one crawl overwrite each other's
|
||||
file; the second is skipped. It exits with status **3** when it stored
|
||||
some resources but skipped others (Go always exited 0).
|
||||
- Proxies follow `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` as in Go; local
|
||||
addresses are never proxied.
|
||||
|
||||
Command line:
|
||||
|
||||
- New option `--workers N` (1–256) on every command.
|
||||
- `--key`/`--vault` no longer need a home directory when given explicitly.
|
||||
- Command-line arguments must be valid UTF-8 (Go accepted any bytes, but
|
||||
vault names had to be UTF-8 already).
|
||||
- `ls`/`list` piped into a reader that stops early end quietly with
|
||||
status 141, as Go's did by `SIGPIPE`.
|
||||
|
||||
Platforms: releases now cover Linux, Windows, macOS and FreeBSD (16
|
||||
binaries, down from 43); `PLATFORMS.md` lists what was dropped and why.
|
||||
|
||||
## Unchanged on purpose
|
||||
|
||||
Formats, limits (1024-byte names, 1 TiB files, 4 MiB objects, 4096-byte
|
||||
records, 16 MiB headers, 1 MiB stanzas), Argon2id parameters, the hybrid
|
||||
suite, filler, decoy behaviour, messages and exit statuses (apart from the
|
||||
above), the shape of `ls` and `list` output, and the agent protocol.
|
||||
@@ -0,0 +1,73 @@
|
||||
# Platforms
|
||||
|
||||
The Go version was built for every target of `go tool dist list` that needs
|
||||
no cgo: 43 binaries. The Rust release is narrower by decision (Linux,
|
||||
Windows, macOS, FreeBSD): 16 binaries. This is the target-by-target record:
|
||||
how each is built, and how far it has actually been verified. "Runtime" means
|
||||
a binary was executed; "build" means it only compiled and linked.
|
||||
|
||||
Tooling used for verification on the development machine (x86-64 Linux):
|
||||
the pinned Rust 1.99.0, zig 0.16.0 with cargo-zigbuild 0.23.4, qemu-user
|
||||
10.2.1 (Ubuntu package, unpacked locally, no system changes), glibc cross
|
||||
sysroots from Ubuntu's `libc6-*-cross` packages, and the Go oracle built for
|
||||
x86-64. `scripts/smoke.sh` is the runtime check: create a key and vault, store
|
||||
files across object boundaries, read them back, then have the Go oracle read
|
||||
what that binary wrote and the binary read what the oracle wrote.
|
||||
|
||||
## Released
|
||||
|
||||
The release covers Linux, Windows, macOS and FreeBSD (a decision made during
|
||||
the port: the Go release's more exotic targets are no longer built).
|
||||
|
||||
| Go name | Rust target | Linking | Built by | Verified |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| linux/amd64 | x86_64-unknown-linux-musl | static | zig | runtime: full test suite (glibc host build) and smoke test of the static binary |
|
||||
| linux/arm64 | aarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-aarch64, cross-checked with the Go oracle |
|
||||
| linux/386 | i686-unknown-linux-musl | static | zig | runtime: smoke test under qemu-i386, cross-checked |
|
||||
| linux/armv6 | arm-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
|
||||
| linux/armv7 | armv7-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
|
||||
| linux/riscv64 | riscv64gc-unknown-linux-musl | static | zig | runtime: smoke test and crypto unit tests under qemu-riscv64, cross-checked (note 1) |
|
||||
| linux/loong64 | loongarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-loongarch64, cross-checked |
|
||||
| linux/ppc64le | powerpc64le-unknown-linux-musl | static | zig | runtime: smoke test under qemu-ppc64le, cross-checked |
|
||||
| linux/s390x | s390x-unknown-linux-gnu | glibc, dynamic | zig | runtime: smoke test under qemu-s390x (big-endian) with an Ubuntu glibc sysroot, cross-checked |
|
||||
| windows/amd64 | x86_64-pc-windows-gnu | static CRT | zig | build; tests cross-compile cleanly; runtime in CI only (windows-2025 job, not yet run) |
|
||||
| windows/386 | i686-pc-windows-gnu | static CRT | zig | build only |
|
||||
| windows/arm64 | aarch64-pc-windows-gnullvm | static CRT | zig | build only |
|
||||
| darwin/amd64 | x86_64-apple-darwin | system | a Mac, or zig (note 2) | build only from Linux; tested in CI on macos-15-intel |
|
||||
| darwin/arm64 | aarch64-apple-darwin | system | a Mac, or zig (note 2) | build only from Linux; tested in CI on macos-15 |
|
||||
| freebsd/amd64 | x86_64-unknown-freebsd | system | cross 0.2.5 (Docker) | build only (no emulator) |
|
||||
| freebsd/386 | i686-unknown-freebsd | system | cross 0.2.5 (Docker) | build only |
|
||||
|
||||
Feature notes: the `unlock` agent works on Linux, macOS and FreeBSD;
|
||||
Windows asks for the passphrase every time. Full agent hardening
|
||||
(`PR_SET_DUMPABLE`) is Linux-only; elsewhere the agent locks its memory and
|
||||
disables core dumps. See `SECURITY.md`.
|
||||
|
||||
1. zig's bundled musl `getrandom()` never reaches the kernel on riscv64 (the
|
||||
process spins). `build.sh` and `packaging/package.sh` build that target
|
||||
with `--cfg getrandom_backend="linux_raw"`, which makes the `getrandom`
|
||||
crate issue the system call itself; with that flag the binary passes.
|
||||
2. Without Apple's SDK there is no Security framework to link, so `build.sh`
|
||||
on Linux cross-builds macOS with `--no-default-features`: those binaries
|
||||
verify TLS for `add URL` and `mirror` against the bundled Mozilla roots
|
||||
instead of the macOS keychain (so a corporate CA added to the keychain is
|
||||
not trusted). Built on a Mac, or through Homebrew
|
||||
(`brew install andydixon/tap/vault`), they use the keychain.
|
||||
|
||||
## No longer released
|
||||
|
||||
| Go name | Why |
|
||||
| --- | --- |
|
||||
| android/arm64 | was a copy of the static Linux arm64 binary; that binary still runs in Termux, but `add URL`/`mirror` need `/etc/resolv.conf` |
|
||||
| linux/ppc64 (big-endian) | zig links it for the ELFv2 loader that BE distributions lack; built with `cross` instead, even a hello-world crashes at start-up under qemu-ppc64, so it cannot be verified |
|
||||
| linux/mips, mipsle, mips64, mips64le | Rust tier 3 (no stable std) since 1.72 |
|
||||
| netbsd/amd64 | builds only with a newer sysroot than cross 0.2.5 provides (std needs `libexecinfo`); netbsd/386, arm, arm64 are tier 3 |
|
||||
| openbsd/* | Rust tier 3; OpenBSD packages Rust itself, so a native build there would work |
|
||||
| freebsd/arm, arm64, riscv64 | Rust tier 3 |
|
||||
| dragonfly/amd64, aix/ppc64 | Rust tier 3 |
|
||||
| illumos/amd64, solaris/amd64 | outside the chosen matrix (illumos builds with cross; Solaris has no cross 0.2.5 image) |
|
||||
| wasip1/wasm | outside the chosen matrix (it builds; no agent, no prompts) |
|
||||
| js/wasm | no filesystem in the browser target |
|
||||
| plan9/* | no Rust target |
|
||||
|
||||
Asking `build.sh` for any of these reports it and fails.
|
||||
|
After Width: | Height: | Size: 169 KiB |
|
Before Width: | Height: | Size: 165 KiB After Width: | Height: | Size: 176 KiB |
|
After Width: | Height: | Size: 142 KiB |
|
Before Width: | Height: | Size: 75 KiB After Width: | Height: | Size: 75 KiB |
|
Before Width: | Height: | Size: 134 KiB After Width: | Height: | Size: 134 KiB |
|
Before Width: | Height: | Size: 75 KiB After Width: | Height: | Size: 74 KiB |
|
Before Width: | Height: | Size: 311 KiB After Width: | Height: | Size: 604 KiB |
|
Before Width: | Height: | Size: 89 KiB After Width: | Height: | Size: 89 KiB |
|
Before Width: | Height: | Size: 129 KiB After Width: | Height: | Size: 172 KiB |
|
After Width: | Height: | Size: 134 KiB |
|
Before Width: | Height: | Size: 134 KiB After Width: | Height: | Size: 149 KiB |
|
Before Width: | Height: | Size: 142 KiB After Width: | Height: | Size: 140 KiB |
|
After Width: | Height: | Size: 196 KiB |
|
Before Width: | Height: | Size: 88 KiB After Width: | Height: | Size: 88 KiB |
|
After Width: | Height: | Size: 195 KiB |
@@ -7,7 +7,9 @@ sandbox path and file owner are shown as /home/sam and "sam sam", key lines
|
||||
are highlighted in amber for the reader, and "moment" frames show a prompt
|
||||
with a cursor, exactly as vault prints it before you type.
|
||||
|
||||
Needs: go, python3, and Chrome or Chromium (set CHROME=/path if not found).
|
||||
Needs: cargo (with the pinned toolchain), cargo-zigbuild and zig for the
|
||||
build screenshot, python3, and Chrome or Chromium (set CHROME=/path if not
|
||||
found).
|
||||
Output: docs/guide/img/*.png
|
||||
"""
|
||||
|
||||
@@ -75,7 +77,8 @@ class Sandbox:
|
||||
self.bin = os.path.join(self.root, "bin")
|
||||
os.makedirs(self.home)
|
||||
os.makedirs(self.bin)
|
||||
subprocess.run(["go", "build", "-o", os.path.join(self.bin, "vault"), "."], cwd=REPO, check=True)
|
||||
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=REPO, check=True)
|
||||
shutil.copy2(os.path.join(REPO, "target", "release", "vault"), os.path.join(self.bin, "vault"))
|
||||
|
||||
def env(self, extra=None):
|
||||
e = {
|
||||
@@ -266,8 +269,9 @@ def main():
|
||||
shoot("add", t)
|
||||
|
||||
t = Term(box)
|
||||
t.cmd("vault ls", P)
|
||||
t.cmd("vault ls letters", P)
|
||||
t.cmd("vault list", P)
|
||||
t.cmd("vault list letters", P)
|
||||
shoot("list", t)
|
||||
|
||||
t = Term(box)
|
||||
@@ -315,6 +319,47 @@ def main():
|
||||
t.cmd("ls Scans")
|
||||
shoot("migrate", t)
|
||||
|
||||
# rm -r: lists the folder, asks, then frees the space
|
||||
t = Term(box)
|
||||
t.cmd("vault ls", P)
|
||||
t.lines.append(prompt() + '<span class="c">vault rm -r receipts</span>')
|
||||
for line in box.run_tty("vault rm -r receipts", P, "y").split("\n"):
|
||||
esc = html.escape(line)
|
||||
if "[y/N]" in line:
|
||||
esc = '<span class="hl">%s</span>' % esc
|
||||
t.lines.append(esc)
|
||||
t.cmd("vault ls", P)
|
||||
shoot("rm-r", t)
|
||||
|
||||
# absorb: a tar archive piped in, here pulled from a gzipped tarball
|
||||
box.write("Projects/website/index.html", "<h1>hello</h1>\n")
|
||||
box.write("Projects/website/css/site.css", "body{}\n")
|
||||
box.write("Projects/website/img/logo.png", os.urandom(24_311))
|
||||
t = Term(box)
|
||||
t.cmd("tar -czf website.tar.gz -C Projects website")
|
||||
t.cmd("vault absorb backups < website.tar.gz", P)
|
||||
t.cmd("tar -c recipes | vault absorb old", P)
|
||||
t.cmd("vault list backups", P)
|
||||
shoot("absorb", t)
|
||||
|
||||
# mirror: a tiny local site, served on localhost
|
||||
site = os.path.join(box.root, "site")
|
||||
for rel, body in {"index.html": '<a href="/about.html">About</a> <link href="/style.css">',
|
||||
"about.html": '<a href="/">Home</a> <img src="/img/me.png">',
|
||||
"style.css": "body{background:url(/img/bg.png)}",
|
||||
"img/me.png": "png", "img/bg.png": "png"}.items():
|
||||
os.makedirs(os.path.dirname(os.path.join(site, rel)), exist_ok=True)
|
||||
with open(os.path.join(site, rel), "w") as f:
|
||||
f.write(body)
|
||||
srv = subprocess.Popen([sys.executable, "-m", "http.server", "8000", "--bind", "127.0.0.1"],
|
||||
cwd=site, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
time.sleep(1)
|
||||
t = Term(box)
|
||||
t.cmd("vault mirror http://localhost:8000/ sites", P)
|
||||
t.cmd("vault get sites/localhost:8000/about.html && cat about.html", P)
|
||||
shoot("mirror", t)
|
||||
srv.terminate()
|
||||
|
||||
# errors
|
||||
t = Term(box)
|
||||
t.cmd("vault list", "hunter2", highlight=("incorrect",))
|
||||
@@ -345,9 +390,26 @@ def main():
|
||||
t.cmd("vault get nope 2>/dev/null; echo \"exit code: $?\"", P)
|
||||
shoot("scripting", t)
|
||||
|
||||
# cp, folder mv, get -r
|
||||
t = Term(box)
|
||||
t.cmd("vault cp -r letters archive/2025", P)
|
||||
t.cmd("vault mv ideas/notes.md archive/", P)
|
||||
t.cmd("vault ls archive", P)
|
||||
t.cmd("vault get -r archive -o restored/archive", P)
|
||||
shoot("cp-mv", t)
|
||||
|
||||
# unlock: one passphrase, then commands stop asking until lock
|
||||
t = Term(box)
|
||||
t.cmd("vault unlock", P)
|
||||
t.cmd("vault ls")
|
||||
t.cmd("vault get archive/notes.md -o /tmp/notes.md")
|
||||
t.cmd("vault lock")
|
||||
t.moment("vault ls", "Enter passphrase:")
|
||||
shoot("unlock", t)
|
||||
|
||||
# building, into a temporary OUT so an existing dist/ is left alone
|
||||
rel = tempfile.mkdtemp(prefix="vault-release-")
|
||||
targets = "linux/amd64 linux/arm darwin/arm64 windows/amd64"
|
||||
targets = "linux/amd64 linux/arm64 linux/armv7 windows/amd64"
|
||||
t = Term(box, title="sam@laptop: ~/src/vault")
|
||||
out = subprocess.run(["./build.sh"], cwd=REPO, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, env=dict(os.environ, TARGETS=targets, OUT=rel)).stdout
|
||||
|
||||
@@ -63,7 +63,7 @@ Got two minutes? Here's all of it:
|
||||
$ vault genkey # make your key (pick a good passphrase)
|
||||
$ vault init # make your vault
|
||||
$ vault add taxes.pdf # put something in
|
||||
$ vault list # see what's in there
|
||||
$ vault ls # see what's in there
|
||||
$ vault get taxes.pdf # get it back out
|
||||
\end{shell}
|
||||
|
||||
@@ -186,6 +186,26 @@ Download the one file for your system, check its SHA-256, rename it to
|
||||
Or build every platform yourself with \cmd{./build.sh}.
|
||||
\end{tldr}
|
||||
|
||||
\section{The easy way on Linux: my package repository}
|
||||
|
||||
On Debian, Ubuntu, Fedora, RHEL, openSUSE or Arch, add my signed repository
|
||||
once and install the \cmd{dixon-vault} package with your usual package
|
||||
manager. You get the manual pages too, and new releases arrive with your
|
||||
normal system updates. On Debian or Ubuntu:
|
||||
|
||||
\begin{shell}
|
||||
$ sudo install -d -m 0755 /etc/apt/keyrings
|
||||
$ curl -fsSL https://repo.dixon.cx/dixon.gpg | sudo tee /etc/apt/keyrings/dixon.gpg >/dev/null
|
||||
$ echo "deb [signed-by=/etc/apt/keyrings/dixon.gpg] https://repo.dixon.cx/apt stable main" |
|
||||
sudo tee /etc/apt/sources.list.d/dixon.list
|
||||
$ sudo apt update && sudo apt install dixon-vault
|
||||
\end{shell}
|
||||
|
||||
The commands for dnf, zypper and pacman are at \file{dixon.cx/repo}. The
|
||||
package is called \cmd{dixon-vault} because plain \cmd{vault} is HashiCorp
|
||||
Vault's name; the command it installs is still \cmd{vault}. Everyone else,
|
||||
read on.
|
||||
|
||||
\section{Pick your file}
|
||||
|
||||
vault is a single self-contained program. No installer, no runtime, no
|
||||
@@ -201,15 +221,17 @@ Linux PC or server & almost everything & vault-linux-amd64 \\
|
||||
Linux on ARM64 & Pi 4/5 (64-bit OS), Graviton, Ampere & vault-linux-arm64 \\
|
||||
Raspberry Pi, 32-bit & Pi 2, 3, 4 on a 32-bit OS & vault-linux-armv7 \\
|
||||
Raspberry Pi Zero / 1 & the tiny older ones & vault-linux-armv6 \\
|
||||
Android & in Termux & vault-android-arm64 \\
|
||||
FreeBSD, OpenBSD, NetBSD & amd64, arm64, 386, arm\ldots & vault-freebsd-amd64 etc. \\
|
||||
Something exotic & RISC-V, POWER, s390x, MIPS, LoongArch & see \cmd{dist/} \\
|
||||
Android & in Termux: the Linux ARM64 file & vault-linux-arm64 \\
|
||||
FreeBSD & amd64 or 386 & vault-freebsd-amd64 etc. \\
|
||||
Something exotic & RISC-V, POWER (little-endian), s390x, LoongArch & see \cmd{dist/} \\
|
||||
\end{vtable}
|
||||
|
||||
\begin{squirrel}
|
||||
The full release covers 43 platform builds, including AIX, Solaris, illumos,
|
||||
Plan~9 and even WebAssembly. Will you ever need \file{vault-plan9-arm}?
|
||||
Almost certainly not. Is it pleasing that it exists? Extremely.
|
||||
The release covers sixteen builds for Linux, Windows, macOS and FreeBSD,
|
||||
from Windows on ARM to IBM mainframes. The older Go builds reached further,
|
||||
to Plan~9 and WebAssembly; \file{docs/PLATFORMS.md} says why those went. Will you ever need
|
||||
\file{vault-linux-s390x}? Almost certainly not. Is it pleasing that it
|
||||
exists? Extremely.
|
||||
\end{squirrel}
|
||||
|
||||
\begin{headsup}
|
||||
@@ -267,8 +289,8 @@ so Gatekeeper quarantines it. Once you've checked the SHA-256, clear the flag:
|
||||
|
||||
\subsection{Android (Termux)}
|
||||
\begin{shell}
|
||||
$ curl -LO https://.../vault-android-arm64
|
||||
$ chmod +x vault-android-arm64 && mv vault-android-arm64 $PREFIX/bin/vault
|
||||
$ curl -LO https://.../vault-linux-arm64
|
||||
$ chmod +x vault-linux-arm64 && mv vault-linux-arm64 $PREFIX/bin/vault
|
||||
\end{shell}
|
||||
|
||||
\section{Did it work?}
|
||||
@@ -279,24 +301,28 @@ Run \cmd{vault help}. You should see the full command list:
|
||||
|
||||
\section{Building it yourself}
|
||||
|
||||
Prefer to build from source? You need Go 1.26.6 or newer. Then:
|
||||
Prefer to build from source? You need Rust, through \cmd{rustup}, which
|
||||
fetches the exact compiler version the project pins. Then:
|
||||
|
||||
\begin{shell}
|
||||
$ go build . # just your platform, produces ./vault
|
||||
$ ./build.sh # every platform, into ./dist, with SHA256SUMS
|
||||
$ cargo build --release --locked # just your platform: target/release/vault
|
||||
$ ./build.sh # every platform it can, into ./dist, with SHA256SUMS
|
||||
\end{shell}
|
||||
|
||||
\cmd{build.sh} cross-compiles every target Go supports without C libraries.
|
||||
\cmd{build.sh} cross-compiles Linux and Windows with \cmd{cargo-zigbuild}
|
||||
and \cmd{zig}, macOS on a Mac, and FreeBSD with \cmd{cross} and Docker; it says which targets your machine can't build.
|
||||
32-bit ARM Linux gets two builds, armv6 and armv7. Want just a few? Set
|
||||
\env{TARGETS}. Want them somewhere else? Set \env{OUT}.
|
||||
|
||||
\shot{build}{A partial build, followed by checking every binary against SHA256SUMS}
|
||||
|
||||
\begin{whyso}
|
||||
The binaries are \emph{static} and \emph{reproducible}: no C code, no system
|
||||
libraries, file paths stripped, build IDs zeroed. Two people building the same
|
||||
commit get byte-identical files, so you can check a release against your own
|
||||
build.
|
||||
The Linux binaries are \emph{static}: they need no system libraries at all.
|
||||
Every build uses the pinned compiler and the locked versions of every
|
||||
dependency, with build paths stripped, so the result doesn't depend on whose
|
||||
machine or which folder built it. Built with the same compiler for the same
|
||||
target, the same commit should give the same file, so you can check a
|
||||
release against your own build.
|
||||
\end{whyso}
|
||||
|
||||
\begin{pocket}
|
||||
@@ -304,7 +330,7 @@ build.
|
||||
\item One file per platform. Rename it \cmd{vault} and put it on your \env{PATH}.
|
||||
\item Always check the SHA-256 against \file{SHA256SUMS}.
|
||||
\item macOS: clear the quarantine flag after checking.
|
||||
\item \cmd{./build.sh} builds all 43 platform binaries from source.
|
||||
\item \cmd{./build.sh} builds every platform your machine can, from source.
|
||||
\end{pocketlist}
|
||||
\end{pocket}
|
||||
|
||||
@@ -511,12 +537,14 @@ type \cmd{--vault} again.
|
||||
|
||||
% ======================================================================
|
||||
\chapter{Everyday moves}
|
||||
\tagline{add, list, get, mv, rm: the five commands you'll actually use}
|
||||
\tagline{add, ls, get, mv, cp, rm: the commands you'll actually use}
|
||||
|
||||
\begin{tldr}
|
||||
\cmd{add} puts files in (optionally into a folder), \cmd{list} shows them,
|
||||
\cmd{get} takes one back out, \cmd{mv} renames, \cmd{rm} deletes. Each asks
|
||||
for your passphrase. That's the whole daily workflow.
|
||||
\cmd{add} puts files in (optionally into a folder), \cmd{ls} shows them a
|
||||
folder at a time, \cmd{get} takes them back out, \cmd{mv} and \cmd{cp} move
|
||||
and copy, \cmd{rm} deletes (folders with \cmd{-r} throughout). Each asks for
|
||||
your passphrase, unless you've run \cmd{unlock}. That's the whole daily
|
||||
workflow.
|
||||
\end{tldr}
|
||||
|
||||
\section{Putting things in: \texttt{add}}
|
||||
@@ -567,6 +595,30 @@ the whole site, one request at a time, into a folder named after its host.
|
||||
Links inside pages and stylesheets are rewritten to relative paths, so the
|
||||
copy still browses once you get it out. Only the same host is followed.
|
||||
|
||||
\shot{mirror}{Mirroring a small site. The links in each page now point at the copy.}
|
||||
|
||||
\subsection{From a tar archive: \texttt{absorb}}
|
||||
\cmd{absorb} reads a tar archive, gzipped or not, from standard input and
|
||||
stores every plain file in it, optionally under a folder:
|
||||
|
||||
\begin{verbatim}
|
||||
tar -cz photos | vault absorb backup
|
||||
ssh server 'tar -c /etc' | vault absorb server1
|
||||
vault absorb old < archive.tar.gz
|
||||
\end{verbatim}
|
||||
|
||||
Each file is encrypted straight from the stream, so nothing is unpacked to
|
||||
disk first. Directories need no entry of their own; links, devices and any
|
||||
name that tries to climb out with \file{..} are skipped with a note.
|
||||
vault asks for your passphrase only once the archive starts arriving, so
|
||||
when it comes out of something that asks first, like
|
||||
\cmd{gpg --decrypt secrets.tar.gpg | vault absorb}, gpg's prompt comes
|
||||
and goes before vault's appears. If the
|
||||
archive stops halfway through a file, that file isn't stored and vault
|
||||
stops; everything before it is already safely in.
|
||||
|
||||
\shot{absorb}{A gzipped tarball and a plain tar stream, both absorbed.}
|
||||
|
||||
\subsection{Moving in: \texttt{migrate}}
|
||||
\cmd{migrate} works exactly like \cmd{add}, but deletes each original once
|
||||
it's safely stored, then tidies away any folders left empty. It won't take a
|
||||
@@ -587,17 +639,25 @@ themselves, should they live inside the folder, so \cmd{migrate} can never
|
||||
eat them. Files larger than 1\,TiB are
|
||||
refused. (If you have a single 1\,TiB file, we have questions.)
|
||||
|
||||
\section{Seeing what's inside: \texttt{list}}
|
||||
\section{Seeing what's inside: \texttt{ls} and \texttt{list}}
|
||||
|
||||
\begin{shell}
|
||||
$ vault list # everything
|
||||
$ vault list letters # just what's inside letters/
|
||||
$ vault ls # the top level: folders (ending in /) and files
|
||||
$ vault ls letters # one level down, inside letters/
|
||||
$ vault list # every file, all the way down, full names
|
||||
\end{shell}
|
||||
|
||||
Each line shows the size, the file's \emph{own} last-modified time from when
|
||||
you added it, and its name, sorted alphabetically.
|
||||
\cmd{ls} works like the shell's: it shows one folder at a time. Each
|
||||
sub-folder appears once, ending in \file{/}, with the total size of
|
||||
everything inside it and its newest date. Files show their size and their
|
||||
\emph{own} last-modified time from when you added them. Folders come first,
|
||||
then files. Keep going down with \cmd{vault ls letters/2025}.
|
||||
|
||||
\shot{list}{The full catalogue, then just one folder}
|
||||
\cmd{list} is the whole catalogue in one go: every file under a folder,
|
||||
with its full name, sorted. It's the one to pipe into other commands, and
|
||||
\cmd{vault ls -r} does the same.
|
||||
|
||||
\shot{list}{One level at a time with \cmd{ls}, then everything with \cmd{list}}
|
||||
|
||||
\begin{whyso}
|
||||
\textbf{Why does \texttt{list} need my passphrase?} Because the names are
|
||||
@@ -610,6 +670,7 @@ than a thief can.
|
||||
\begin{shell}
|
||||
$ vault get NAME # writes the file here, under its own name
|
||||
$ vault get NAME -o OUTPUT # writes it wherever you say
|
||||
$ vault get -r FOLDER # a whole folder, into ./FOLDER
|
||||
\end{shell}
|
||||
|
||||
\shot{get}{Getting files back, including vault refusing to overwrite}
|
||||
@@ -627,39 +688,96 @@ Three safety habits are built in, so you don't need to think about them:
|
||||
and modification time.
|
||||
\end{checks}
|
||||
|
||||
\section{Renaming and moving: \texttt{mv}}
|
||||
\section{Renaming, moving and copying: \texttt{mv} and \texttt{cp}}
|
||||
|
||||
\begin{shell}
|
||||
$ vault mv OLD NEW
|
||||
$ vault mv OLD NEW # rename a file or a whole folder
|
||||
$ vault mv notes.md ideas/ # move into a folder, keeping the name
|
||||
$ vault cp OLD NEW # copy a file
|
||||
$ vault cp -r OLD NEW # copy a folder
|
||||
\end{shell}
|
||||
|
||||
\cmd{NEW} is the \emph{full} new name, folder included. So to move
|
||||
\file{notes.md} into \file{ideas}, it's \cmd{vault mv notes.md
|
||||
ideas/notes.md}, not \cmd{\ldots\ ideas/}. It won't overwrite an existing name,
|
||||
and it doesn't re-encrypt the file, so it's instant however big the file is.
|
||||
They work like the everyday \cmd{mv} and \cmd{cp}: if \cmd{NEW} ends in
|
||||
\cmd{/} or is a folder already, \cmd{OLD} goes inside it. Neither will
|
||||
overwrite an existing name, and neither re-encrypts anything, so both are
|
||||
instant however big the files are. A copy shares the original's encrypted
|
||||
data, so it takes no extra space, and deleting one leaves the other intact.
|
||||
\cmd{get -r} brings a whole folder back out, keeping its shape.
|
||||
|
||||
\shot{cp-mv}{A folder copied, a file moved into it, and the lot brought back out}
|
||||
|
||||
\section{Deleting: \texttt{rm}}
|
||||
|
||||
\begin{shell}
|
||||
$ vault rm NAME
|
||||
$ vault rm NAME # one file
|
||||
$ vault rm -r FOLDER # a folder and everything in it; asks first
|
||||
$ vault rm -rf FOLDER # the same, without asking
|
||||
\end{shell}
|
||||
|
||||
Removes the file's catalogue entry and its encrypted contents. One file per
|
||||
command, on purpose. There's no recursive delete to fat-finger.
|
||||
Removes the file's catalogue entry and every encrypted piece of it, so the
|
||||
space comes back straight away.
|
||||
|
||||
\shot{mv-rm}{Replace, move, delete, and the result}
|
||||
|
||||
\begin{danger}
|
||||
\textbf{\texttt{rm} is immediate and there's no undo.} No recycle bin, no
|
||||
``are you sure?''. If it matters, make sure it's backed up somewhere first.
|
||||
\textbf{\texttt{rm} is immediate and there's no undo.} No recycle bin. If it
|
||||
matters, make sure it's backed up somewhere first.
|
||||
\end{danger}
|
||||
|
||||
\begin{squirrel}
|
||||
Need to delete a whole folder? Loop over \cmd{list}. The name starts at
|
||||
character 31 of each line, so:\\
|
||||
\cmd{vault list old | cut -c31- | while read -r f; do vault rm "\$f"; done}\\
|
||||
With \env{VAULT\_PASSPHRASE} set, or it'll ask every time. See chapter 7.
|
||||
\end{squirrel}
|
||||
\subsection{Deleting a whole folder}
|
||||
\cmd{rm -r} deletes a folder and everything under it. Because that can be a
|
||||
lot, it first lists what's going (the first 20 names, then a count) with
|
||||
the total size, and asks. Anything but \texttt{y} deletes nothing, and so
|
||||
does having no terminal to ask on. \cmd{-f} skips the question, and also
|
||||
means ``don't complain if there's nothing there'', so \cmd{rm -rf} is the
|
||||
one for scripts. \cmd{vault rm -r /} is refused rather than emptying the
|
||||
vault.
|
||||
|
||||
\shot{rm-r}{A folder deleted: vault lists it, asks, then frees the space}
|
||||
|
||||
\section{Not typing your passphrase every time: \texttt{unlock}}
|
||||
|
||||
\begin{shell}
|
||||
$ vault unlock # ask once, remember the key for 5 minutes
|
||||
$ vault unlock --for 30m # or for as long as you say
|
||||
$ vault unlock --idle # 5 minutes since you last used it
|
||||
$ vault lock # forget it right now
|
||||
\end{shell}
|
||||
|
||||
After \cmd{unlock}, every command skips the passphrase prompt until the time
|
||||
is up. Normally that's a hard deadline, counted from when you unlocked, so a
|
||||
laptop left open still locks itself; run \cmd{unlock} again for a fresh
|
||||
countdown. If you'd rather it stayed unlocked while you're busy, \cmd{--idle}
|
||||
counts from the last command that used the key instead, so it locks only
|
||||
after that long without use.
|
||||
|
||||
\shot{unlock}{Unlock once, work without prompts, lock when you're done}
|
||||
|
||||
\subsection{How it keeps the key safe}
|
||||
\cmd{unlock} starts a small helper process, much like \cmd{ssh-agent}, that
|
||||
holds your unlocked key in memory and nowhere else:
|
||||
|
||||
\begin{checks}
|
||||
\item \textbf{Never on disk.} The key goes from \cmd{unlock} to the helper
|
||||
through a pipe. Its memory is locked so it can't be swapped out, and
|
||||
on Linux the helper can't be debugged or core-dumped, even by you.
|
||||
\item \textbf{Only you can reach it.} It answers on a socket inside a folder
|
||||
only your user account can open (in \file{\$XDG\_RUNTIME\_DIR}, or
|
||||
your temporary folder). vault checks that folder's owner and
|
||||
permissions every time.
|
||||
\item \textbf{It wipes itself.} At the deadline, or on \cmd{vault lock}, the
|
||||
key is overwritten and the helper exits. Rebooting ends it too.
|
||||
\item \textbf{An explicit passphrase wins.} If \env{VAULT\_PASSPHRASE} is
|
||||
set, vault uses that and ignores the helper, so scripts behave
|
||||
exactly as before.
|
||||
\end{checks}
|
||||
|
||||
\begin{headsup}
|
||||
\textbf{While it's unlocked, anyone at your keyboard is you.} Anything
|
||||
running as your user can use the key without a passphrase. Lock before you
|
||||
walk away, and before handing your machine to anyone, decoy or not (see
|
||||
\emph{The honest limits} in the next chapter).
|
||||
\end{headsup}
|
||||
|
||||
\section{Flags work anywhere}
|
||||
|
||||
@@ -676,7 +794,9 @@ $ vault add report.pdf --vault=~/work
|
||||
\begin{pocketlist}
|
||||
\item \cmd{add FILE [FOLDER]}: keeps the file's name, optionally in a folder. Folders add recursively.
|
||||
\item Adding an existing name replaces it, safely.
|
||||
\item \cmd{list [FOLDER]}, \cmd{get NAME [-o OUT]}, \cmd{mv OLD NEW}, \cmd{rm NAME}.
|
||||
\item \cmd{ls [FOLDER]} one level at a time, \cmd{list [FOLDER]} everything.
|
||||
\item \cmd{get [-r] NAME [-o OUT]}, \cmd{mv OLD NEW}, \cmd{cp [-r] OLD NEW}, \cmd{rm NAME}, \cmd{rm -r FOLDER}.
|
||||
\item \cmd{unlock} remembers the key for 5 minutes; \cmd{lock} forgets it.
|
||||
\item \cmd{get} never overwrites and never publishes a half-checked file.
|
||||
\item \cmd{rm} has no undo.
|
||||
\end{pocketlist}
|
||||
@@ -817,6 +937,9 @@ something was.
|
||||
\begin{nopes}
|
||||
\item It won't help if someone saw you type the real passphrase, or has
|
||||
malware on your machine.
|
||||
\item It won't help if the real key is still unlocked. A command run
|
||||
while \cmd{vault unlock} holds it uses the real key, whatever
|
||||
passphrase you're asked to give. Run \cmd{vault lock} first.
|
||||
\item It doesn't stop anyone deleting the raw files. It hides; it doesn't guard.
|
||||
\item It isn't legal advice. Some places treat misleading an official as an
|
||||
offence in its own right. Know where you are.
|
||||
@@ -846,7 +969,7 @@ it worked.
|
||||
|
||||
\begin{vtable}{>{\leavevmode\ttfamily\small\color{vvioletdeep}}p{50mm}Y}
|
||||
\thead{VARIABLE} & \thead{WHAT IT DOES} \\ \midrule
|
||||
VAULT\_PASSPHRASE & The passphrase. Skips the prompt. With \cmd{genkey}, it's the new key's passphrase. \\
|
||||
VAULT\_PASSPHRASE & The passphrase. Skips the prompt, and any key held by \cmd{vault unlock}. With \cmd{genkey}, it's the new key's passphrase. \\
|
||||
VAULT\_DECOY\_PASSPHRASE & \cmd{genkey} only: the decoy passphrase. If \env{VAULT\_PASSPHRASE} is set and this isn't, no decoy is made. \\
|
||||
VAULT\_KEY & The key file. Default \file{\tld/.vault/key}. Same as \cmd{--key}. \\
|
||||
VAULT\_DIR & The vault folder. Default \file{\tld/.vault/store}. Same as \cmd{--vault}. \\
|
||||
@@ -940,7 +1063,8 @@ covered.
|
||||
|
||||
\section{Locking is automatic}
|
||||
|
||||
Commands that change the vault (\cmd{add}, \cmd{mv}, \cmd{rm}) take an
|
||||
Commands that change the vault (\cmd{add}, \cmd{migrate}, \cmd{mirror},
|
||||
\cmd{absorb}, \cmd{mv}, \cmd{rm}) take an
|
||||
\hl{exclusive} lock. Commands that only read (\cmd{list}, \cmd{get}) take a
|
||||
\hl{shared} one, so several can read at once. If a command has to wait, it
|
||||
says so, then carries on by itself:
|
||||
@@ -953,8 +1077,8 @@ who's slow at typing never blocks you.
|
||||
\begin{whyso}
|
||||
\textbf{What if vault crashes while holding the lock?} On Linux, macOS, the
|
||||
BSDs and Windows, the operating system drops the lock the moment the process
|
||||
dies. Nothing to clean up. On Plan~9 and WebAssembly, which have no file
|
||||
locks, vault uses a \file{.lock} file instead. If one's ever left behind,
|
||||
dies. Nothing to clean up. Older Go builds of vault use the very same locks,
|
||||
so mixing versions is safe. If one's ever left behind,
|
||||
vault tells you after a minute exactly which file to delete.
|
||||
\end{whyso}
|
||||
|
||||
@@ -1184,18 +1308,40 @@ open \ldots/key: no such file or directory & No key there yet. Run \cmd{vault ge
|
||||
\ldots\ is not a vault; run vault init & Wrong \cmd{--vault} or \env{VAULT\_DIR}, or you haven't run \cmd{init} yet. \\
|
||||
\ldots\ is already a vault & You ran \cmd{init} on an existing vault. Nothing was changed. \\
|
||||
NAME: not in the vault & Check the exact name with \cmd{vault list}: it's case-sensitive and includes the folder. Or you typed the other passphrase. \\
|
||||
\ldots\ already exists & \cmd{get}: the output file exists, so use \cmd{-o}. \cmd{mv}: the new name is taken. \cmd{genkey}: you already have a key. \\
|
||||
\ldots\ already exists & \cmd{get}: the output file exists, so use \cmd{-o}. \cmd{mv}, \cmd{cp}: the new name is taken. \cmd{genkey}: you already have a key. \\
|
||||
invalid vault path "\ldots" & Names can't start with \cmd{/} or \cmd{../}, end in \cmd{/}, or contain backslashes or control characters. \\
|
||||
\ldots: not a regular file of at most 1\,TiB & You tried to add a device, a socket or a gigantic file. \\
|
||||
\ldots\ changed while being added & The file was being written while vault read it. Try again once it's finished. \\
|
||||
\ldots\ is a folder; use -r & \cmd{cp} copies a folder only with \cmd{-r}. \\
|
||||
cannot put \ldots\ inside itself & You tried to move or copy a folder into itself. \\
|
||||
refusing to remove the whole vault & \cmd{rm -r /} (or an empty folder name). Name the folder you mean. \\
|
||||
rm cancelled; nothing changed & You didn't answer \texttt{y} to \cmd{rm -r}, or there was no terminal to ask on. \cmd{-f} skips the question. \\
|
||||
skipping \ldots: not a regular file & Adding a folder: a symlink or special file was skipped. Everything else went in. \\
|
||||
\end{vtable}
|
||||
|
||||
\subsection{Passphrases, the unlock helper, and damage}
|
||||
|
||||
\begin{vtable}{>{\ttfamily\footnotesize}p{58mm}Y}
|
||||
\thead{MESSAGE} & \thead{WHAT IT MEANS, AND THE FIX} \\ \midrule
|
||||
passphrases do not match & The two entries differed. Nothing was created; run it again. \\
|
||||
passphrase must not be empty & Blank passphrases aren't allowed, for good reason. \\
|
||||
decoy passphrase must differ from the passphrase & The decoy can't be the same as the real one. \\
|
||||
VAULT\_PASSPHRASE is set but empty & Unset it, or give it a value. \\
|
||||
could not read passphrase (set VAULT\_PASSPHRASE\ldots) & No terminal to ask on, typically cron or CI. Set the variable. \\
|
||||
\ldots\ is not a private directory of this user & The folder for \cmd{unlock}'s socket belongs to someone else or is open to others. Someone may be up to something; check who made it. Commands still work, asking for the passphrase. \\
|
||||
agent did not start: \ldots & \cmd{unlock} couldn't start its helper. The reason follows; commands still work, asking for the passphrase. \\
|
||||
nothing was unlocked & \cmd{vault lock} found nothing to lock. Not an error. \\
|
||||
waiting for another vault process to finish & Not an error. Another command is writing; this one continues when it's done. \\
|
||||
vault still locked after a minute\ldots & Plan~9 and WebAssembly only. If nothing else is running, delete the file it names. \\
|
||||
cannot lock the vault: \ldots & The lock file couldn't be opened or locked; the reason follows. Check the vault folder's permissions. \\
|
||||
\ldots\ changed while being added & The file was being written while vault read it. Nothing was stored; try again when it's finished. \\
|
||||
\ldots\ changed after it was stored; not deleting it & \cmd{migrate} stored the file, but it changed before it could be deleted. The original is kept; the vault has the version it read. \\
|
||||
\ldots\ already exists & \cmd{get} never overwrites, not even a symlink. Pick another name with \cmd{-o}. \\
|
||||
\ldots\ refusing to extract through it & \cmd{get -r} found a symlink (or a file) where a folder should be inside the output folder, and stopped rather than follow it. \\
|
||||
cannot lock agent memory \ldots, cannot make the agent undumpable \ldots & \cmd{unlock}'s helper couldn't protect the key in memory, so it refused it. Commands still work, asking for the passphrase. \\
|
||||
vault unlock is not supported on this platform\ldots & Windows: every command asks for the passphrase. \\
|
||||
download exceeds \ldots\ bytes & A download passed its size limit. Raise it with \cmd{--max-bytes} if you meant it. \\
|
||||
redirected to another host: \ldots & \cmd{mirror} only follows redirects within the site it is copying. \\
|
||||
mirror incomplete: stored \ldots, skipped \ldots & Some pages failed; the reasons were printed as they happened. Exit status 3. \\
|
||||
skipping unreadable record \ldots & A catalogue entry is damaged. Everything else still works; restore \file{records/} from backup to recover that file. \\
|
||||
object does not match its catalogue record & The encrypted contents were swapped or damaged. Nothing was written. Restore \file{objects/} from backup. \\
|
||||
object size does not match its catalogue record & Same as above: damage or tampering, caught before any output appeared. \\
|
||||
@@ -1244,12 +1390,20 @@ passphrase. Nothing in this chapter helps with that. Back up your key.
|
||||
enhanced,colback=vmist,colframe=vline,boxrule=0.4pt,arc=4pt,left=9pt,right=9pt,top=7pt,bottom=7pt]
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault genkey}\\[2pt]\small Make a key. Asks for a passphrase and an optional decoy.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault init}\\[2pt]\small Make an empty vault, with filler. \texttt{--filler 2G} for more.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault add FILE|DIR [FOLDER]}\\[2pt]\small Encrypt in, keeping the name. Replaces an existing name.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault list [FOLDER]}\\[2pt]\small Size, modified time and name, sorted.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault get NAME [-o OUT]}\\[2pt]\small Decrypt out. Never overwrites. Verifies first.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault mv OLD NEW}\\[2pt]\small Rename, using the full new name. Instant.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault rm NAME}\\[2pt]\small Delete one file. No undo.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault add FILE|DIR|URL [FOLDER]}\\[2pt]\small Encrypt in, keeping the name. Replaces an existing name.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault migrate FILE|DIR [FOLDER]}\\[2pt]\small Like add, then delete the originals. Asks before replacing.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault mirror URL [FOLDER]}\\[2pt]\small Crawl a website in, links rewritten to work offline.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault absorb [FOLDER] < TAR}\\[2pt]\small Every file in a tar or .tar.gz from standard input.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault ls [FOLDER]}\\[2pt]\small One folder: sub-folders (ending in /) and files.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault list [FOLDER]}\\[2pt]\small Every file under it, full names. Pipes cleanly.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault get [-r] NAME [-o OUT]}\\[2pt]\small Decrypt out, or a folder with -r. Never overwrites. Verifies first.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault mv OLD NEW}\\[2pt]\small Rename or move a file or folder. Instant.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault cp [-r] OLD NEW}\\[2pt]\small Copy a file, or a folder with -r. Instant, no extra space.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault rm [-r] [-f] NAME}\\[2pt]\small Delete a file, or a folder with -r. Frees the space. No undo.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault unlock [--for 5m] [--idle]}\\[2pt]\small Remember the key, so commands stop asking. Fixed deadline, or since last use.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault lock}\\[2pt]\small Forget the remembered key now.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault help}\\[2pt]\small Show the command summary.\end{tcolorbox}
|
||||
\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries man vault}\\[2pt]\small A manual page for vault and for every command.\end{tcolorbox}
|
||||
\end{tcbraster}
|
||||
|
||||
\section*{Options and variables}
|
||||
@@ -1261,6 +1415,11 @@ passphrase. Nothing in this chapter helps with that. Back up your key.
|
||||
\textrm{(genkey prompt)} & VAULT\_DECOY\_PASSPHRASE & asked; empty means none \\
|
||||
-o OUT & \textrm{---} & the file's own name, here \\
|
||||
--filler SIZE & \textrm{---} & a few tens of MB, for \cmd{init} \\
|
||||
--yes & \textrm{---} & ask, for \cmd{migrate} \\
|
||||
-r & \textrm{---} & \cmd{rm}, \cmd{get}, \cmd{cp}: one file; \cmd{ls}: one level \\
|
||||
--for DURATION & \textrm{---} & \cmd{5m}, for \cmd{unlock} \\
|
||||
--idle & \textrm{---} & \cmd{unlock}: count from the unlock, not the last use \\
|
||||
-f & \textrm{---} & \cmd{rm}: ask before \cmd{-r}, fail if missing \\
|
||||
\end{vtable}
|
||||
|
||||
\section*{Numbers worth knowing}
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
.TH VAULT-ABSORB 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-absorb \- add the files in a tar archive to an encrypted vault
|
||||
.SH SYNOPSIS
|
||||
.B vault absorb
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RI [ FOLDER ]
|
||||
.B <
|
||||
.I ARCHIVE
|
||||
.SH DESCRIPTION
|
||||
.B vault absorb
|
||||
reads a tar archive from standard input and stores every regular file in
|
||||
it, beneath
|
||||
.I FOLDER
|
||||
if one is given, keeping the paths, permissions and modification times
|
||||
recorded in the archive. A gzip-compressed archive is detected and
|
||||
unpacked automatically.
|
||||
.PP
|
||||
Each file is encrypted straight from the stream; nothing is unpacked to
|
||||
disk. Anything already stored under the same name is replaced.
|
||||
.PP
|
||||
Directory entries are not stored (directories exist implicitly). Symbolic
|
||||
and hard links, devices and other special entries are reported on standard
|
||||
error and skipped, as is any entry whose name would escape
|
||||
.I FOLDER
|
||||
through
|
||||
.BR .. .
|
||||
A leading
|
||||
.B ./
|
||||
or
|
||||
.B /
|
||||
is dropped.
|
||||
.PP
|
||||
The passphrase prompt reads from the terminal, not standard input, so
|
||||
it still works with an archive piped in. It waits until the archive starts
|
||||
arriving, so a program feeding the pipe, such as
|
||||
.BR gpg (1)
|
||||
asking for its own passphrase, has finished with the terminal first.
|
||||
An empty pipe fails without asking at all.
|
||||
The vault is locked exclusively for the whole run.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH EXIT STATUS
|
||||
0 if at least one file was stored and the archive was read to the end.
|
||||
1 if the archive is malformed or ends early, holds no regular files, or
|
||||
anything fails; files stored before the failure stay in the vault.
|
||||
.SH EXAMPLES
|
||||
Unpack an existing archive into the vault under
|
||||
.BR backup/ :
|
||||
.PP
|
||||
.RS
|
||||
vault absorb backup < photos.tar.gz
|
||||
.RE
|
||||
.PP
|
||||
Store a directory under
|
||||
.BR backup/photos/ :
|
||||
.PP
|
||||
.RS
|
||||
tar \-cz photos | vault absorb backup
|
||||
.RE
|
||||
.PP
|
||||
Pull a remote directory straight into the vault:
|
||||
.PP
|
||||
.RS
|
||||
ssh server 'tar \-C /etc \-c .' | vault absorb server1/etc
|
||||
.RE
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-add (1),
|
||||
.BR tar (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-ADD 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-ADD 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-add \- encrypt files, directories or URLs into the vault
|
||||
.SH SYNOPSIS
|
||||
@@ -7,6 +7,10 @@ vault-add \- encrypt files, directories or URLs into the vault
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RB [ \-\-max\-bytes
|
||||
.IR SIZE ]
|
||||
.RB [ \-\-timeout
|
||||
.IR DURATION ]
|
||||
.IR FILE | DIR | URL
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
@@ -17,7 +21,14 @@ and the key are skipped. A stored name that already exists is replaced.
|
||||
.PP
|
||||
An \fBhttp://\fR or \fBhttps://\fR \fIURL\fR is downloaded with a Chrome user
|
||||
agent and stored under the last element of its final (post-redirect) path, or
|
||||
the host name if there is none. Only a 200 response is accepted.
|
||||
the host name if there is none. Only a 200 response is accepted. Certificates
|
||||
are always verified; at most 10 redirects are followed, to http or https
|
||||
only. The download is staged in a private temporary file (it must be
|
||||
complete before it can be stored), which is removed afterwards whatever
|
||||
happens; it is plaintext on disk meanwhile.
|
||||
.PP
|
||||
A file that changes while it is being added (it grows, shrinks, or is
|
||||
replaced) is not stored, and the command fails.
|
||||
.PP
|
||||
\fIFOLDER\fR places the result inside a folder in the vault. File contents
|
||||
are cut into 4 MiB chunks, the last bucket-padded.
|
||||
@@ -28,6 +39,14 @@ Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.BI \-\-max\-bytes " SIZE"
|
||||
Largest download accepted, counted after decompression, such as 500M or 50G.
|
||||
Default 16G; at most 1T.
|
||||
.TP
|
||||
.BI \-\-timeout " DURATION"
|
||||
Deadline for the whole download, such as 30m. Default 2h. Connecting must
|
||||
take under 30 seconds and the response headers must arrive within a minute.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
@@ -41,6 +60,7 @@ vault add https://example.com/paper.pdf papers
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-migrate (1),
|
||||
.BR vault\-mirror (1)
|
||||
.BR vault\-mirror (1),
|
||||
.BR vault\-absorb (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
.TH VAULT-CP 1 "September 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-cp \- copy a stored file or folder
|
||||
.SH SYNOPSIS
|
||||
.B vault cp
|
||||
.RB [ \-r ]
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.I OLD NEW
|
||||
.SH DESCRIPTION
|
||||
Copies \fIOLD\fR to \fINEW\fR, following the same naming rules as
|
||||
.BR vault\-mv (1).
|
||||
A folder needs \fB\-r\fR. The copy shares the original's encrypted
|
||||
objects rather than re-encrypting them, so it is instant and takes no
|
||||
extra space; deleting either name leaves the other intact.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-r
|
||||
Copy a folder and everything under it.
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-mv (1),
|
||||
.BR vault\-rm (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
@@ -1,8 +1,9 @@
|
||||
.TH VAULT-GET 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-GET 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-get \- decrypt a stored file
|
||||
.SH SYNOPSIS
|
||||
.B vault get
|
||||
.RB [ \-r ]
|
||||
.RB [ \-o
|
||||
.IR OUTPUT ]
|
||||
.RB [ \-\-key
|
||||
@@ -12,12 +13,26 @@ vault-get \- decrypt a stored file
|
||||
.I NAME
|
||||
.SH DESCRIPTION
|
||||
Decrypts \fINAME\fR. Nothing appears at the output path until every object has
|
||||
authenticated. The stored mode and modification time are restored. It refuses
|
||||
to overwrite an existing file. Takes a shared lock.
|
||||
authenticated, padding and end of file included; until then the output is a
|
||||
private temporary file in the same directory, removed on failure. The stored
|
||||
mode and modification time are restored. It never replaces anything already
|
||||
at the output path, including a symbolic link (even a dangling one) or a
|
||||
file created while it runs. Takes a shared lock.
|
||||
.PP
|
||||
With \fB\-r\fR, \fINAME\fR is a folder, and every file under it is
|
||||
decrypted into the output directory, keeping the folder structure. Each file
|
||||
is checked as above; a clash stops the run, leaving the files already
|
||||
written. Directories are created as needed, mode 0700; a symbolic link (or
|
||||
anything but a directory) where a directory belongs inside the output
|
||||
directory stops the run rather than being followed.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-o " OUTPUT"
|
||||
Output path. Default: the base name of \fINAME\fR in the current directory.
|
||||
Output path, or with \fB\-r\fR output directory. Default: the base name of
|
||||
\fINAME\fR in the current directory.
|
||||
.TP
|
||||
.B \-r
|
||||
Decrypt a folder and everything under it.
|
||||
.PP
|
||||
See also \fB\-\-key\fR and \fB\-\-vault\fR in
|
||||
.BR vault (1).
|
||||
|
||||
@@ -1,23 +1,65 @@
|
||||
.TH VAULT-LIST 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-LIST 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-list \- list files in the vault
|
||||
vault-list, vault-ls \- list files and folders in the vault
|
||||
.SH SYNOPSIS
|
||||
.B vault ls
|
||||
.RB [ \-r ]
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RI [ FOLDER ]
|
||||
.br
|
||||
.B vault list
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RI [ PREFIX ]
|
||||
.br
|
||||
.B vault ls
|
||||
\&...
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
Prints one line per stored file, sorted by name: human-readable size,
|
||||
modification time (local, minute resolution) and name. With \fIPREFIX\fR only
|
||||
that file, or files under that folder, are shown. Records the passphrase
|
||||
cannot open are skipped silently. Takes a shared lock.
|
||||
Both print one line per entry: human-readable size, modification time
|
||||
(local, minute resolution) and name, in fixed-width columns so the name
|
||||
always starts at character 31.
|
||||
.PP
|
||||
.B vault ls
|
||||
shows one level, like
|
||||
.BR ls (1):
|
||||
each sub-folder of
|
||||
.I FOLDER
|
||||
(the top of the vault if none is given) once, with a trailing
|
||||
.BR / ,
|
||||
and the files directly in it.
|
||||
A folder's size is the total of everything under it and its time is the
|
||||
newest among them. Names are relative to
|
||||
.IR FOLDER ,
|
||||
so
|
||||
.B vault ls photos
|
||||
then
|
||||
.B vault ls photos/2025
|
||||
walks down the tree. Folders are listed first, then files, each sorted by
|
||||
name. If
|
||||
.I FOLDER
|
||||
is a file's name, that file alone is shown.
|
||||
.PP
|
||||
.B vault list
|
||||
shows every file at or under
|
||||
.IR FOLDER ,
|
||||
recursively, with full names, sorted. It is the form to pipe into other
|
||||
commands, and
|
||||
.B vault ls \-r
|
||||
is the same.
|
||||
.PP
|
||||
Records the passphrase cannot open are skipped silently. Takes a shared lock.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-r
|
||||
.BR ls :
|
||||
list everything under
|
||||
.I FOLDER
|
||||
with full names, as
|
||||
.B vault list
|
||||
does.
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
@@ -27,8 +69,23 @@ Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH EXIT STATUS
|
||||
0 on success. 1 if
|
||||
.B vault ls
|
||||
is given a
|
||||
.I FOLDER
|
||||
with nothing in or under it, or on any other error.
|
||||
.SH EXAMPLES
|
||||
.RS
|
||||
vault ls
|
||||
.br
|
||||
vault ls photos/2025
|
||||
.br
|
||||
vault list | cut \-c31\- # every name, one per line
|
||||
.RE
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-get (1)
|
||||
.BR vault\-get (1),
|
||||
.BR vault\-rm (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
.so man1/vault-unlock.1
|
||||
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-MIGRATE 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-MIGRATE 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-migrate \- move files into the vault, deleting the originals
|
||||
.SH SYNOPSIS
|
||||
@@ -13,7 +13,10 @@ vault-migrate \- move files into the vault, deleting the originals
|
||||
.SH DESCRIPTION
|
||||
Works like
|
||||
.BR vault\-add (1)
|
||||
but deletes each original once it is stored, then any directories left empty.
|
||||
but deletes each original once it is stored (and its record is on disk),
|
||||
then any directories left empty. An original is deleted only if it is still
|
||||
the same file, unchanged, that was stored; otherwise it is kept and the
|
||||
command fails.
|
||||
Directories still holding skipped files stay. A URL cannot be migrated.
|
||||
.PP
|
||||
Before storing anything it lists the vault names that would be overwritten and
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-MIRROR 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-MIRROR 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-mirror \- mirror a website into an encrypted vault
|
||||
.SH SYNOPSIS
|
||||
@@ -7,6 +7,12 @@ vault-mirror \- mirror a website into an encrypted vault
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RB [ \-\-max\-pages
|
||||
.IR N ]
|
||||
.RB [ \-\-max\-bytes
|
||||
.IR SIZE ]
|
||||
.RB [ \-\-timeout
|
||||
.IR DURATION ]
|
||||
.I URL
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
@@ -20,21 +26,30 @@ and stores every page and asset it finds in the vault, beneath
|
||||
if no folder is given).
|
||||
.PP
|
||||
The crawl is single threaded: one request at a time, in breadth-first order.
|
||||
It follows only links on the same host as
|
||||
It follows only links on the same host (and port) as
|
||||
.IR URL ,
|
||||
and requests are sent with a Chrome user agent string.
|
||||
and only redirects to that host; a redirect elsewhere skips the page.
|
||||
Requests are sent with a Chrome user agent string, and certificates are
|
||||
always verified.
|
||||
.PP
|
||||
Links in HTML
|
||||
Links in HTML start tags
|
||||
.RB ( href ,
|
||||
.BR src ,
|
||||
.BR action
|
||||
and
|
||||
.B poster
|
||||
attributes) and in CSS
|
||||
.RB ( url(...) )
|
||||
that point into the mirror are rewritten to relative paths, so the copy
|
||||
browses correctly once fetched out of the vault. Links to other hosts, and
|
||||
fragments, are left as they were.
|
||||
attributes) and CSS
|
||||
.RB ( url(...)
|
||||
in style sheets,
|
||||
.B <style>
|
||||
elements and
|
||||
.B style
|
||||
attributes) that point into the mirror are rewritten to relative paths, so
|
||||
the copy browses correctly once fetched out of the vault. Relative links are
|
||||
resolved against the page's final address and its
|
||||
.B <base href>
|
||||
if it has one. Links to other hosts, script and comment text, and every
|
||||
other byte of the page are left as they were. Fragments are kept.
|
||||
.PP
|
||||
Pages are named after their URL path. A path that is empty or ends in
|
||||
.B /
|
||||
@@ -42,19 +57,39 @@ becomes
|
||||
.BR index.html ;
|
||||
a query string is appended as
|
||||
.BI _ query
|
||||
(URL-escaped). Anything already stored under the same name is replaced.
|
||||
(URL-escaped). A name already stored by an earlier run is replaced, but if
|
||||
two different URLs in one crawl would get the same name, the second is
|
||||
skipped rather than overwriting the first.
|
||||
.PP
|
||||
A page that fails to download, or answers with anything but
|
||||
A page that fails to download, answers with anything but
|
||||
.BR "200 OK" ,
|
||||
is reported on standard error and skipped; the crawl carries on.
|
||||
The vault is locked exclusively for the whole run.
|
||||
or exceeds a limit is reported on standard error as
|
||||
.BI "skipping " URL ": " reason
|
||||
and skipped; the crawl carries on. HTML and CSS are rewritten in memory;
|
||||
other files are staged in a private temporary file (plaintext on disk
|
||||
meanwhile) that is always removed. The vault is locked exclusively for the
|
||||
whole run.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-max\-pages " N"
|
||||
Pages and assets to fetch at most. Default 10000.
|
||||
.TP
|
||||
.BI \-\-max\-bytes " SIZE"
|
||||
Total bytes to download at most, counted after decompression. Default 4G.
|
||||
.TP
|
||||
.BI \-\-timeout " DURATION"
|
||||
Deadline for each request. Default 5m.
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.PP
|
||||
A single HTML or CSS file larger than 16 MiB is skipped.
|
||||
.SH EXIT STATUS
|
||||
0 when everything found was stored, 3 when some of it was skipped, 1 on
|
||||
an error that stops the crawl.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
@@ -75,11 +110,19 @@ vault mirror https://example.com/ archive/2026
|
||||
vault get archive/2026/example.com/index.html
|
||||
.RE
|
||||
.SH LIMITATIONS
|
||||
Links are found by pattern matching, not by parsing HTML: URLs assembled by
|
||||
scripts, and
|
||||
URLs assembled by scripts, and
|
||||
.B srcset
|
||||
lists, are neither followed nor rewritten. There is no depth limit, rate
|
||||
limit or robots.txt handling; only mirror sites you may copy.
|
||||
lists, are neither followed nor rewritten. In attribute values only numeric
|
||||
character references and
|
||||
.BR & ,
|
||||
.BR < ,
|
||||
.BR > ,
|
||||
.BR " ,
|
||||
.B '
|
||||
and
|
||||
.B
|
||||
are decoded. There is no rate limit or robots.txt handling; only mirror
|
||||
sites you may copy.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1)
|
||||
.SH AUTHOR
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
.TH VAULT-MV 1 "September 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-mv \- rename a stored file
|
||||
vault-mv \- rename or move a stored file or folder
|
||||
.SH SYNOPSIS
|
||||
.B vault mv
|
||||
.RB [ \-\-key
|
||||
@@ -12,9 +12,12 @@ vault-mv \- rename a stored file
|
||||
.B vault rename
|
||||
\&...
|
||||
.SH DESCRIPTION
|
||||
Renames \fIOLD\fR to \fINEW\fR, which may be in another folder. Fails if
|
||||
\fINEW\fR exists or is not a valid vault path. File contents are not
|
||||
re-encrypted.
|
||||
Renames \fIOLD\fR, a file or a folder and everything under it, to
|
||||
\fINEW\fR. If \fINEW\fR ends in / or names an existing folder, \fIOLD\fR
|
||||
moves into it, keeping its base name, as with
|
||||
.BR mv (1).
|
||||
Fails, changing nothing, if any new name exists or is not a valid vault
|
||||
path. File contents are not re-encrypted.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
@@ -28,6 +31,7 @@ Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-cp (1),
|
||||
.BR vault\-rm (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
|
||||
@@ -1,21 +1,66 @@
|
||||
.TH VAULT-RM 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-RM 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-rm \- delete a stored file
|
||||
vault-rm \- delete stored files or folders
|
||||
.SH SYNOPSIS
|
||||
.B vault rm
|
||||
.RB [ \-f ]
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.I NAME
|
||||
.br
|
||||
.B vault rm \-r
|
||||
.RB [ \-f ]
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.I FOLDER
|
||||
.br
|
||||
.B vault delete
|
||||
\&...
|
||||
.SH DESCRIPTION
|
||||
Deletes the stored file \fINAME\fR from the vault. It does not ask for
|
||||
confirmation.
|
||||
Deletes the stored file
|
||||
.I NAME
|
||||
from the vault: its catalogue record and every encrypted object holding
|
||||
its contents, so the space is freed on disk straight away.
|
||||
.PP
|
||||
With
|
||||
.BR \-r ,
|
||||
deletes
|
||||
.I FOLDER
|
||||
and everything at or under it. Because there is no undo, it first lists
|
||||
what it will delete (the first 20 names, then a count) with the total size,
|
||||
and asks on the terminal. Anything but
|
||||
.B y
|
||||
or
|
||||
.B yes
|
||||
deletes nothing, and so does having no terminal to ask on. A trailing
|
||||
.B /
|
||||
on
|
||||
.I FOLDER
|
||||
is ignored, and
|
||||
.B vault rm \-r /
|
||||
is refused rather than emptying the vault.
|
||||
.PP
|
||||
Each deleted file is reported on standard error, and
|
||||
.B \-r
|
||||
ends with the number of files and their total size. The vault is locked
|
||||
exclusively for the whole run. A key sees only its own files, so the
|
||||
decoy passphrase never deletes real files, or the other way round.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.B \-r
|
||||
Delete a folder and everything under it.
|
||||
.TP
|
||||
.B \-f
|
||||
Do not ask before a recursive delete, and do not fail when there is
|
||||
nothing to delete.
|
||||
.TP
|
||||
.BR \-rf ", " \-fr
|
||||
Both of the above.
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
@@ -25,6 +70,19 @@ Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH EXIT STATUS
|
||||
0 if everything asked for was deleted, or there was nothing to delete with
|
||||
.BR \-f .
|
||||
1 if the name is not in the vault, the delete was declined, or anything
|
||||
fails; files already deleted stay deleted.
|
||||
.SH EXAMPLES
|
||||
.RS
|
||||
vault rm notes.md
|
||||
.br
|
||||
vault rm \-r old/2024 # asks first
|
||||
.br
|
||||
vault rm \-rf tmp # no questions
|
||||
.RE
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR vault\-list (1)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
.TH VAULT-UNLOCK 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-unlock, vault-lock \- remember the unlocked key for a while, or forget it
|
||||
.SH SYNOPSIS
|
||||
.B vault unlock
|
||||
.RB [ \-\-for
|
||||
.IR DURATION ]
|
||||
.RB [ \-\-idle ]
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.br
|
||||
.B vault lock
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.SH DESCRIPTION
|
||||
.B unlock
|
||||
asks for the passphrase, checks it, and starts a helper process, much like
|
||||
.BR ssh\-agent (1),
|
||||
that holds the unlocked key in memory until \fIDURATION\fR has passed
|
||||
(default 5m). Until then every command using the same key file skips the
|
||||
passphrase prompt. By default the deadline is fixed when you unlock and
|
||||
using the key does not extend it; with \fB\-\-idle\fR it is \fIDURATION\fR
|
||||
after the last command that used the key. Running
|
||||
.B unlock
|
||||
again replaces the helper with a fresh one.
|
||||
.PP
|
||||
.B lock
|
||||
tells the helper to wipe the key and exit at once.
|
||||
.PP
|
||||
The key reaches the helper over a pipe and is never written to disk. Its
|
||||
memory is locked against swapping and excluded from core dumps; on Linux the
|
||||
helper is also not debuggable. It answers only on a Unix socket inside
|
||||
\fI$XDG_RUNTIME_DIR/vault-UID\fR (or the temporary directory), a directory
|
||||
that must belong to this user with mode 0700. Rebooting ends it too.
|
||||
If the helper cannot lock its memory, disable core dumps or (on Linux) make
|
||||
itself undebuggable, it refuses the key and
|
||||
.B unlock
|
||||
fails, saying why. The helper is the same program, started with the
|
||||
passphrase variables removed from its environment. Helpers started by
|
||||
earlier (Go) versions of vault are compatible and can be used and locked.
|
||||
.PP
|
||||
Not available on Windows, where every command asks for the passphrase.
|
||||
.PP
|
||||
While the key is unlocked, any program running as this user can use it, and
|
||||
it is whichever key the passphrase opened, real or decoy. Lock before
|
||||
handing the machine to anyone.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-for " DURATION"
|
||||
How long to remember the key, such as 90s, 5m or 2h. Default 5m.
|
||||
.TP
|
||||
.B \-\-idle
|
||||
Count \fIDURATION\fR from the last time a command used the key, rather than
|
||||
from the unlock, so the key stays while you keep working.
|
||||
.PP
|
||||
See also \fB\-\-key\fR in
|
||||
.BR vault (1).
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase for
|
||||
.BR unlock .
|
||||
When set for any other command, it is used instead of the remembered key.
|
||||
.TP
|
||||
.B XDG_RUNTIME_DIR
|
||||
Where the helper's socket directory goes.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1),
|
||||
.BR ssh\-agent (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
@@ -1,4 +1,4 @@
|
||||
.TH VAULT 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault \- post-quantum encrypted file vault
|
||||
.SH SYNOPSIS
|
||||
@@ -33,21 +33,40 @@ Like add, then delete each original.
|
||||
.BR vault\-mirror (1)
|
||||
Crawl a website into the vault.
|
||||
.TP
|
||||
.BR vault\-absorb (1)
|
||||
Add every file in a tar archive read from standard input.
|
||||
.TP
|
||||
.BR vault\-list (1)
|
||||
List stored files (alias:
|
||||
.BR ls ).
|
||||
.B ls
|
||||
lists one folder, with sub-folders marked;
|
||||
.B list
|
||||
lists every file under a folder.
|
||||
.TP
|
||||
.BR vault\-get (1)
|
||||
Decrypt a stored file.
|
||||
Decrypt a stored file, or a whole folder with
|
||||
.BR \-r .
|
||||
.TP
|
||||
.BR vault\-mv (1)
|
||||
Rename a stored file (alias:
|
||||
Rename or move a stored file or folder (alias:
|
||||
.BR rename ).
|
||||
.TP
|
||||
.BR vault\-cp (1)
|
||||
Copy a stored file, or a whole folder with
|
||||
.BR \-r .
|
||||
.TP
|
||||
.BR vault\-rm (1)
|
||||
Delete a stored file (alias:
|
||||
Delete a stored file, or a whole folder with
|
||||
.B \-r
|
||||
(alias:
|
||||
.BR delete ).
|
||||
.TP
|
||||
.BR vault\-unlock (1)
|
||||
Remember the unlocked key for a while (default 5 minutes), so commands
|
||||
skip the passphrase.
|
||||
.TP
|
||||
.BR vault\-lock (1)
|
||||
Forget it now.
|
||||
.TP
|
||||
.B help
|
||||
Print usage.
|
||||
.SH OPTIONS
|
||||
@@ -58,6 +77,11 @@ Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.BI \-\-workers " N"
|
||||
Threads for encryption, decryption and file I/O, 1 to 256. Default: the
|
||||
number of CPUs, at most 8. Up to eight 4 MiB objects are in flight at once
|
||||
whatever the count.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_KEY
|
||||
@@ -67,11 +91,25 @@ Default key file.
|
||||
Default vault directory.
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
Passphrase, instead of prompting on the terminal or using a key remembered by
|
||||
.BR vault\-unlock (1).
|
||||
.TP
|
||||
.B VAULT_DECOY_PASSPHRASE
|
||||
Decoy passphrase used by
|
||||
.BR "vault genkey" .
|
||||
.TP
|
||||
.BR HTTPS_PROXY ", " HTTP_PROXY ", " NO_PROXY
|
||||
Proxies for
|
||||
.B add
|
||||
.I URL
|
||||
and
|
||||
.BR mirror ,
|
||||
as other programs use them; local addresses are never proxied.
|
||||
.TP
|
||||
.BR XDG_RUNTIME_DIR ", " TMPDIR
|
||||
Where
|
||||
.BR vault\-unlock (1)
|
||||
keeps its socket, and where downloads are staged.
|
||||
.SH LOCKING
|
||||
Commands that change the vault hold an exclusive lock on
|
||||
.IR DIR/.lock ;
|
||||
@@ -86,7 +124,10 @@ and
|
||||
fills each vault with unreadable filler, so hidden files look like filler to
|
||||
anyone holding only the decoy passphrase.
|
||||
.SH EXIT STATUS
|
||||
0 on success; 1 on error (diagnostics on standard error); 2 when run with no command.
|
||||
0 on success; 1 on error (diagnostics on standard error); 2 when run with no
|
||||
command; 3 when
|
||||
.BR vault\-mirror (1)
|
||||
stored some resources but had to skip others.
|
||||
.SH EXAMPLES
|
||||
.nf
|
||||
vault genkey
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
[package]
|
||||
name = "vault-fuzz"
|
||||
version = "0.0.0"
|
||||
publish = false
|
||||
edition = "2024"
|
||||
|
||||
[package.metadata]
|
||||
cargo-fuzz = true
|
||||
|
||||
[dependencies]
|
||||
libfuzzer-sys = "0.4"
|
||||
tar = { version = "0.4.46", default-features = false }
|
||||
vault = { path = ".." }
|
||||
|
||||
# Not part of the main package's workspace.
|
||||
[workspace]
|
||||
|
||||
[profile.release]
|
||||
debug = 1
|
||||
|
||||
[[bin]]
|
||||
name = "header"
|
||||
path = "fuzz_targets/header.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "stream"
|
||||
path = "fuzz_targets/stream.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "keyfile"
|
||||
path = "fuzz_targets/keyfile.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "record"
|
||||
path = "fuzz_targets/record.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "names"
|
||||
path = "fuzz_targets/names.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "html"
|
||||
path = "fuzz_targets/html.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "tar"
|
||||
path = "fuzz_targets/tar.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The Fury header parser never panics, and accepts only canonical headers:
|
||||
//! whatever it accepts re-serialises to exactly the bytes it consumed.
|
||||
#![no_main]
|
||||
use std::io::Read;
|
||||
use vault::crypto::header;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Ok((h, mut rest)) = header::parse(data) {
|
||||
let mut tail = Vec::new();
|
||||
rest.read_to_end(&mut tail).unwrap();
|
||||
let consumed = &data[..data.len() - tail.len()];
|
||||
assert_eq!(h.marshal(), consumed, "non-canonical header accepted");
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The mirror's HTML and CSS rewriters never panic, and rewriting nothing
|
||||
//! returns the input byte for byte.
|
||||
#![no_main]
|
||||
use vault::net::html;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let _ = html::tokenize(data);
|
||||
assert_eq!(html::rewrite_html(data, &mut |_| None, &mut |_| None), data);
|
||||
assert_eq!(html::rewrite_css(data, false, &mut |_| None), data);
|
||||
let _ = html::rewrite_html(data, &mut |_| None, &mut |u| Some(format!("x/{}", u.len())));
|
||||
if let Ok(s) = std::str::from_utf8(data) {
|
||||
let _ = vault::net::Url::parse(s).map(|u| u.join("../a?b#c"));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! PEM decoding never panics, and decoding what it encoded is the identity.
|
||||
#![no_main]
|
||||
use vault::keyfile::{decode_pem, encode_pem, parse_body};
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Some(body) = decode_pem(data) {
|
||||
let _ = parse_body(&body);
|
||||
assert_eq!(decode_pem(encode_pem(&body).as_bytes()), Some(body));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Path validation: a valid vault name is clean, relative and confined; the
|
||||
//! tar name check only ever yields valid-shaped relative names.
|
||||
#![no_main]
|
||||
use vault::store::VaultName;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let Ok(s) = std::str::from_utf8(data) else { return };
|
||||
if let Some(n) = VaultName::new(s) {
|
||||
let n = n.as_str();
|
||||
assert_eq!(vault::util::clean(n), n);
|
||||
assert!(!n.starts_with('/') && !n.contains('\\') && !n.split('/').any(|c| c == ".." || c == "." || c.is_empty()));
|
||||
assert!(n.len() <= 1024);
|
||||
}
|
||||
if let Some(n) = vault::absorb::safe_name(s) {
|
||||
assert!(!n.is_empty() && !n.starts_with('/') && !n.split('/').any(|c| c == ".."));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Record JSON parsing never panics; an accepted record re-encodes (Go's
|
||||
//! JSON) to something that parses back to the same values.
|
||||
#![no_main]
|
||||
use vault::store::{Id, Record};
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Ok(r) = Record::from_json(data, Id([0; 16])) {
|
||||
let Some(j) = r.to_json() else { return };
|
||||
let r2 = Record::from_json(&j, Id([0; 16])).expect("own encoding parses");
|
||||
assert_eq!(r.name, r2.name);
|
||||
assert_eq!((r.size, r.mode, r.modified, r.added), (r2.size, r2.mode, r2.modified, r2.added));
|
||||
assert!(r.seed.same_as(&r2.seed));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The streaming, in-memory and random-access STREAM readers agree on every
|
||||
//! input: the same plaintext or all reject. Inputs are mostly forgeries;
|
||||
//! the first byte selects a mode that starts from a valid stream instead.
|
||||
#![no_main]
|
||||
use std::io::{Read, Write};
|
||||
use vault::crypto::stream::{StreamReader, StreamReaderAt, StreamWriter, open_all};
|
||||
|
||||
const KEY: [u8; 32] = [7; 32];
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let Some((&mode, rest)) = data.split_first() else { return };
|
||||
let input = if mode & 1 == 1 {
|
||||
// A valid stream of the given plaintext, then a mutation.
|
||||
let mut w = StreamWriter::new(&KEY, Vec::new());
|
||||
let n = rest.len().min(200_000);
|
||||
w.write_all(&rest[..n]).unwrap();
|
||||
let mut c = w.finish().unwrap();
|
||||
if mode & 2 == 2 && !c.is_empty() {
|
||||
let i = (mode as usize * 7919) % c.len();
|
||||
c[i] ^= mode | 1;
|
||||
}
|
||||
if mode & 4 == 4 {
|
||||
c.truncate(c.len().saturating_sub(mode as usize % 40));
|
||||
}
|
||||
c
|
||||
} else {
|
||||
rest.to_vec()
|
||||
};
|
||||
let mut a = Vec::new();
|
||||
let streaming = StreamReader::new(&KEY, &input[..]).read_to_end(&mut a).map(|_| a);
|
||||
let mut b = input.clone();
|
||||
let whole = open_all(&KEY, &mut b).map(|n| b[..n].to_vec());
|
||||
assert_eq!(streaming.is_ok(), whole.is_ok());
|
||||
if let (Ok(x), Ok(y)) = (&streaming, &whole) {
|
||||
assert_eq!(x, y);
|
||||
}
|
||||
if let Ok(r) = StreamReaderAt::new(&KEY, &input[..], input.len() as u64) {
|
||||
let mut out = vec![0u8; r.size() as usize];
|
||||
let got = r.read_at(&mut out, 0).map(|n| out[..n].to_vec());
|
||||
assert_eq!(got.is_ok(), whole.is_ok());
|
||||
if let (Ok(x), Ok(y)) = (got, whole) {
|
||||
assert_eq!(x, y);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Walking arbitrary tar input as absorb does never panics, and every name
|
||||
//! it would store is confined.
|
||||
#![no_main]
|
||||
use std::io::Read;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let mut a = tar::Archive::new(data);
|
||||
let Ok(entries) = a.entries() else { return };
|
||||
for e in entries.take(64) {
|
||||
let Ok(mut e) = e else { return };
|
||||
let name = e.path_bytes().into_owned();
|
||||
if let Some(n) = std::str::from_utf8(&name).ok().and_then(vault::absorb::safe_name) {
|
||||
assert!(!n.split('/').any(|c| c == ".."));
|
||||
}
|
||||
let mut sink = Vec::new();
|
||||
let _ = e.by_ref().take(1 << 20).read_to_end(&mut sink);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
# Go oracle (test-only)
|
||||
|
||||
`oracle/go` is a frozen copy of the Go implementation of vault, taken from
|
||||
commit `4c1507c` ("Add unlock --idle, and packaging/repo.sh for
|
||||
repo.dixon.cx"), the last Go release on `master`. It is kept as the
|
||||
reference that the Rust implementation is checked against. It is **not**
|
||||
part of the shipped product: release builds, packages and the `vault`
|
||||
binary contain no Go code and need no Go toolchain.
|
||||
|
||||
Provenance:
|
||||
|
||||
- All 32 Go sources and `go.mod`/`go.sum` under `oracle/go` are
|
||||
byte-identical to the same paths at the repository root at `4c1507c`.
|
||||
- The only additions are test files whose names contain `oracle`
|
||||
(`*oracle_vectors_test.go`, `oracle_decrypt_test.go`,
|
||||
`*oracle_bench_test.go`). They call the frozen code and record what it
|
||||
computes; they skip themselves unless `VAULT_ORACLE_OUT` or
|
||||
`VAULT_ORACLE_DECRYPT` is set (benchmarks run only with `-bench`), and add
|
||||
nothing to the oracle binary.
|
||||
- Dependencies are pinned by `go.mod`/`go.sum`: CIRCL v1.6.4,
|
||||
golang.org/x/crypto v0.53.0, x/sys v0.46.0, x/term v0.44.0. The module
|
||||
needs Go 1.26.6 or newer.
|
||||
|
||||
Check the copy is still faithful:
|
||||
|
||||
```sh
|
||||
for f in $(git ls-tree -r --name-only 4c1507c | grep -E '\.go$|^go\.(mod|sum)$'); do
|
||||
cmp <(git show 4c1507c:$f) oracle/go/$f || echo "DIFFERS: $f"
|
||||
done
|
||||
```
|
||||
|
||||
Use:
|
||||
|
||||
```sh
|
||||
oracle/gen-vectors.sh # rewrite tests/vectors from the oracle
|
||||
(cd oracle/go && go build -o ../../target/vault-go .) # build the oracle CLI
|
||||
(cd oracle/go && go test ./...) # its own test-suite
|
||||
```
|
||||
|
||||
`cargo test` builds the oracle binary itself for the interoperability tests
|
||||
when `go` is on `PATH` (or uses `$VAULT_GO_ORACLE`), and reports them as
|
||||
skipped otherwise.
|
||||
|
||||
All passphrases, seeds and plaintexts in the vectors are synthetic and
|
||||
public (`oracle-real passphrase`, `oracle-decoy passphrase`, fixed byte
|
||||
patterns). Never commit real keys or real vault contents.
|
||||
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the Go-oracle vectors and fixtures in tests/vectors.
|
||||
#
|
||||
# Everything is computed by the frozen Go implementation in oracle/go (see
|
||||
# oracle/README.md); nothing comes from the Rust code under test. Key files
|
||||
# and ciphertexts use fresh randomness, so regenerating changes them; the
|
||||
# deterministic vectors (*.json fields derived from fixed seeds) do not.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
out=$PWD/tests/vectors
|
||||
mkdir -p "$out"
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream)
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./)
|
||||
echo "vectors written to $out"
|
||||
@@ -0,0 +1,75 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// absorb reads a tar archive, gzipped or not, from r and adds every regular
|
||||
// file in it beneath dest, streaming each straight into the vault so no
|
||||
// plaintext touches the disk. Other entries, and names that would escape
|
||||
// dest, are reported and skipped.
|
||||
func absorb(s *store, cat map[string][]*record, r io.Reader, dest string) error {
|
||||
br := bufio.NewReader(r)
|
||||
if magic, _ := br.Peek(2); bytes.Equal(magic, []byte{0x1f, 0x8b}) {
|
||||
zr, err := gzip.NewReader(br)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zr.Close()
|
||||
r = zr
|
||||
} else {
|
||||
r = br
|
||||
}
|
||||
tr := tar.NewReader(r)
|
||||
n := 0
|
||||
for {
|
||||
hdr, err := tr.Next()
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading archive: %w", err)
|
||||
}
|
||||
if hdr.Typeflag != tar.TypeReg {
|
||||
if hdr.Typeflag != tar.TypeDir {
|
||||
fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", hdr.Name)
|
||||
}
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(path.Clean("/"+hdr.Name), "/")
|
||||
if strings.Contains(hdr.Name, "\\") || name == "" || name != path.Clean(strings.TrimPrefix(strings.TrimPrefix(hdr.Name, "./"), "/")) {
|
||||
fmt.Fprintf(os.Stderr, "skipping %s: unsafe name\n", hdr.Name)
|
||||
continue
|
||||
}
|
||||
name = path.Join(dest, name)
|
||||
replaced, err := s.addReader(cat, tr, hdr.Size, hdr.FileInfo().Mode(), hdr.ModTime, name)
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
err = fmt.Errorf("%s: archive ends early", hdr.Name)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
verb := "added"
|
||||
if replaced {
|
||||
verb = "replaced"
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
|
||||
n++
|
||||
}
|
||||
if n == 0 {
|
||||
return errors.New("no files in archive")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/vault/internal/pqcrypt"
|
||||
"github.com/andydixon/vault/internal/term"
|
||||
)
|
||||
|
||||
// The agent remembers an unlocked key for a while, like ssh-agent, so
|
||||
// commands skip the passphrase. `vault unlock` checks the passphrase, then
|
||||
// starts a copy of itself (agentCmd) and hands it the suite ID and master
|
||||
// seed over a pipe; it never touches the disk. The agent serves them on a
|
||||
// Unix socket in a directory only this user can enter, keeps them in locked,
|
||||
// undumpable memory where the platform allows, and wipes them and exits at
|
||||
// a deadline, or on `vault lock`. The deadline is fixed, or with --idle it
|
||||
// moves on each time a command uses the key.
|
||||
|
||||
const (
|
||||
agentCmd = "__agent" // hidden: the agent process itself
|
||||
agentGet = 'g'
|
||||
agentQuit = 'q'
|
||||
agentReply = 2 + seedSize
|
||||
agentTimeout = 2 * time.Second
|
||||
)
|
||||
|
||||
// agentSocket is the socket of the agent for the key file at keyPath, one
|
||||
// per key file, in a private per-user directory it creates if needed.
|
||||
func agentSocket(keyPath string) (string, error) {
|
||||
abs, err := filepath.Abs(keyPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
base := os.Getenv("XDG_RUNTIME_DIR")
|
||||
if base == "" {
|
||||
base = os.TempDir()
|
||||
}
|
||||
dir := filepath.Join(base, fmt.Sprintf("vault-%d", os.Getuid()))
|
||||
if err := os.Mkdir(dir, 0o700); err != nil && !os.IsExist(err) {
|
||||
return "", err
|
||||
}
|
||||
if err := privateDir(dir); err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256([]byte(abs))
|
||||
return filepath.Join(dir, "agent-"+hex.EncodeToString(sum[:8])), nil
|
||||
}
|
||||
|
||||
// agentDial connects to the agent for keyPath and sends it op.
|
||||
func agentDial(keyPath string, op byte) (net.Conn, error) {
|
||||
sock, err := agentSocket(keyPath)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c, err := net.DialTimeout("unix", sock, agentTimeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.SetDeadline(time.Now().Add(agentTimeout))
|
||||
if _, err := c.Write([]byte{op}); err != nil {
|
||||
c.Close()
|
||||
return nil, err
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// agentIdentity asks a running agent for the key at keyPath.
|
||||
func agentIdentity(keyPath string) (*pqcrypt.PQIdentity, error) {
|
||||
c, err := agentDial(keyPath, agentGet)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer c.Close()
|
||||
buf := make([]byte, agentReply)
|
||||
defer clear(buf)
|
||||
if _, err := io.ReadFull(c, buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pqcrypt.NewPQIdentityFromSeed(binary.BigEndian.Uint16(buf), buf[2:])
|
||||
}
|
||||
|
||||
// agentStop tells the agent for keyPath to forget its key, reporting
|
||||
// whether one was running.
|
||||
func agentStop(keyPath string) bool {
|
||||
c, err := agentDial(keyPath, agentQuit)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
defer c.Close()
|
||||
io.Copy(io.Discard, c) // returns once the agent has wiped the key
|
||||
return true
|
||||
}
|
||||
|
||||
// agentStart replaces any agent for keyPath with one holding id until ttl
|
||||
// has passed, since it started or, with idle, since the key was last used.
|
||||
func agentStart(keyPath string, id *pqcrypt.PQIdentity, ttl time.Duration, idle bool) error {
|
||||
sock, err := agentSocket(keyPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
agentStop(keyPath)
|
||||
self, err := os.Executable()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
mode := "fixed"
|
||||
if idle {
|
||||
mode = "idle"
|
||||
}
|
||||
cmd := exec.Command(self, agentCmd, sock, ttl.String(), mode)
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := cmd.Start(); err != nil {
|
||||
return err
|
||||
}
|
||||
secret := binary.BigEndian.AppendUint16(make([]byte, 0, agentReply), id.SuiteID())
|
||||
seed := id.Seed()
|
||||
secret = append(secret, seed...)
|
||||
clear(seed)
|
||||
_, err = stdin.Write(secret)
|
||||
clear(secret)
|
||||
stdin.Close()
|
||||
// The agent closes its standard output once listening, after saying
|
||||
// "ok" or why not.
|
||||
msg, _ := io.ReadAll(stdout)
|
||||
if err != nil || string(msg) != "ok" {
|
||||
cmd.Process.Kill()
|
||||
cmd.Wait()
|
||||
if err == nil {
|
||||
err = fmt.Errorf("agent did not start: %s", strings.TrimSpace(string(msg)))
|
||||
}
|
||||
return err
|
||||
}
|
||||
return cmd.Process.Release()
|
||||
}
|
||||
|
||||
// runAgent is the agent process: it reads the key from in, says "ok" or
|
||||
// why not on out and closes it, then serves the key on sock until ttl has
|
||||
// passed, since it started or with mode "idle" since the key was last
|
||||
// handed out, or until it is told to quit.
|
||||
func runAgent(sock, ttlArg, mode string, in io.Reader, out io.WriteCloser) {
|
||||
report := func(msg string) { io.WriteString(out, msg); out.Close() }
|
||||
ttl, err := time.ParseDuration(ttlArg)
|
||||
if err != nil || ttl <= 0 || mode != "fixed" && mode != "idle" {
|
||||
report("bad duration or mode")
|
||||
return
|
||||
}
|
||||
secret := make([]byte, agentReply)
|
||||
protectMemory(secret)
|
||||
if _, err := io.ReadFull(in, secret); err != nil {
|
||||
report("no key received")
|
||||
return
|
||||
}
|
||||
// A socket left by an agent that died stops Listen; agentStart has
|
||||
// already stopped any live one.
|
||||
os.Remove(sock)
|
||||
l, err := net.Listen("unix", sock)
|
||||
if err != nil {
|
||||
clear(secret)
|
||||
report(err.Error())
|
||||
return
|
||||
}
|
||||
os.Chmod(sock, 0o600)
|
||||
report("ok")
|
||||
expiry := time.AfterFunc(ttl, func() { l.Close() })
|
||||
for {
|
||||
c, err := l.Accept()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
op := serveAgent(c, secret)
|
||||
c.Close()
|
||||
if op == agentQuit {
|
||||
break
|
||||
}
|
||||
if op == agentGet && mode == "idle" {
|
||||
expiry.Reset(ttl)
|
||||
}
|
||||
}
|
||||
l.Close()
|
||||
clear(secret)
|
||||
os.Remove(sock)
|
||||
}
|
||||
|
||||
// serveAgent answers one request and returns its operation.
|
||||
func serveAgent(c net.Conn, secret []byte) byte {
|
||||
c.SetDeadline(time.Now().Add(agentTimeout))
|
||||
op := make([]byte, 1)
|
||||
if _, err := io.ReadFull(c, op); err != nil {
|
||||
return 0
|
||||
}
|
||||
switch op[0] {
|
||||
case agentGet:
|
||||
c.Write(secret)
|
||||
case agentQuit:
|
||||
clear(secret)
|
||||
}
|
||||
return op[0]
|
||||
}
|
||||
|
||||
// keyIdentity unlocks the key at keyPath: from the agent if one is running
|
||||
// and no passphrase is in the environment, else with the passphrase.
|
||||
func keyIdentity(keyPath string) (*pqcrypt.PQIdentity, error) {
|
||||
if _, ok := os.LookupEnv(passphraseEnv); !ok {
|
||||
if id, err := agentIdentity(keyPath); err == nil {
|
||||
return id, nil
|
||||
}
|
||||
}
|
||||
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return unlockKeyFile(keyPath, pass)
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import "golang.org/x/sys/unix"
|
||||
|
||||
// Not dumpable also means not ptrace-able by the same user.
|
||||
func init() {
|
||||
undumpable = func() { unix.Prctl(unix.PR_SET_DUMPABLE, 0, 0, 0, 0) }
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//go:build !unix
|
||||
|
||||
package main
|
||||
|
||||
// ponytail: no memory locking off unix; VirtualLock on Windows if wanted.
|
||||
func protectMemory([]byte) {}
|
||||
|
||||
// privateDir trusts the per-user temporary directory; Windows keeps it
|
||||
// private with its ACL.
|
||||
func privateDir(string) error { return nil }
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//go:build unix
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"syscall"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// undumpable stops other processes of this user reading the agent's memory,
|
||||
// where the platform can.
|
||||
var undumpable = func() {}
|
||||
|
||||
// protectMemory keeps b out of swap and the agent out of core dumps.
|
||||
func protectMemory(b []byte) {
|
||||
unix.Mlock(b)
|
||||
unix.Setrlimit(unix.RLIMIT_CORE, &unix.Rlimit{})
|
||||
undumpable()
|
||||
}
|
||||
|
||||
// privateDir checks dir is a real directory that only this user can use,
|
||||
// so nobody else can reach, swap or plant the agent's socket.
|
||||
func privateDir(dir string) error {
|
||||
fi, err := os.Lstat(dir)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
st, ok := fi.Sys().(*syscall.Stat_t)
|
||||
if !fi.IsDir() || !ok || int(st.Uid) != os.Getuid() || fi.Mode().Perm() != 0o700 {
|
||||
return fmt.Errorf("%s is not a private directory of this user", dir)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkOracleDerive(b *testing.B) {
|
||||
seed := pattern(32, 1)
|
||||
for b.Loop() {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, seed)
|
||||
id.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOracleWrap(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
r := id.Recipient()
|
||||
fk := pattern(32, 2)
|
||||
for b.Loop() {
|
||||
r.Wrap(fk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOracleUnwrap(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
st, _ := id.Recipient().Wrap(pattern(32, 2))
|
||||
for b.Loop() {
|
||||
if _, err := id.Unwrap(st); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Whole-file encryption and decryption of 4 MiB: one vault object.
|
||||
func BenchmarkOracleObject4MiB(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
pt := pattern(4<<20, 3)
|
||||
var ct bytes.Buffer
|
||||
b.SetBytes(int64(len(pt)))
|
||||
for b.Loop() {
|
||||
ct.Reset()
|
||||
w, _ := Encrypt(&ct, id.Recipient())
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
r, err := Decrypt(bytes.NewReader(ct.Bytes()), id)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
io.Copy(io.Discard, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only: decrypts files another implementation wrote, for the Rust
|
||||
// port's interoperability tests. Skips unless VAULT_ORACLE_DECRYPT is set.
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOracleDecrypt opens every DIR/pattern-SIZE.fury with the identity of
|
||||
// master seed pattern(32, 1) and checks it holds pattern(SIZE, 0x5a), and
|
||||
// that every DIR/bad-*.fury is rejected.
|
||||
func TestOracleDecrypt(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_DECRYPT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_DECRYPT")
|
||||
}
|
||||
id, err := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, _ := filepath.Glob(filepath.Join(dir, "*.fury"))
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no files")
|
||||
}
|
||||
for _, f := range files {
|
||||
data, _ := os.ReadFile(f)
|
||||
r, err := Decrypt(bytes.NewReader(data), id)
|
||||
var got []byte
|
||||
if err == nil {
|
||||
got, err = io.ReadAll(r)
|
||||
}
|
||||
base := filepath.Base(f)
|
||||
if strings.HasPrefix(base, "bad-") {
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", base)
|
||||
}
|
||||
continue
|
||||
}
|
||||
size, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(base, "pattern-"), ".fury"))
|
||||
if err != nil || !bytes.Equal(got, pattern(size, 0x5a)) {
|
||||
t.Errorf("%s: %v", base, err)
|
||||
}
|
||||
t.Logf("%s ok", base)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port. It adds nothing to the
|
||||
// production code: it only calls the frozen implementation and records what
|
||||
// it computes. Run through oracle/gen-vectors.sh.
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/vault/internal/format"
|
||||
"github.com/cloudflare/circl/kem"
|
||||
"github.com/cloudflare/circl/kem/frodo/frodo640shake"
|
||||
"github.com/cloudflare/circl/kem/mlkem/mlkem768"
|
||||
"golang.org/x/crypto/hkdf"
|
||||
)
|
||||
|
||||
type kemVector struct {
|
||||
Seed string `json:"seed"`
|
||||
PubSHA256 string `json:"pub_sha256"`
|
||||
PrivSHA256 string `json:"priv_sha256"`
|
||||
EncapSeed string `json:"encap_seed"`
|
||||
CtSHA256 string `json:"ct_sha256"`
|
||||
SharedSecret string `json:"ss"`
|
||||
RejectFlipIdx int `json:"reject_flip_index"`
|
||||
RejectSS string `json:"reject_ss"`
|
||||
}
|
||||
|
||||
type hybridVector struct {
|
||||
MasterSeed string `json:"master_seed"`
|
||||
FrodoSeed string `json:"frodo_seed"`
|
||||
MLKEMSeed string `json:"mlkem_seed"`
|
||||
PubSHA256 string `json:"pub_sha256"`
|
||||
PrivSHA256 string `json:"priv_sha256"`
|
||||
PubPrefix string `json:"pub_prefix"`
|
||||
}
|
||||
|
||||
type wrapVector struct {
|
||||
MasterSeed string `json:"master_seed"`
|
||||
FrodoEncap string `json:"frodo_encap_seed"`
|
||||
MLKEMEncap string `json:"mlkem_encap_seed"`
|
||||
FileKey string `json:"file_key"`
|
||||
KEK string `json:"kek"`
|
||||
Header string `json:"header_hex"` // full serialised header, MAC line included
|
||||
MAC string `json:"mac"`
|
||||
Nonce string `json:"nonce"`
|
||||
PayloadKey string `json:"payload_key"`
|
||||
StanzaArg0 string `json:"stanza_arg0_sha256"`
|
||||
StanzaBody string `json:"stanza_body"`
|
||||
}
|
||||
|
||||
type headerCase struct {
|
||||
Input string `json:"input_hex"`
|
||||
Accept bool `json:"accept"`
|
||||
Rest string `json:"rest_hex,omitempty"` // payload bytes after the header
|
||||
}
|
||||
|
||||
func sum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
|
||||
func pattern(n int, salt byte) []byte {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(i*7) ^ salt
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func kemVectors(t *testing.T, s kem.Scheme, n int) []kemVector {
|
||||
var out []kemVector
|
||||
for i := range n {
|
||||
seed := pattern(s.SeedSize(), byte(i*31))
|
||||
pk, sk := s.DeriveKeyPair(seed)
|
||||
pkb, _ := pk.MarshalBinary()
|
||||
skb, _ := sk.MarshalBinary()
|
||||
es := pattern(s.EncapsulationSeedSize(), byte(i*17+5))
|
||||
ct, ss, err := s.EncapsulateDeterministically(pk, es)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ss2, err := s.Decapsulate(sk, ct)
|
||||
if err != nil || !bytes.Equal(ss, ss2) {
|
||||
t.Fatal("decapsulation mismatch")
|
||||
}
|
||||
flip := (i * 997) % len(ct)
|
||||
bad := append([]byte(nil), ct...)
|
||||
bad[flip] ^= 0x01
|
||||
rss, err := s.Decapsulate(sk, bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, kemVector{
|
||||
Seed: hex.EncodeToString(seed), PubSHA256: sum(pkb), PrivSHA256: sum(skb),
|
||||
EncapSeed: hex.EncodeToString(es), CtSHA256: sum(ct), SharedSecret: hex.EncodeToString(ss),
|
||||
RejectFlipIdx: flip, RejectSS: hex.EncodeToString(rss),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
suite := frodoMLKEM768{}
|
||||
frodo, ml := frodo640shake.Scheme(), mlkem768.Scheme()
|
||||
|
||||
var hybrids []hybridVector
|
||||
for i := range 6 {
|
||||
master := pattern(32, byte(i*53+1))
|
||||
if i == 5 {
|
||||
master = bytes.Repeat([]byte{0xff}, 32)
|
||||
}
|
||||
r := hkdf.New(sha256.New, master, nil, []byte("fury/v1 seed-expand "+suite.tag()))
|
||||
fs := make([]byte, frodo.SeedSize())
|
||||
ms := make([]byte, ml.SeedSize())
|
||||
io.ReadFull(r, fs)
|
||||
io.ReadFull(r, ms)
|
||||
pub, priv, err := suite.deriveKeypair(master)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hybrids = append(hybrids, hybridVector{
|
||||
MasterSeed: hex.EncodeToString(master), FrodoSeed: hex.EncodeToString(fs), MLKEMSeed: hex.EncodeToString(ms),
|
||||
PubSHA256: sum(pub), PrivSHA256: sum(priv), PubPrefix: hex.EncodeToString(pub[:64]),
|
||||
})
|
||||
}
|
||||
|
||||
// A wrap with controlled encapsulation randomness, built from the real
|
||||
// deriveKEK, aeadEncrypt, header marshalling, headerMAC and streamKey.
|
||||
var wraps []wrapVector
|
||||
for i := range 3 {
|
||||
master := pattern(32, byte(i*53+1))
|
||||
id, err := NewPQIdentityFromSeed(suite.id(), master)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := id.Recipient().Bytes()
|
||||
fpk, _ := frodo.UnmarshalBinaryPublicKey(pub[:frodo.PublicKeySize()])
|
||||
mpk, _ := ml.UnmarshalBinaryPublicKey(pub[frodo.PublicKeySize():])
|
||||
fe, me := pattern(16, byte(i+100)), pattern(32, byte(i+150))
|
||||
ct1, ss1, _ := frodo.EncapsulateDeterministically(fpk, fe)
|
||||
ct2, ss2, _ := ml.EncapsulateDeterministically(mpk, me)
|
||||
kek := suite.deriveKEK(ss1, ss2, ct1, ct2)
|
||||
fileKey := pattern(32, byte(i+200))
|
||||
body, _ := aeadEncrypt(kek, fileKey)
|
||||
st := &format.Stanza{Type: suite.tag(), Args: []string{format.EncodeToString(ct1), format.EncodeToString(ct2)}, Body: body}
|
||||
hdr := &format.Header{Recipients: []*format.Stanza{st}}
|
||||
mac, err := headerMAC(fileKey, hdr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hdr.MAC = mac
|
||||
var hb bytes.Buffer
|
||||
hdr.Marshal(&hb)
|
||||
// The real identity must open what was built.
|
||||
got, err := id.Unwrap([]*Stanza{(*Stanza)(st)})
|
||||
if err != nil || !bytes.Equal(got, fileKey) {
|
||||
t.Fatal("identity did not unwrap the vector")
|
||||
}
|
||||
nonce := pattern(16, byte(i+250))
|
||||
wraps = append(wraps, wrapVector{
|
||||
MasterSeed: hex.EncodeToString(master), FrodoEncap: hex.EncodeToString(fe), MLKEMEncap: hex.EncodeToString(me),
|
||||
FileKey: hex.EncodeToString(fileKey), KEK: hex.EncodeToString(kek), Header: hex.EncodeToString(hb.Bytes()),
|
||||
MAC: hex.EncodeToString(mac), Nonce: hex.EncodeToString(nonce), PayloadKey: hex.EncodeToString(streamKey(fileKey, nonce)),
|
||||
StanzaArg0: sum([]byte(st.Args[0])), StanzaBody: hex.EncodeToString(body),
|
||||
})
|
||||
}
|
||||
|
||||
// Whole files from the real Encrypt, with fresh randomness, for the
|
||||
// first hybrid identity. Plaintexts are pattern(size, 0x5a).
|
||||
id, _ := NewPQIdentityFromSeed(suite.id(), pattern(32, 1))
|
||||
fdir := filepath.Join(dir, "fury")
|
||||
os.MkdirAll(fdir, 0o755)
|
||||
for _, size := range []int{0, 1, 65535, 65536, 65537, 131072, 200000} {
|
||||
var buf bytes.Buffer
|
||||
w, err := Encrypt(&buf, id.Recipient())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(pattern(size, 0x5a))
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(fdir, "pattern-"+itoa(size)+".fury"), buf.Bytes(), 0o644)
|
||||
}
|
||||
|
||||
// Header parser verdicts on valid and malformed headers.
|
||||
var cases []headerCase
|
||||
valid, _ := hex.DecodeString(wraps[0].Header)
|
||||
add := func(b []byte) {
|
||||
h, payload, err := format.Parse(bytes.NewReader(b))
|
||||
c := headerCase{Input: hex.EncodeToString(b), Accept: err == nil}
|
||||
if err == nil {
|
||||
rest, _ := io.ReadAll(payload)
|
||||
c.Rest = hex.EncodeToString(rest)
|
||||
var again bytes.Buffer
|
||||
h.Marshal(&again)
|
||||
}
|
||||
cases = append(cases, c)
|
||||
}
|
||||
for _, m := range headerMutations(valid) {
|
||||
add(m)
|
||||
}
|
||||
|
||||
write(t, filepath.Join(dir, "crypto.json"), map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, internal/pqcrypt/oracle_vectors_test.go",
|
||||
"frodo640shake": kemVectors(t, frodo, 8),
|
||||
"mlkem768": kemVectors(t, ml, 8),
|
||||
"hybrid": hybrids,
|
||||
"wrap": wraps,
|
||||
"header_cases": cases,
|
||||
})
|
||||
}
|
||||
|
||||
func itoa(n int) string { return string(appendInt(nil, n)) }
|
||||
|
||||
func appendInt(b []byte, n int) []byte {
|
||||
if n >= 10 {
|
||||
b = appendInt(b, n/10)
|
||||
}
|
||||
return append(b, byte('0'+n%10))
|
||||
}
|
||||
|
||||
// headerMutations returns valid headers plus a range of malformed ones.
|
||||
func headerMutations(valid []byte) [][]byte {
|
||||
s := string(valid)
|
||||
introEnd := bytes.IndexByte(valid, '\n') + 1
|
||||
footer := bytes.LastIndex(valid, []byte("\n---")) + 1
|
||||
lines := bytes.SplitAfter(valid, []byte("\n"))
|
||||
join := func(ls [][]byte) []byte { return bytes.Join(ls, nil) }
|
||||
var out [][]byte
|
||||
push := func(b []byte) { out = append(out, append([]byte(nil), b...)) }
|
||||
push(valid)
|
||||
push(append(append([]byte(nil), valid...), []byte("payload\x00\x01")...))
|
||||
push([]byte(""))
|
||||
push([]byte("fury-encryption.org/v1\n--- " + s[len(s)-44:])) // no stanzas
|
||||
push([]byte("fury-encryption.org/v2\n" + s[introEnd:])) // wrong intro
|
||||
push([]byte("fury-encryption.org/v1" + s[introEnd:])) // intro without newline
|
||||
push(valid[:footer]) // no footer
|
||||
push(valid[:len(valid)-1]) // footer without newline
|
||||
push(append(append([]byte(nil), valid[:len(valid)-1]...), '=', '\n')) // padded MAC
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:])) // double space
|
||||
push([]byte(s[:footer] + "---x " + s[footer+4:])) // bad footer prefix
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-2] + "\n")) // short MAC
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-1] + " x\n"))
|
||||
// Stanza line edits.
|
||||
stanza := lines[1]
|
||||
for _, edit := range []string{"-> \n", "->\n", "-> x\n", "-> x\x7f\n", "-> x é\n", "-x a\n", "-> x y\n", "-> x\r\n"} {
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls[1] = []byte(edit)
|
||||
push(join(ls))
|
||||
_ = stanza
|
||||
}
|
||||
// Body edits: the body is the two lines after the stanza line.
|
||||
body := 2
|
||||
for _, edit := range []string{"\n", "AAAA\n", "AAAA=\n", "A\n", "AB\n", "AAB\n", "AA==\n", " AAAA\n", "AAAA\r\n",
|
||||
string(bytes.Repeat([]byte("A"), 64)) + "\n", string(bytes.Repeat([]byte("A"), 68)) + "\n", string(bytes.Repeat([]byte("A"), 65)) + "\n"} {
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls[body] = []byte(edit)
|
||||
push(join(ls))
|
||||
}
|
||||
{
|
||||
// Drop the empty line that ends a 48-byte body: the stanza then runs
|
||||
// into the footer.
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls = append(ls[:3], ls[4:]...)
|
||||
push(join(ls))
|
||||
}
|
||||
{
|
||||
// A second, foreign stanza ahead of the real one.
|
||||
ls := append([][]byte(nil), lines[:1]...)
|
||||
ls = append(ls, []byte("-> other-suite arg\n"), []byte("AAAA\n"))
|
||||
ls = append(ls, lines[1:]...)
|
||||
push(join(ls))
|
||||
}
|
||||
// The 1 MiB stanza and 16 MiB header limits are exercised by the Rust
|
||||
// tests directly; vectors that size would bloat the repository.
|
||||
return out
|
||||
}
|
||||
|
||||
func write(t *testing.T, path string, v any) {
|
||||
b, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package stream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// STREAM seal and open of 64 MiB, one goroutine (the sequential path).
|
||||
func BenchmarkOracleStream64MiB(b *testing.B) {
|
||||
key := make([]byte, 32)
|
||||
pt := make([]byte, 64<<20)
|
||||
var ct bytes.Buffer
|
||||
b.SetBytes(int64(len(pt)))
|
||||
for b.Loop() {
|
||||
ct.Reset()
|
||||
w, _ := NewEncryptWriter(key, &ct)
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
r, _ := NewDecryptReader(key, bytes.NewReader(ct.Bytes()))
|
||||
io.Copy(io.Discard, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port; see oracle/gen-vectors.sh.
|
||||
|
||||
package stream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
key := make([]byte, 32)
|
||||
for i := range key {
|
||||
key[i] = byte(i + 1)
|
||||
}
|
||||
type vec struct {
|
||||
Size int `json:"size"`
|
||||
CtLen int `json:"ct_len"`
|
||||
CtSHA256 string `json:"ct_sha256"`
|
||||
CtHex string `json:"ct_hex,omitempty"`
|
||||
}
|
||||
var vs []vec
|
||||
for _, size := range []int{0, 1, 15, 16, 17, ChunkSize - 1, ChunkSize, ChunkSize + 1, 2 * ChunkSize, 2*ChunkSize + 1, 3*ChunkSize - 16, 300000} {
|
||||
pt := make([]byte, size)
|
||||
for i := range pt {
|
||||
pt[i] = byte(i*13 + 7)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, _ := NewEncryptWriter(key, &buf)
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
// The reader-side encryptor must agree.
|
||||
r, _ := NewEncryptReader(key, bytes.NewReader(pt))
|
||||
alt, _ := io.ReadAll(r)
|
||||
if !bytes.Equal(alt, buf.Bytes()) {
|
||||
t.Fatal("EncryptReader disagrees with EncryptWriter")
|
||||
}
|
||||
s := sha256.Sum256(buf.Bytes())
|
||||
v := vec{Size: size, CtLen: buf.Len(), CtSHA256: hex.EncodeToString(s[:])}
|
||||
if size <= 17 {
|
||||
v.CtHex = hex.EncodeToString(buf.Bytes())
|
||||
}
|
||||
vs = append(vs, v)
|
||||
}
|
||||
b, _ := json.MarshalIndent(map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, internal/stream/oracle_vectors_test.go",
|
||||
"key": hex.EncodeToString(key),
|
||||
"plaintext": "byte(i*13+7)",
|
||||
"vectors": vs,
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "stream.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
@@ -17,6 +18,7 @@ import (
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/vault/internal/term"
|
||||
)
|
||||
@@ -35,16 +37,37 @@ const usage = `Usage:
|
||||
vault mirror URL [FOLDER] crawl a website (same host only, one request
|
||||
at a time) into FOLDER/HOST, rewriting its
|
||||
links to relative paths
|
||||
vault list [PREFIX] list stored files
|
||||
vault absorb [FOLDER] read a tar archive (optionally gzipped) from
|
||||
standard input and add every file in it,
|
||||
optionally under a folder inside the vault
|
||||
vault ls [FOLDER] list one folder: its files, and its
|
||||
sub-folders (ending in /) with their
|
||||
total size; -r lists everything under it
|
||||
vault list [FOLDER] list every stored file under FOLDER,
|
||||
with full names
|
||||
vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name)
|
||||
vault mv OLD NEW rename a stored file
|
||||
vault get -r FOLDER [-o DIR] decrypt a folder and everything under it
|
||||
into DIR (default: the folder's base name)
|
||||
vault mv OLD NEW rename or move a file or folder; NEW
|
||||
ending in / or naming an existing folder
|
||||
moves OLD into it
|
||||
vault cp [-r] OLD NEW copy a file, or with -r a folder, like mv;
|
||||
copies share their encrypted data
|
||||
vault rm NAME delete a stored file
|
||||
vault rm -r FOLDER [-f] delete a folder and everything under it;
|
||||
asks first unless -f (or -rf) is given
|
||||
vault unlock [--for DURATION] remember the passphrase for DURATION
|
||||
[--idle] (default 5m), so commands skip asking;
|
||||
with --idle, DURATION since last used
|
||||
vault lock forget it now
|
||||
|
||||
Options (any command):
|
||||
--key PATH key file (default $VAULT_KEY, else ~/.vault/key)
|
||||
--vault DIR vault directory (default $VAULT_DIR, else ~/.vault/store)
|
||||
|
||||
The passphrase is read from the terminal, or from $VAULT_PASSPHRASE.
|
||||
The passphrase is read from the terminal, or from $VAULT_PASSPHRASE. After
|
||||
vault unlock, commands use the remembered key instead, unless
|
||||
$VAULT_PASSPHRASE is set.
|
||||
|
||||
Decoy passphrase: genkey offers a second, optional passphrase
|
||||
($VAULT_DECOY_PASSPHRASE when scripted). It unlocks a separate key, so
|
||||
@@ -69,6 +92,10 @@ func main() {
|
||||
os.Exit(2)
|
||||
}
|
||||
cmd, args := os.Args[1], os.Args[2:]
|
||||
if cmd == agentCmd && len(args) == 3 {
|
||||
runAgent(args[0], args[1], args[2], os.Stdin, os.Stdout)
|
||||
return
|
||||
}
|
||||
fset := flag.NewFlagSet("vault "+cmd, flag.ContinueOnError)
|
||||
fset.SetOutput(io.Discard)
|
||||
keyPath := fset.String("key", defaultPath("VAULT_KEY", "key"), "")
|
||||
@@ -76,6 +103,12 @@ func main() {
|
||||
output := fset.String("o", "", "")
|
||||
filler := fset.String("filler", "0", "")
|
||||
yes := fset.Bool("yes", false, "")
|
||||
ttl := fset.Duration("for", 5*time.Minute, "")
|
||||
idle := fset.Bool("idle", false, "")
|
||||
// rm's -r and -f; Go's flag package has no combined short flags, so
|
||||
// -rf and -fr are spelled out.
|
||||
recurse, force := fset.Bool("r", false, ""), fset.Bool("f", false, "")
|
||||
rf, fr := fset.Bool("rf", false, ""), fset.Bool("fr", false, "")
|
||||
pos, err := parseInterleaved(fset, args)
|
||||
if err != nil {
|
||||
fail("%v\n\n%s", err, usage)
|
||||
@@ -98,6 +131,7 @@ func main() {
|
||||
fail("decoy passphrase must differ from the passphrase")
|
||||
}
|
||||
check(writeKeyFile(*keyPath, pass, decoy))
|
||||
agentStop(*keyPath) // one left from an old key at this path would serve it
|
||||
fmt.Fprintf(os.Stderr, "key written to %s\n", *keyPath)
|
||||
case "init":
|
||||
want(0, 0)
|
||||
@@ -146,6 +180,26 @@ func main() {
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
check(mirror(s, cat, pos[0], dest))
|
||||
case "absorb":
|
||||
want(0, 1)
|
||||
if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
|
||||
fail("absorb reads a tar archive from standard input; pipe one in")
|
||||
}
|
||||
dest := ""
|
||||
if len(pos) == 1 {
|
||||
dest = strings.Trim(pos[0], "/")
|
||||
}
|
||||
// Wait for the archive to start arriving before asking for the
|
||||
// passphrase, so a program feeding the pipe (gpg, ssh) has finished
|
||||
// with the terminal and cannot bury the prompt.
|
||||
in := bufio.NewReader(os.Stdin)
|
||||
if _, err := in.Peek(1); err != nil {
|
||||
fail("nothing to absorb on standard input")
|
||||
}
|
||||
s := open(*keyPath, *vaultDir, true)
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
check(absorb(s, cat, in, dest))
|
||||
case "list", "ls":
|
||||
want(0, 1)
|
||||
prefix := ""
|
||||
@@ -154,20 +208,110 @@ func main() {
|
||||
}
|
||||
cat, err := open(*keyPath, *vaultDir, false).catalogue()
|
||||
check(err)
|
||||
if cmd == "ls" && !*recurse {
|
||||
check(ls(os.Stdout, cat, prefix))
|
||||
break
|
||||
}
|
||||
list(cat, prefix)
|
||||
case "get":
|
||||
want(1, 1)
|
||||
out := *output
|
||||
if out == "" {
|
||||
out = path.Base(pos[0])
|
||||
out = path.Base(strings.TrimRight(pos[0], "/"))
|
||||
}
|
||||
check(open(*keyPath, *vaultDir, false).get(pos[0], out))
|
||||
case "mv", "rename":
|
||||
s := open(*keyPath, *vaultDir, false)
|
||||
if !*recurse {
|
||||
check(s.get(pos[0], out))
|
||||
break
|
||||
}
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
folder := strings.Trim(pos[0], "/")
|
||||
names := under(cat, folder)
|
||||
if folder == "" || len(names) == 0 || len(cat[folder]) > 0 {
|
||||
fail("%s: not a folder in the vault", pos[0])
|
||||
}
|
||||
for _, n := range names {
|
||||
dst := filepath.Join(out, filepath.FromSlash(n[len(folder)+1:]))
|
||||
check(os.MkdirAll(filepath.Dir(dst), 0o700))
|
||||
check(s.getRecord(cat[n][0], dst))
|
||||
fmt.Fprintf(os.Stderr, "extracted %s\n", dst)
|
||||
}
|
||||
case "mv", "rename", "cp":
|
||||
want(2, 2)
|
||||
check(open(*keyPath, *vaultDir, true).rename(pos[0], pos[1]))
|
||||
s := open(*keyPath, *vaultDir, true)
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
pairs, err := transfers(cat, pos[0], pos[1], cmd != "cp" || *recurse)
|
||||
check(err)
|
||||
verb := "copied"
|
||||
if cmd != "cp" {
|
||||
verb = "moved"
|
||||
}
|
||||
check(s.copyAll(cat, pairs, cmd != "cp", func(from, to string) {
|
||||
fmt.Fprintf(os.Stderr, "%s %s -> %s\n", verb, from, to)
|
||||
}))
|
||||
case "rm", "delete":
|
||||
want(1, 1)
|
||||
check(open(*keyPath, *vaultDir, true).delete(pos[0]))
|
||||
recursive, forced := *recurse || *rf || *fr, *force || *rf || *fr
|
||||
name := pos[0]
|
||||
if recursive {
|
||||
name = strings.Trim(name, "/")
|
||||
if name == "" {
|
||||
fail("refusing to remove the whole vault")
|
||||
}
|
||||
}
|
||||
s := open(*keyPath, *vaultDir, true)
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
var names []string
|
||||
if recursive {
|
||||
names = under(cat, name)
|
||||
} else if len(cat[name]) > 0 {
|
||||
names = []string{name}
|
||||
}
|
||||
if len(names) == 0 {
|
||||
if forced {
|
||||
break
|
||||
}
|
||||
fail("%s: not in the vault", name)
|
||||
}
|
||||
var size int64
|
||||
for _, n := range names {
|
||||
size += cat[n][0].Size
|
||||
}
|
||||
if recursive && !forced && !confirm("these files will be deleted, with no undo:", names,
|
||||
fmt.Sprintf("delete %d file(s), %s? [y/N]", len(names), humanSize(size))) {
|
||||
fail("rm cancelled; nothing changed (-f skips the question)")
|
||||
}
|
||||
check(s.removeAll(cat, names, func(n string) { fmt.Fprintf(os.Stderr, "removed %s\n", n) }))
|
||||
if recursive {
|
||||
fmt.Fprintf(os.Stderr, "removed %d file(s), %s\n", len(names), humanSize(size))
|
||||
}
|
||||
case "unlock":
|
||||
want(0, 0)
|
||||
if *ttl <= 0 {
|
||||
fail("--for needs a positive duration, such as 5m or 1h")
|
||||
}
|
||||
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
|
||||
check(err)
|
||||
id, err := unlockKeyFile(*keyPath, pass)
|
||||
check(err)
|
||||
check(agentStart(*keyPath, id, *ttl, *idle))
|
||||
id.Close()
|
||||
if *idle {
|
||||
fmt.Fprintf(os.Stderr, "unlocked until unused for %s; vault lock forgets the key sooner\n", *ttl)
|
||||
} else {
|
||||
fmt.Fprintf(os.Stderr, "unlocked until %s; vault lock forgets the key sooner\n",
|
||||
time.Now().Add(*ttl).Format("15:04:05"))
|
||||
}
|
||||
case "lock":
|
||||
want(0, 0)
|
||||
if agentStop(*keyPath) {
|
||||
fmt.Fprintln(os.Stderr, "locked")
|
||||
} else {
|
||||
fmt.Fprintln(os.Stderr, "nothing was unlocked")
|
||||
}
|
||||
case "help", "-h", "--help":
|
||||
fmt.Println(usage)
|
||||
default:
|
||||
@@ -175,6 +319,51 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// transfers works out, like mv(1) and cp(1), which stored names src and dst
|
||||
// mean: each file to copy or move, paired with its new name. src is a file,
|
||||
// or with recursive a folder. dst ending in / or naming an existing folder
|
||||
// means into it. No new name may already exist.
|
||||
func transfers(cat map[string][]*record, src, dst string, recursive bool) ([][2]string, error) {
|
||||
into := dst == "" || strings.HasSuffix(dst, "/")
|
||||
src, dst = strings.Trim(src, "/"), strings.Trim(dst, "/")
|
||||
if src == "" {
|
||||
return nil, errors.New("name a file or folder to copy or move")
|
||||
}
|
||||
if dst == "" || len(cat[dst]) == 0 && len(under(cat, dst)) > 0 {
|
||||
into = true
|
||||
}
|
||||
if into {
|
||||
dst = path.Join(dst, path.Base(src))
|
||||
}
|
||||
var pairs [][2]string
|
||||
if len(cat[src]) > 0 {
|
||||
pairs = [][2]string{{src, dst}}
|
||||
} else {
|
||||
names := under(cat, src)
|
||||
if len(names) == 0 {
|
||||
return nil, fmt.Errorf("%s: not in the vault", src)
|
||||
}
|
||||
if !recursive {
|
||||
return nil, fmt.Errorf("%s is a folder; use -r", src)
|
||||
}
|
||||
if dst == src || strings.HasPrefix(dst, src+"/") {
|
||||
return nil, fmt.Errorf("cannot put %s inside itself", src)
|
||||
}
|
||||
for _, n := range names {
|
||||
pairs = append(pairs, [2]string{n, dst + n[len(src):]})
|
||||
}
|
||||
}
|
||||
for _, p := range pairs {
|
||||
if !validName(p[1]) {
|
||||
return nil, fmt.Errorf("invalid vault path %q", p[1])
|
||||
}
|
||||
if len(cat[p[1]]) > 0 {
|
||||
return nil, fmt.Errorf("%s already exists", p[1])
|
||||
}
|
||||
}
|
||||
return pairs, nil
|
||||
}
|
||||
|
||||
// parseInterleaved lets flags appear before, between or after positional
|
||||
// arguments.
|
||||
func parseInterleaved(fset *flag.FlagSet, args []string) ([]string, error) {
|
||||
@@ -221,9 +410,7 @@ func defaultPath(env, name string) string {
|
||||
// open unlocks the key and opens the vault, holding the vault lock
|
||||
// exclusively for writers and shared for readers.
|
||||
func open(keyPath, vaultDir string, exclusive bool) *store {
|
||||
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
|
||||
check(err)
|
||||
id, err := unlockKeyFile(keyPath, pass)
|
||||
id, err := keyIdentity(keyPath)
|
||||
check(err)
|
||||
s, err := openStore(vaultDir, id)
|
||||
check(err)
|
||||
@@ -360,11 +547,22 @@ func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool,
|
||||
// confirmOverwrite lists the vault names a migrate would replace and asks
|
||||
// on the terminal whether to go ahead. No terminal means no.
|
||||
func confirmOverwrite(names []string) bool {
|
||||
fmt.Fprintln(os.Stderr, "these files are already in the vault and would be overwritten:")
|
||||
for _, name := range names {
|
||||
return confirm("these files are already in the vault and would be overwritten:", names,
|
||||
fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names)))
|
||||
}
|
||||
|
||||
// confirm prints intro and names, the first 20 of them, then asks question
|
||||
// on the terminal. No terminal means no.
|
||||
func confirm(intro string, names []string, question string) bool {
|
||||
fmt.Fprintln(os.Stderr, intro)
|
||||
for i, name := range names {
|
||||
if i == 20 {
|
||||
fmt.Fprintf(os.Stderr, " … and %d more\n", len(names)-i)
|
||||
break
|
||||
}
|
||||
fmt.Fprintf(os.Stderr, " %s\n", name)
|
||||
}
|
||||
ok, err := term.Confirm(fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names)))
|
||||
ok, err := term.Confirm(question)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err)
|
||||
}
|
||||
@@ -431,14 +629,73 @@ func addOne(s *store, cat map[string][]*record, src, name string, migrate bool)
|
||||
return nil
|
||||
}
|
||||
|
||||
func list(cat map[string][]*record, prefix string) {
|
||||
var names []string
|
||||
for name := range cat {
|
||||
if prefix == "" || name == prefix || strings.HasPrefix(name, prefix+"/") {
|
||||
// ls prints one level of the vault, like ls(1): the files directly in
|
||||
// folder, and each sub-folder once, marked with a trailing slash and shown
|
||||
// with the total size and newest date of everything under it. Folders come
|
||||
// first. A folder that is a file's name lists just that file.
|
||||
func ls(w io.Writer, cat map[string][]*record, folder string) error {
|
||||
type entry struct {
|
||||
size int64
|
||||
modified time.Time
|
||||
}
|
||||
entries := map[string]*entry{}
|
||||
for name, rs := range cat {
|
||||
rel := name
|
||||
switch {
|
||||
case folder == "":
|
||||
case name == folder:
|
||||
rel = path.Base(name)
|
||||
case strings.HasPrefix(name, folder+"/"):
|
||||
rel = name[len(folder)+1:]
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if i := strings.IndexByte(rel, '/'); i >= 0 {
|
||||
rel = rel[:i+1]
|
||||
}
|
||||
e := entries[rel]
|
||||
if e == nil {
|
||||
e = &entry{}
|
||||
entries[rel] = e
|
||||
}
|
||||
r := rs[0]
|
||||
e.size += r.Size
|
||||
if r.Modified.After(e.modified) {
|
||||
e.modified = r.Modified
|
||||
}
|
||||
}
|
||||
if len(entries) == 0 && folder != "" {
|
||||
return fmt.Errorf("%s: not in the vault", folder)
|
||||
}
|
||||
names := make([]string, 0, len(entries))
|
||||
for name := range entries {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Slice(names, func(i, j int) bool {
|
||||
di, dj := strings.HasSuffix(names[i], "/"), strings.HasSuffix(names[j], "/")
|
||||
if di != dj {
|
||||
return di
|
||||
}
|
||||
return names[i] < names[j]
|
||||
})
|
||||
for _, name := range names {
|
||||
e := entries[name]
|
||||
fmt.Fprintf(w, "%10s %s %s\n", humanSize(e.size), e.modified.Local().Format("2006-01-02 15:04"), name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// list prints every file at or under prefix, with its full name.
|
||||
func list(cat map[string][]*record, prefix string) {
|
||||
var names []string
|
||||
if prefix == "" {
|
||||
for name := range cat {
|
||||
names = append(names, name)
|
||||
}
|
||||
sort.Strings(names)
|
||||
} else {
|
||||
names = under(cat, prefix)
|
||||
}
|
||||
for _, name := range names {
|
||||
r := cat[name][0]
|
||||
fmt.Printf("%10s %s %s\n", humanSize(r.Size), r.Modified.Local().Format("2006-01-02 15:04"), name)
|
||||
@@ -0,0 +1,100 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port's URL handling and mirror
|
||||
// link mapping; see oracle/gen-vectors.sh.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOracleNetVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
type parsed struct {
|
||||
Input string `json:"input"`
|
||||
Err bool `json:"err"`
|
||||
String string `json:"string,omitempty"`
|
||||
Scheme string `json:"scheme,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
RawQuery string `json:"raw_query,omitempty"`
|
||||
Fragment string `json:"fragment,omitempty"`
|
||||
Local string `json:"local_path,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
describe := func(in string, u *url.URL, err error) parsed {
|
||||
if err != nil {
|
||||
return parsed{Input: in, Err: true}
|
||||
}
|
||||
return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path,
|
||||
RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)}
|
||||
}
|
||||
refs := []string{
|
||||
"", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080",
|
||||
"?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A",
|
||||
"https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..",
|
||||
"a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f",
|
||||
"http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only",
|
||||
}
|
||||
bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"}
|
||||
var parses []parsed
|
||||
for _, r := range refs {
|
||||
u, err := url.Parse(r)
|
||||
parses = append(parses, describe(r, u, err))
|
||||
}
|
||||
type resolved struct {
|
||||
Base string `json:"base"`
|
||||
Ref string `json:"ref"`
|
||||
Result parsed `json:"result"`
|
||||
}
|
||||
var resolves []resolved
|
||||
for _, b := range bases {
|
||||
bu, err := url.Parse(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range refs {
|
||||
u, err := bu.Parse(r)
|
||||
resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)})
|
||||
}
|
||||
}
|
||||
rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"},
|
||||
{"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}}
|
||||
var relOut [][3]string
|
||||
for _, p := range rels {
|
||||
relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])})
|
||||
}
|
||||
escapes := map[string]string{}
|
||||
for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} {
|
||||
escapes[s] = url.QueryEscape(s)
|
||||
}
|
||||
times := map[string]any{}
|
||||
for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} {
|
||||
if tm, err := http.ParseTime(s); err == nil {
|
||||
times[s] = tm.Unix()
|
||||
} else {
|
||||
times[s] = nil
|
||||
}
|
||||
}
|
||||
b, _ := json.MarshalIndent(map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go",
|
||||
"parse": parses,
|
||||
"resolve": resolves,
|
||||
"relative": relOut,
|
||||
"query_escape": escapes,
|
||||
"http_time": times,
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Unlocking a key file: Argon2id for both slots.
|
||||
func BenchmarkOracleUnlock(b *testing.B) {
|
||||
p := filepath.Join(b.TempDir(), "key")
|
||||
if err := writeKeyFile(p, "bench", ""); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for b.Loop() {
|
||||
if _, err := unlockKeyFile(p, "bench"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector and fixture generator for the Rust port. It calls the
|
||||
// frozen implementation and records what it computes; it changes nothing in
|
||||
// the vault binary. Run through oracle/gen-vectors.sh.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/vault/internal/pqcrypt"
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
// Synthetic, public test passphrases. Never use them for anything real.
|
||||
const (
|
||||
oracleReal = "oracle-real passphrase"
|
||||
oracleDecoy = "oracle-decoy passphrase"
|
||||
)
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
out := map[string]any{"provenance": "oracle/go at 4c1507c, oracle_vectors_test.go"}
|
||||
|
||||
// Key files.
|
||||
keys := map[string]any{}
|
||||
for _, k := range []struct{ name, real, decoy string }{
|
||||
{"key-with-decoy.pem", oracleReal, oracleDecoy},
|
||||
{"key-no-decoy.pem", oracleReal, ""},
|
||||
} {
|
||||
p := filepath.Join(dir, k.name)
|
||||
os.Remove(p)
|
||||
if err := writeKeyFile(p, k.real, k.decoy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry := map[string]any{}
|
||||
for label, pass := range map[string]string{"real": k.real, "decoy": k.decoy} {
|
||||
if pass == "" {
|
||||
continue
|
||||
}
|
||||
id, err := unlockKeyFile(p, pass)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry[label] = map[string]any{"passphrase": pass, "suite": id.SuiteID(), "seed": hex.EncodeToString(id.Seed())}
|
||||
}
|
||||
keys[k.name] = entry
|
||||
}
|
||||
out["keys"] = keys
|
||||
|
||||
var argon []map[string]string
|
||||
for i, pass := range []string{"", "a", "correct horse", "pässwörd ü", "x\x00y"} {
|
||||
salt := make([]byte, 16)
|
||||
for j := range salt {
|
||||
salt[j] = byte(i*16 + j)
|
||||
}
|
||||
k := argon2.IDKey([]byte(pass), salt, argonTime, argonMemory, argonThreads, 32)
|
||||
argon = append(argon, map[string]string{"passphrase_hex": hex.EncodeToString([]byte(pass)), "salt": hex.EncodeToString(salt), "key": hex.EncodeToString(k)})
|
||||
}
|
||||
out["argon2id"] = argon
|
||||
out["key_header_hex"] = hex.EncodeToString(keyHeader())
|
||||
|
||||
// Object IDs and sizes.
|
||||
seed := "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
|
||||
var ids []string
|
||||
for i := range int64(4) {
|
||||
ids = append(ids, objectID(seed, i))
|
||||
}
|
||||
out["object_ids"] = map[string]any{"seed": seed, "ids": ids, "big_index": objectID(seed, 1<<40), "big_index_value": int64(1 << 40)}
|
||||
padded := map[string]int64{}
|
||||
counts := map[string]int64{}
|
||||
for _, n := range []int64{0, 1, 16, 65535, 65536, 65537, 1 << 20, chunkSize - 1, chunkSize, chunkData - 1, chunkData, chunkData + 1, 2 * chunkData, fileLimit} {
|
||||
padded[fmt.Sprint(n)] = paddedSize(n)
|
||||
counts[fmt.Sprint(n)] = objectCount(n)
|
||||
}
|
||||
out["padded_size"] = padded
|
||||
out["object_count"] = counts
|
||||
|
||||
names := map[string]bool{}
|
||||
for _, n := range []string{"", "a", "a/b", "/a", "a/", "a//b", "./a", "a/./b", "../a", "a/..", "..", ".", "a/../b", "a\\b", "a\x00b",
|
||||
"a\nb", "a\x7fb", "a\u0085b", "ü/ñ", "\xff", "a b", " a", "...", "a/...", "..a", "a/.b", string(make([]byte, 1024)), longName(1024), longName(1025), "ab", "
"} {
|
||||
names[hex.EncodeToString([]byte(n))] = validName(n)
|
||||
}
|
||||
out["valid_name_hex"] = names
|
||||
|
||||
sizes := map[string]any{}
|
||||
for _, s := range []string{"0", "512", "2k", "500M", "2G", "1TB", "", "G", "-1", "2X", "1.5G", "2T", "+5", "1KB", "1MK", "1kb", "B", "1T", "1099511627776", "1099511627777", " 1", "0x10", "1024G", "1025G", "-0", "1bb"} {
|
||||
n, err := parseSize(s)
|
||||
if err != nil {
|
||||
sizes[s] = nil
|
||||
} else {
|
||||
sizes[s] = n
|
||||
}
|
||||
}
|
||||
out["parse_size"] = sizes
|
||||
|
||||
human := map[string]string{}
|
||||
for _, n := range []int64{0, 1, 1023, 1024, 1025, 1536, 1101, 10239, 10240, 1048575, 1048576, 1 << 30, 1<<40 - 1, 1 << 40, 1<<62 + 12345, 1<<63 - 1, 1075, 1126, 1177, 1638, 1689} {
|
||||
human[fmt.Sprint(n)] = humanSize(n)
|
||||
}
|
||||
out["human_size"] = human
|
||||
|
||||
clean := map[string]string{}
|
||||
for _, p := range []string{"", ".", "..", "/", "//", "a/b/../c", "/../a", "a/./b/", "../../a", "a//b", "./", "/a/b/..", "a/b/../../..", "abc/def/ghi/../.."} {
|
||||
clean[p] = path.Clean(p)
|
||||
}
|
||||
out["path_clean"] = clean
|
||||
|
||||
durs := map[string]any{}
|
||||
for _, d := range []string{"5m", "1h", "90s", "1.5h", "300ms", "1h30m", "0", "-5m", "", "5", "1d", "2h45m30.5s", "1us", "1µs", "1ns", ".5s", "5.s", "+5m", "9223372036854775807ns", "9223372036854775808ns"} {
|
||||
v, err := time.ParseDuration(d)
|
||||
if err != nil {
|
||||
durs[d] = nil
|
||||
} else {
|
||||
durs[d] = map[string]any{"ns": int64(v), "string": v.String()}
|
||||
}
|
||||
}
|
||||
out["durations"] = durs
|
||||
|
||||
// Record JSON, exactly as written.
|
||||
var recs []map[string]any
|
||||
for _, r := range []record{
|
||||
{Name: "plain.txt", Seed: seed, Size: 5, Mode: 0o644, Modified: time.Unix(1700000000, 123456789).UTC(), Added: time.Unix(1700000001, 0).UTC()},
|
||||
{Name: "<a&b>/
'\"\\x", Seed: seed, Size: 0, Mode: 0o755, Modified: time.Unix(0, 0).UTC(), Added: time.Unix(1, 100).UTC()},
|
||||
{Name: "ü/日本語", Seed: seed, Size: 1 << 40, Mode: 0o600, Modified: time.Unix(-1, 500000000).UTC(), Added: time.Unix(253402300799, 999999999).UTC()},
|
||||
{Name: "tab\there", Seed: seed, Size: 1, Mode: 0, Modified: time.Unix(1234567890, 120000000).UTC(), Added: time.Unix(1234567890, 1000).UTC()},
|
||||
} {
|
||||
b, err := json.Marshal(&r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recs = append(recs, map[string]any{"json": string(b), "name": r.Name, "modified_unix_nano": r.Modified.UnixNano(), "added_unix_nano": r.Added.UnixNano()})
|
||||
}
|
||||
out["records"] = recs
|
||||
|
||||
writeMiniVault(t, filepath.Join(dir, "mini-vault"), filepath.Join(dir, "key-with-decoy.pem"))
|
||||
|
||||
b, _ := json.MarshalIndent(out, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "store.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func longName(n int) string {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = 'a' + byte(i%26)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// writeMiniVault builds a small vault through the real store code, without
|
||||
// init's filler (which would make the fixture megabytes): three real files,
|
||||
// one decoy file and one record sealed to a throwaway key.
|
||||
func writeMiniVault(t *testing.T, root, keyPath string) {
|
||||
os.RemoveAll(root)
|
||||
for _, d := range []string{recordsDir, objectsDir} {
|
||||
for i := range 256 {
|
||||
if err := os.MkdirAll(filepath.Join(root, d, fmt.Sprintf("%02x", i)), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
os.WriteFile(filepath.Join(root, markerName), []byte(markerContent), 0o600)
|
||||
src := filepath.Join(t.TempDir(), "src")
|
||||
put := func(s *store, name string, body []byte, mode os.FileMode, mtime time.Time) {
|
||||
os.WriteFile(src, body, 0o600)
|
||||
os.Chmod(src, mode)
|
||||
os.Chtimes(src, mtime, mtime)
|
||||
cat, err := s.catalogue()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.add(cat, src, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
real, _ := unlockKeyFile(keyPath, oracleReal)
|
||||
decoy, _ := unlockKeyFile(keyPath, oracleDecoy)
|
||||
throwaway, _ := pqcrypt.GeneratePQIdentity()
|
||||
rs := &store{root: root, id: real}
|
||||
boundary := make([]byte, minPadding-idSize)
|
||||
for i := range boundary {
|
||||
boundary[i] = byte(i * 7)
|
||||
}
|
||||
put(rs, "hello.txt", []byte("hello"), 0o644, time.Unix(1700000000, 123456789))
|
||||
put(rs, "dir/empty", nil, 0o600, time.Unix(1600000000, 0))
|
||||
put(rs, "dir/ünïcode.bin", boundary, 0o755, time.Unix(1500000000, 999999999))
|
||||
put(&store{root: root, id: decoy}, "shopping.txt", []byte("milk\n"), 0o600, time.Unix(1700000000, 0))
|
||||
put(&store{root: root, id: throwaway}, "foreign", []byte("not yours"), 0o600, time.Unix(1700000000, 0))
|
||||
for _, d := range []string{recordsDir, objectsDir} {
|
||||
shards, _ := filepath.Glob(filepath.Join(root, d, "??"))
|
||||
for _, p := range append(shards, filepath.Join(root, d)) {
|
||||
os.Chtimes(p, epoch, epoch)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"math"
|
||||
mrand "math/rand/v2"
|
||||
"os"
|
||||
@@ -382,30 +383,43 @@ func (s *store) add(cat map[string][]*record, src, name string) (bool, error) {
|
||||
if !info.Mode().IsRegular() || info.Size() > fileLimit {
|
||||
return false, fmt.Errorf("%s: not a regular file of at most 1 TiB", src)
|
||||
}
|
||||
replaced, err := s.addReader(cat, f, info.Size(), info.Mode().Perm(), info.ModTime(), name)
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
err = fmt.Errorf("%s changed while being added", src)
|
||||
}
|
||||
return replaced, err
|
||||
}
|
||||
|
||||
// addReader stores size bytes from r as name, like add. Reading fewer than
|
||||
// size bytes stores nothing and fails with io.ErrUnexpectedEOF or io.EOF.
|
||||
func (s *store) addReader(cat map[string][]*record, r io.Reader, size int64, mode fs.FileMode, mtime time.Time, name string) (bool, error) {
|
||||
if !validName(name) {
|
||||
return false, fmt.Errorf("invalid vault path %q", name)
|
||||
}
|
||||
if size < 0 || size > fileLimit {
|
||||
return false, fmt.Errorf("%s: larger than 1 TiB", name)
|
||||
}
|
||||
seed, err := randomHex(objectSeedSize)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := s.writeObjects(f, seed, info.Size()); err != nil {
|
||||
s.dropObjects(seed, info.Size())
|
||||
if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||
err = fmt.Errorf("%s changed while being added", src)
|
||||
}
|
||||
if err := s.writeObjects(r, seed, size); err != nil {
|
||||
s.dropObjects(seed, size)
|
||||
return false, err
|
||||
}
|
||||
r := &record{
|
||||
Name: name, Seed: seed, Size: info.Size(), Mode: uint32(info.Mode().Perm()),
|
||||
Modified: info.ModTime().UTC(), Added: time.Now().UTC(),
|
||||
rec := &record{
|
||||
Name: name, Seed: seed, Size: size, Mode: uint32(mode.Perm()),
|
||||
Modified: mtime.UTC(), Added: time.Now().UTC(),
|
||||
}
|
||||
if err := s.writeRecord(r); err != nil {
|
||||
s.dropObjects(seed, info.Size())
|
||||
if err := s.writeRecord(rec); err != nil {
|
||||
s.dropObjects(seed, size)
|
||||
return false, err
|
||||
}
|
||||
replaced := len(cat[name]) > 0
|
||||
if err := s.remove(cat, name, true); err != nil {
|
||||
return replaced, err
|
||||
}
|
||||
cat[name] = []*record{r}
|
||||
cat[name] = []*record{rec}
|
||||
return replaced, nil
|
||||
}
|
||||
|
||||
@@ -523,26 +537,77 @@ func (s *store) delete(name string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.remove(cat, name, true)
|
||||
return s.removeAll(cat, []string{name}, func(string) {})
|
||||
}
|
||||
|
||||
func (s *store) rename(oldName, newName string) error {
|
||||
if !validName(newName) {
|
||||
return fmt.Errorf("invalid vault path %q", newName)
|
||||
// under returns the names in cat at or beneath the folder prefix, sorted.
|
||||
func under(cat map[string][]*record, prefix string) []string {
|
||||
var names []string
|
||||
for name := range cat {
|
||||
if name == prefix || strings.HasPrefix(name, prefix+"/") {
|
||||
names = append(names, name)
|
||||
}
|
||||
cat, r, err := s.lookup(oldName)
|
||||
if err != nil {
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// removeAll deletes every record for names and the objects they hold, then
|
||||
// drops them from cat, calling done after each name. Like remove, but it
|
||||
// works out once which objects other records still use, so deleting a large
|
||||
// folder stays linear.
|
||||
func (s *store) removeAll(cat map[string][]*record, names []string, done func(name string)) error {
|
||||
gone := map[string]bool{}
|
||||
for _, name := range names {
|
||||
gone[name] = true
|
||||
}
|
||||
inUse := map[string]bool{}
|
||||
for name, rs := range cat {
|
||||
if !gone[name] {
|
||||
for _, r := range rs {
|
||||
inUse[r.Seed] = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range names {
|
||||
for _, r := range cat[name] {
|
||||
if err := s.removeFile(recordsDir, r.id); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(cat[newName]) > 0 {
|
||||
return fmt.Errorf("%s already exists", newName)
|
||||
}
|
||||
moved := *r
|
||||
moved.Name = newName
|
||||
if err := s.writeRecord(&moved); err != nil {
|
||||
if !inUse[r.Seed] {
|
||||
if err := s.dropObjects(r.Seed, r.Size); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.remove(cat, oldName, false)
|
||||
}
|
||||
}
|
||||
delete(cat, name)
|
||||
done(name)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// copyAll stores each pair's first name again under its second, sharing its
|
||||
// objects, then with move deletes the first names. New records go first, so
|
||||
// an interruption leaves a file under both names, never neither.
|
||||
func (s *store) copyAll(cat map[string][]*record, pairs [][2]string, move bool, done func(from, to string)) error {
|
||||
var old []string
|
||||
for _, p := range pairs {
|
||||
r := *cat[p[0]][0]
|
||||
r.Name = p[1]
|
||||
if !move {
|
||||
r.Added = time.Now().UTC()
|
||||
}
|
||||
if err := s.writeRecord(&r); err != nil {
|
||||
return err
|
||||
}
|
||||
cat[p[1]] = []*record{&r}
|
||||
old = append(old, p[0])
|
||||
done(p[0], p[1])
|
||||
}
|
||||
if !move {
|
||||
return nil
|
||||
}
|
||||
return s.removeAll(cat, old, func(string) {})
|
||||
}
|
||||
|
||||
// get decrypts name to out. Nothing appears at out until every object,
|
||||
@@ -552,6 +617,10 @@ func (s *store) get(name, out string) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.getRecord(r, out)
|
||||
}
|
||||
|
||||
func (s *store) getRecord(r *record, out string) error {
|
||||
if _, err := os.Lstat(out); err == nil {
|
||||
return fmt.Errorf("%s already exists", out)
|
||||
}
|
||||
@@ -4,9 +4,12 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"bufio"
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"crypto/rand"
|
||||
"encoding/binary"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/http"
|
||||
@@ -15,8 +18,11 @@ import (
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/vault/internal/pqcrypt"
|
||||
)
|
||||
|
||||
func TestKeyFile(t *testing.T) {
|
||||
@@ -72,7 +78,11 @@ func TestStoreLifecycle(t *testing.T) {
|
||||
if cat, _ = s.catalogue(); len(cat) != 2 || len(cat["a/b"]) != 1 {
|
||||
t.Fatalf("catalogue = %v", cat)
|
||||
}
|
||||
if err := s.rename("a/b", "d"); err != nil {
|
||||
pairs, err := transfers(cat, "a/b", "d", false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.copyAll(cat, pairs, true, func(string, string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
@@ -430,3 +440,289 @@ func TestMirror(t *testing.T) {
|
||||
os.Remove(out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAbsorb(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newTestStore(t, keyPath, "pass")
|
||||
var buf bytes.Buffer
|
||||
zw := gzip.NewWriter(&buf)
|
||||
tw := tar.NewWriter(zw)
|
||||
mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
|
||||
for _, h := range []*tar.Header{
|
||||
{Name: "./d/", Typeflag: tar.TypeDir, Mode: 0o755},
|
||||
{Name: "./d/a.txt", Size: 5, Mode: 0o600, ModTime: mtime},
|
||||
{Name: "../escape", Size: 5, Mode: 0o600},
|
||||
{Name: "./d/link", Typeflag: tar.TypeSymlink, Linkname: "a.txt"},
|
||||
} {
|
||||
tw.WriteHeader(h)
|
||||
if h.Size > 0 {
|
||||
io.WriteString(tw, "hello")
|
||||
}
|
||||
}
|
||||
tw.Close()
|
||||
zw.Close()
|
||||
|
||||
cat, _ := s.catalogue()
|
||||
if err := absorb(s, cat, &buf, "in"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(cat) != 1 || len(cat["in/d/a.txt"]) != 1 {
|
||||
t.Fatalf("catalogue = %v, want only in/d/a.txt", cat)
|
||||
}
|
||||
if r := cat["in/d/a.txt"][0]; !r.Modified.Equal(mtime) || r.Mode != 0o600 {
|
||||
t.Errorf("record = %v, %o", r.Modified, r.Mode)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
if err := s.get("in/d/a.txt", out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if b, _ := os.ReadFile(out); string(b) != "hello" {
|
||||
t.Errorf("got %q", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRemoveTree(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newTestStore(t, keyPath, "pass")
|
||||
objects := filepath.Join(dir, "store", "objects")
|
||||
before := len(ciphertexts(t, objects))
|
||||
cat, _ := s.catalogue()
|
||||
src := filepath.Join(dir, "in")
|
||||
big := make([]byte, chunkData+1) // two objects
|
||||
rand.Read(big)
|
||||
for name, body := range map[string][]byte{"a/x": big, "a/b/y": []byte("y"), "ab/z": []byte("z"), "b/w": []byte("w")} {
|
||||
os.WriteFile(src, body, 0o600)
|
||||
if _, err := s.add(cat, src, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if n := len(ciphertexts(t, objects)) - before; n != 5 {
|
||||
t.Fatalf("%d objects added, want 5", n)
|
||||
}
|
||||
|
||||
names := under(cat, "a")
|
||||
if !slices.Equal(names, []string{"a/b/y", "a/x"}) {
|
||||
t.Fatalf("under a: %v", names)
|
||||
}
|
||||
var done []string
|
||||
if err := s.removeAll(cat, names, func(n string) { done = append(done, n) }); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Equal(done, names) || len(cat) != 2 || len(cat["ab/z"]) != 1 || len(cat["b/w"]) != 1 {
|
||||
t.Fatalf("done %v, left %v", done, cat)
|
||||
}
|
||||
if n := len(ciphertexts(t, objects)) - before; n != 2 {
|
||||
t.Fatalf("%d objects left, want 2: the space was not freed", n)
|
||||
}
|
||||
fresh, _ := s.catalogue()
|
||||
if len(fresh) != 2 {
|
||||
t.Fatalf("on disk: %v", fresh)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
if err := s.get("ab/z", out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLs(t *testing.T) {
|
||||
at := func(m int) time.Time { return time.Date(2026, 1, 1, 12, m, 0, 0, time.UTC) }
|
||||
cat := map[string][]*record{
|
||||
"notes.md": {{Size: 10, Modified: at(1)}},
|
||||
"a/x": {{Size: 100, Modified: at(2)}},
|
||||
"a/b/y": {{Size: 1000, Modified: at(5)}},
|
||||
"a/b/c/z": {{Size: 1, Modified: at(3)}},
|
||||
"ab/z": {{Size: 7, Modified: at(4)}},
|
||||
"photos/2025/img1": {{Size: 2048, Modified: at(6)}},
|
||||
}
|
||||
names := func(folder string) []string {
|
||||
t.Helper()
|
||||
var b bytes.Buffer
|
||||
if err := ls(&b, cat, folder); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var got []string
|
||||
for _, line := range strings.Split(strings.TrimRight(b.String(), "\n"), "\n") {
|
||||
got = append(got, strings.TrimSpace(line[10:]))
|
||||
}
|
||||
return got
|
||||
}
|
||||
day := func(m int) string { return at(m).Local().Format("2006-01-02 15:04") + " " }
|
||||
if got, want := names(""), []string{day(5) + "a/", day(4) + "ab/", day(6) + "photos/", day(1) + "notes.md"}; !slices.Equal(got, want) {
|
||||
t.Fatalf("ls: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := names("a"), []string{day(5) + "b/", day(2) + "x"}; !slices.Equal(got, want) {
|
||||
t.Fatalf("ls a: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := names("a/b"), []string{day(3) + "c/", day(5) + "y"}; !slices.Equal(got, want) {
|
||||
t.Fatalf("ls a/b: %q, want %q", got, want)
|
||||
}
|
||||
if got, want := names("a/x"), []string{day(2) + "x"}; !slices.Equal(got, want) {
|
||||
t.Fatalf("ls a/x: %q, want %q", got, want)
|
||||
}
|
||||
var b bytes.Buffer
|
||||
ls(&b, cat, "")
|
||||
if !strings.HasPrefix(b.String(), " 1.1 KiB ") {
|
||||
t.Fatalf("folder a/ should total 1101 bytes: %q", b.String())
|
||||
}
|
||||
if err := ls(&b, cat, "nope"); err == nil {
|
||||
t.Fatal("ls of a missing folder succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMove(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newTestStore(t, keyPath, "pass")
|
||||
cat, _ := s.catalogue()
|
||||
src := filepath.Join(dir, "in")
|
||||
for _, name := range []string{"a/x", "a/b/y", "ab/z", "c/w"} {
|
||||
os.WriteFile(src, []byte(name), 0o600)
|
||||
if _, err := s.add(cat, src, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
run := func(src, dst string, recursive, move bool) error {
|
||||
pairs, err := transfers(cat, src, dst, recursive)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return s.copyAll(cat, pairs, move, func(string, string) {})
|
||||
}
|
||||
for _, bad := range [][2]string{{"a", "d"}, {"a/x", "ab/z"}, {"nope", "d"}, {"", "d"}} {
|
||||
if run(bad[0], bad[1], false, false) == nil {
|
||||
t.Errorf("cp %s %s succeeded", bad[0], bad[1])
|
||||
}
|
||||
}
|
||||
if run("a", "a/b", true, true) == nil {
|
||||
t.Error("moved a folder into itself")
|
||||
}
|
||||
if err := run("a", "d", true, false); err != nil { // cp -r a d
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run("d", "c/", true, true); err != nil { // mv d c/
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := run("ab/z", "c", false, true); err != nil { // mv ab/z into existing folder c
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.removeAll(cat, under(cat, "a"), func(string) {}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fresh, _ := s.catalogue()
|
||||
var names []string
|
||||
for n := range fresh {
|
||||
names = append(names, n)
|
||||
}
|
||||
slices.Sort(names)
|
||||
if want := []string{"c/d/b/y", "c/d/x", "c/w", "c/z"}; !slices.Equal(names, want) {
|
||||
t.Fatalf("names %v, want %v", names, want)
|
||||
}
|
||||
out := filepath.Join(dir, "out")
|
||||
if err := s.get("c/d/x", out); err != nil {
|
||||
t.Fatal("copy lost its data when the original went: ", err)
|
||||
}
|
||||
if b, _ := os.ReadFile(out); string(b) != "a/x" {
|
||||
t.Errorf("got %q", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("XDG_RUNTIME_DIR", dir)
|
||||
t.Setenv(passphraseEnv, "") // restored after the test
|
||||
os.Unsetenv(passphraseEnv) // so keyIdentity asks the agent
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := unlockKeyFile(keyPath, "pass")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := agentIdentity(keyPath); err == nil {
|
||||
t.Fatal("identity from an agent that is not running")
|
||||
}
|
||||
sock, err := agentSocket(keyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := binary.BigEndian.AppendUint16(nil, id.SuiteID())
|
||||
secret = append(secret, id.Seed()...)
|
||||
pr, pw := io.Pipe()
|
||||
stopped := make(chan struct{})
|
||||
go func() { runAgent(sock, "1m", "fixed", bytes.NewReader(secret), pw); close(stopped) }()
|
||||
if msg, _ := io.ReadAll(pr); string(msg) != "ok" {
|
||||
t.Fatalf("agent said %q", msg)
|
||||
}
|
||||
got, err := keyIdentity(keyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.Equal(got.Seed(), id.Seed()) || got.SuiteID() != id.SuiteID() {
|
||||
t.Fatal("agent returned a different key")
|
||||
}
|
||||
if !agentStop(keyPath) {
|
||||
t.Fatal("agent was not running")
|
||||
}
|
||||
<-stopped
|
||||
if _, err := os.Stat(sock); !os.IsNotExist(err) {
|
||||
t.Fatal("socket left behind")
|
||||
}
|
||||
if agentStop(keyPath) {
|
||||
t.Fatal("agent still running")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAgentIdle(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
t.Setenv("XDG_RUNTIME_DIR", dir)
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
sock, err := agentSocket(keyPath)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
id, err := pqcrypt.GeneratePQIdentity()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
secret := binary.BigEndian.AppendUint16(nil, id.SuiteID())
|
||||
secret = append(secret, id.Seed()...)
|
||||
pr, pw := io.Pipe()
|
||||
stopped := make(chan struct{})
|
||||
go func() { runAgent(sock, "400ms", "idle", bytes.NewReader(secret), pw); close(stopped) }()
|
||||
if msg, _ := io.ReadAll(pr); string(msg) != "ok" {
|
||||
t.Fatalf("agent said %q", msg)
|
||||
}
|
||||
// Used every 150ms, it outlives its 400ms by well over twice.
|
||||
for range 7 {
|
||||
time.Sleep(150 * time.Millisecond)
|
||||
if _, err := agentIdentity(keyPath); err != nil {
|
||||
t.Fatal("idle agent expired while in use: ", err)
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-stopped:
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("idle agent outlived its idle time")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the dixon-vault .deb, .rpm and Arch packages locally and show what
|
||||
# is in them. Nothing is signed or published: packaging/repo.sh does that,
|
||||
# reusing this script's build step.
|
||||
#
|
||||
# packaging/package.sh # from the working tree, into dist/packages
|
||||
# packaging/package.sh v0.6 # from a tag (a clean export of it)
|
||||
# OUT=/tmp/pkgs ARCHS="amd64 arm64" packaging/package.sh
|
||||
#
|
||||
# Needs: the pinned Rust toolchain with cargo-zigbuild and zig (the static
|
||||
# Linux binaries are cross-built), and go, only to run the pinned nfpm
|
||||
# packager. dpkg-deb, rpm and zstd are used for inspection when present.
|
||||
# None of these is needed to run vault itself.
|
||||
#
|
||||
# ARCHS package architectures (default below)
|
||||
# RPM_SIGNING_KEY armoured private key file to sign the .rpm with
|
||||
# (repo.sh sets it; unset here means unsigned)
|
||||
set -euo pipefail
|
||||
|
||||
# ---- project ---------------------------------------------------------------
|
||||
NAME=vault # the command
|
||||
# The package: plain "vault" is HashiCorp Vault in Arch's [extra] and in
|
||||
# HashiCorp's own apt and dnf repositories, which would win upgrades. Both
|
||||
# install /usr/bin/vault, so they conflict.
|
||||
PACKAGE=dixon-vault
|
||||
CONFLICTS=vault
|
||||
SUMMARY="Post-quantum encrypted file vault with a decoy passphrase"
|
||||
HOMEPAGE=https://dixon.cx/vault
|
||||
LICENSE=GPL-3.0-or-later
|
||||
MANPAGES=docs/*.1
|
||||
DOCS="README.md LICENSE SECURITY.md"
|
||||
# Package architecture -> Rust target (all static musl binaries).
|
||||
triple() {
|
||||
case $1 in
|
||||
amd64) echo x86_64-unknown-linux-musl ;;
|
||||
arm64) echo aarch64-unknown-linux-musl ;;
|
||||
arm) echo armv7-unknown-linux-musleabihf ;;
|
||||
386) echo i686-unknown-linux-musl ;;
|
||||
riscv64) echo riscv64gc-unknown-linux-musl ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
ARCHS=${ARCHS:-"amd64 arm64 arm 386 riscv64"}
|
||||
NFPM="go run github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.41.1"
|
||||
|
||||
say() { echo "package: $*" >&2; }
|
||||
die() { say "$*"; exit 1; }
|
||||
|
||||
# build_packages SRC OUTDIR VERSION: binaries, then one .deb, .rpm and
|
||||
# .pkg.tar.zst per architecture, into OUTDIR.
|
||||
build_packages() {
|
||||
local src=$1 outdir=$2 version=$3 arch nfarch t work
|
||||
for tool in cargo cargo-zigbuild zig go; do
|
||||
command -v $tool >/dev/null || die "needs $tool"
|
||||
done
|
||||
work=$(mktemp -d)
|
||||
mkdir -p "$outdir"
|
||||
for arch in $ARCHS; do
|
||||
t=$(triple "$arch") || die "no Rust target for $arch"
|
||||
nfarch=$arch
|
||||
[[ $arch == arm ]] && nfarch=arm7
|
||||
say "building $NAME for $arch ($t)"
|
||||
flags="--remap-path-prefix=$src=vault"
|
||||
# zig's musl getrandom() never reaches the kernel on riscv64; have
|
||||
# the getrandom crate issue the system call itself.
|
||||
[[ $t == riscv64*-musl ]] && flags+=' --cfg getrandom_backend="linux_raw"'
|
||||
(cd "$src" && RUSTFLAGS=$flags cargo zigbuild --quiet --release --locked --target "$t") || die "build for $arch failed"
|
||||
install -D -m 0755 "$src/target/$t/release/$NAME" "$work/bin-$arch/$NAME"
|
||||
{
|
||||
cat <<-EOF
|
||||
name: $PACKAGE
|
||||
arch: $nfarch
|
||||
platform: linux
|
||||
version: $version
|
||||
release: 1
|
||||
maintainer: Andy Dixon <andy@dixon.cx>
|
||||
description: $SUMMARY
|
||||
homepage: $HOMEPAGE
|
||||
license: $LICENSE
|
||||
conflicts: [${CONFLICTS:-}]
|
||||
contents:
|
||||
- src: $work/bin-$arch/$NAME
|
||||
dst: /usr/bin/$NAME
|
||||
file_info: {mode: 0755}
|
||||
EOF
|
||||
for m in $(cd "$src" && ls $MANPAGES); do
|
||||
printf ' - src: %s\n dst: /usr/share/man/man1/%s\n file_info: {mode: 0644}\n' \
|
||||
"$src/$m" "$(basename "$m")"
|
||||
done
|
||||
for d in $DOCS; do
|
||||
printf ' - src: %s\n dst: /usr/share/doc/%s/%s\n file_info: {mode: 0644}\n' \
|
||||
"$src/$d" "$PACKAGE" "$d"
|
||||
done
|
||||
if [[ -n ${RPM_SIGNING_KEY:-} ]]; then
|
||||
printf 'rpm:\n signature:\n key_file: %s\n' "$RPM_SIGNING_KEY"
|
||||
fi
|
||||
} >"$work/nfpm-$arch.yaml"
|
||||
for fmt in deb rpm archlinux; do
|
||||
$NFPM pkg -f "$work/nfpm-$arch.yaml" -p $fmt -t "$outdir/" >/dev/null
|
||||
done
|
||||
done
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
|
||||
# inspect_packages DIR: list each package's metadata and files.
|
||||
inspect_packages() {
|
||||
local p
|
||||
for p in "$1"/*; do
|
||||
echo "== $(basename "$p")"
|
||||
case $p in
|
||||
*.deb) if command -v dpkg-deb >/dev/null; then dpkg-deb -f "$p" Package Version Architecture Conflicts; dpkg-deb -c "$p"; fi ;;
|
||||
*.rpm) if command -v rpm >/dev/null; then rpm -qpi "$p" 2>/dev/null | head -12; rpm -qplv "$p"; fi ;;
|
||||
*.pkg.tar.zst) if command -v zstd >/dev/null; then zstd -dcq "$p" | tar -tvf -; fi ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# When run (not sourced by repo.sh): build from the tree or a tag, inspect.
|
||||
if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
out=${OUT:-dist/packages}
|
||||
mkdir -p -- "$out"
|
||||
out=$(cd -- "$out" && pwd -P)
|
||||
rm -f -- "$out"/*.deb "$out"/*.rpm "$out"/*.pkg.tar.zst
|
||||
if [[ $# -gt 0 ]]; then
|
||||
git rev-parse -q --verify "refs/tags/$1" >/dev/null || die "no tag $1"
|
||||
src=$(mktemp -d)
|
||||
trap 'rm -rf -- "$src"' EXIT
|
||||
git archive "$1" | tar -x -C "$src"
|
||||
version=${1#v}
|
||||
else
|
||||
src=$PWD
|
||||
version=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"][0]["version"])')
|
||||
fi
|
||||
build_packages "$src" "$out" "$version"
|
||||
inspect_packages "$out"
|
||||
say "$(ls "$out" | wc -l) packages in $out (unsigned, not published)"
|
||||
fi
|
||||
@@ -0,0 +1,151 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build Linux packages (.deb, .rpm, Arch .pkg.tar.zst) for a release tag and
|
||||
# publish them to the signed apt, dnf and pacman repositories served at
|
||||
# https://repo.dixon.cx.
|
||||
#
|
||||
# packaging/repo.sh # the newest tag
|
||||
# packaging/repo.sh 0.5 # a given tag
|
||||
#
|
||||
# Builds from a clean export of the tag, so the working tree doesn't matter.
|
||||
# Every index is rebuilt over all the packages in the repository, so other
|
||||
# projects publishing there with their copy of this script are kept.
|
||||
#
|
||||
# Needs: the pinned Rust toolchain with cargo-zigbuild and zig, go (to run
|
||||
# nfpm), docker, gpg, apt-ftparchive (apt-utils), flock.
|
||||
# REPO_DIR where the repository lives (default below)
|
||||
# REPO_GNUPGHOME GnuPG home holding the signing key (default below)
|
||||
# ARCHS architectures to package (see packaging/package.sh)
|
||||
#
|
||||
# To build and inspect packages without signing or publishing, run
|
||||
# packaging/package.sh instead.
|
||||
#
|
||||
# The same script, with its own project block, is in ../mapsize/packaging.
|
||||
set -euo pipefail
|
||||
|
||||
# The project (package name, contents, architectures) and the build live in
|
||||
# packaging/package.sh, which can also be run on its own to build and inspect
|
||||
# packages without signing or publishing anything.
|
||||
. "$(dirname "$0")/package.sh"
|
||||
|
||||
REPO_DIR=${REPO_DIR:-/home/andy/domains/repo.dixon.cx/public_html}
|
||||
REPO_GNUPGHOME=${REPO_GNUPGHOME:-$HOME/.config/repo.dixon.cx/gnupg}
|
||||
SUITE=stable
|
||||
COMPONENT=main
|
||||
ARCH_REPO=dixon # the pacman repository name: [dixon]
|
||||
|
||||
say() { echo "repo: $*" >&2; }
|
||||
die() { say "$*"; exit 1; }
|
||||
|
||||
for tool in cargo cargo-zigbuild zig go docker gpg apt-ftparchive flock; do
|
||||
command -v $tool >/dev/null || die "needs $tool"
|
||||
done
|
||||
[[ -d $REPO_DIR ]] || die "$REPO_DIR does not exist"
|
||||
export GNUPGHOME=$REPO_GNUPGHOME
|
||||
KEY=$(gpg --list-secret-keys --with-colons 2>/dev/null | awk -F: '/^fpr/{print $10; exit}')
|
||||
[[ -n $KEY ]] || die "no signing key in $GNUPGHOME"
|
||||
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
TAG=${1:-$(git describe --tags --abbrev=0)}
|
||||
git rev-parse -q --verify "refs/tags/$TAG" >/dev/null || die "no tag $TAG"
|
||||
VERSION=${TAG#v}
|
||||
export VERSION
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
mkdir -p "$WORK/src"
|
||||
git archive "$TAG" | tar -x -C "$WORK/src"
|
||||
gpg --batch --armor --export-secret-keys "$KEY" >"$WORK/signing.asc"
|
||||
|
||||
# ---- build ----------------------------------------------------------------
|
||||
say "building $PACKAGE $VERSION for $ARCHS"
|
||||
RPM_SIGNING_KEY=$WORK/signing.asc build_packages "$WORK/src" "$WORK/out" "$VERSION"
|
||||
|
||||
# ---- publish ---------------------------------------------------------------
|
||||
# One publisher at a time: the indexes cover every project's packages.
|
||||
exec 9>"$REPO_DIR/../.publish.lock"
|
||||
flock 9
|
||||
say "publishing to $REPO_DIR"
|
||||
gpg --batch --armor --export "$KEY" >"$REPO_DIR/dixon.asc"
|
||||
gpg --batch --export "$KEY" >"$REPO_DIR/dixon.gpg"
|
||||
sign() { gpg --batch --yes --local-user "$KEY" "$@"; }
|
||||
|
||||
# apt: pool/main/<letter>/<name>/, indexed by apt-ftparchive.
|
||||
pool=$REPO_DIR/apt/pool/$COMPONENT/${PACKAGE:0:1}/$PACKAGE
|
||||
mkdir -p "$pool"
|
||||
cp "$WORK"/out/*.deb "$pool/"
|
||||
(
|
||||
cd "$REPO_DIR/apt"
|
||||
# Built beside the live suite and swapped in, so clients never see half.
|
||||
dists=dists/.$SUITE.new
|
||||
rm -rf "$dists"
|
||||
debarchs=$(find pool -name '*.deb' | sed -E 's/.*_([^_]+)\.deb$/\1/' | sort -u | tr '\n' ' ')
|
||||
for a in $debarchs; do
|
||||
d=$dists/$COMPONENT/binary-$a
|
||||
mkdir -p "$d"
|
||||
apt-ftparchive --arch "$a" packages pool >"$d/Packages"
|
||||
gzip -9kn "$d/Packages"
|
||||
done
|
||||
apt-ftparchive \
|
||||
-o APT::FTPArchive::Release::Origin=repo.dixon.cx \
|
||||
-o APT::FTPArchive::Release::Label=repo.dixon.cx \
|
||||
-o APT::FTPArchive::Release::Suite=$SUITE \
|
||||
-o APT::FTPArchive::Release::Codename=$SUITE \
|
||||
-o APT::FTPArchive::Release::Components=$COMPONENT \
|
||||
-o "APT::FTPArchive::Release::Architectures=${debarchs% }" \
|
||||
release "$dists" >"$WORK/Release"
|
||||
mv "$WORK/Release" "$dists/Release"
|
||||
sign --clearsign -o "$dists/InRelease" "$dists/Release"
|
||||
sign --armor --detach-sign -o "$dists/Release.gpg" "$dists/Release"
|
||||
rm -rf "dists/.$SUITE.old"
|
||||
[[ -d dists/$SUITE ]] && mv "dists/$SUITE" "dists/.$SUITE.old"
|
||||
mv "$dists" "dists/$SUITE"
|
||||
rm -rf "dists/.$SUITE.old"
|
||||
)
|
||||
|
||||
# dnf: every architecture in one directory; dnf picks its own.
|
||||
mkdir -p "$REPO_DIR/rpm"
|
||||
cp "$WORK"/out/*.rpm "$REPO_DIR/rpm/"
|
||||
docker image inspect repo-tools:fedora >/dev/null 2>&1 ||
|
||||
docker build -q -t repo-tools:fedora - >/dev/null <<-'EOF'
|
||||
FROM fedora:42
|
||||
RUN dnf -y install createrepo_c && dnf clean all
|
||||
EOF
|
||||
docker run --rm --user "$(id -u):$(id -g)" -v "$REPO_DIR/rpm:/repo" repo-tools:fedora \
|
||||
createrepo_c --quiet --update --general-compress-type=gz /repo
|
||||
sign --armor --detach-sign -o "$REPO_DIR/rpm/repodata/repomd.xml.asc" "$REPO_DIR/rpm/repodata/repomd.xml"
|
||||
cat >"$REPO_DIR/rpm/dixon.repo" <<EOF
|
||||
[dixon]
|
||||
name=repo.dixon.cx
|
||||
baseurl=https://repo.dixon.cx/rpm
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
repo_gpgcheck=1
|
||||
gpgkey=https://repo.dixon.cx/dixon.asc
|
||||
EOF
|
||||
|
||||
# pacman: one directory per architecture, each package signed, then
|
||||
# repo-add (from an Arch container) and a signed database.
|
||||
added=
|
||||
for pkg in "$WORK"/out/*.pkg.tar.zst; do
|
||||
a=$(basename "$pkg" .pkg.tar.zst)
|
||||
a=${a##*-}
|
||||
mkdir -p "$REPO_DIR/arch/$a"
|
||||
cp "$pkg" "$REPO_DIR/arch/$a/"
|
||||
sign --detach-sign -o "$REPO_DIR/arch/$a/$(basename "$pkg").sig" "$pkg"
|
||||
added+=" $a"
|
||||
done
|
||||
for a in $(tr ' ' '\n' <<<"$added" | sort -u); do
|
||||
dir=$REPO_DIR/arch/$a
|
||||
# A fresh database of every package here; -p keeps the newest version.
|
||||
rm -f "$dir/$ARCH_REPO".{db,files}{,.tar.gz}{,.sig}
|
||||
docker run --rm --user "$(id -u):$(id -g)" -e HOME=/tmp -v "$dir:/repo" -w /repo archlinux:latest \
|
||||
sh -c "repo-add --quiet --prevent-downgrade $ARCH_REPO.db.tar.gz *.pkg.tar.zst" 2>&1 |
|
||||
{ grep -v "newer version\|^$" >&2 || true; }
|
||||
for db in $ARCH_REPO.db $ARCH_REPO.files; do
|
||||
sign --detach-sign -o "$dir/$db.tar.gz.sig" "$dir/$db.tar.gz"
|
||||
ln -sfn "$db.tar.gz.sig" "$dir/$db.sig"
|
||||
done
|
||||
rm -f "$dir"/*.old "$dir"/*.old.sig
|
||||
done
|
||||
|
||||
say "published $PACKAGE $VERSION: $(cd "$WORK/out" && ls | wc -l) packages"
|
||||
@@ -0,0 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.99.0"
|
||||
components = ["rustfmt", "clippy"]
|
||||
profile = "minimal"
|
||||
@@ -0,0 +1 @@
|
||||
max_width = 140
|
||||
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compare the Go oracle and the Rust build on this machine, end to end.
|
||||
#
|
||||
# scripts/bench.sh [WORKDIR] (default: target/bench, on the repo's disk)
|
||||
# RUNS=5 BIG_MB=2048 SMALL=5000 scripts/bench.sh
|
||||
#
|
||||
# For each implementation, in a fresh key and vault on the same filesystem:
|
||||
# add one BIG_MB file, and add SMALL small files (1-16 KiB) as a folder, each
|
||||
# into a fresh vault; then, in one vault holding both, get the big file,
|
||||
# list the catalogue (filler plus SMALL+1 records), and get one small file. Each step is timed RUNS times (median, min, max), with peak RSS and
|
||||
# peak thread count. Every step includes unlocking the key (two Argon2id
|
||||
# runs). Both implementations fsync every object and record, so `add`
|
||||
# timings include durable writes. Page caches are warm: dropping them needs
|
||||
# root, so cold-cache numbers are not measured here.
|
||||
#
|
||||
# Needs: go (to build the oracle), cargo, /usr/bin/time (GNU).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
root=$PWD
|
||||
work=${1:-$root/target/bench}
|
||||
runs=${RUNS:-3}
|
||||
big_mb=${BIG_MB:-1024}
|
||||
small=${SMALL:-500}
|
||||
|
||||
mkdir -p -- "$work"
|
||||
work=$(cd -- "$work" && pwd -P)
|
||||
[[ $work == "$root" || $work == / ]] && { echo "refusing WORKDIR=$work" >&2; exit 2; }
|
||||
|
||||
cargo build --release --locked --quiet
|
||||
(cd oracle/go && CGO_ENABLED=0 go build -trimpath -o "$work/vault-go" .)
|
||||
cp target/release/vault "$work/vault-rust"
|
||||
|
||||
data=$work/data
|
||||
if [[ ! -f $data/big || $(stat -c %s "$data/big") != $((big_mb << 20)) ]]; then
|
||||
rm -rf -- "$data"
|
||||
mkdir -p -- "$data/small"
|
||||
head -c $((big_mb << 20)) /dev/urandom >"$data/big"
|
||||
for i in $(seq -w 1 "$small"); do
|
||||
head -c $((1024 + RANDOM % 15360)) /dev/urandom >"$data/small/f$i"
|
||||
done
|
||||
fi
|
||||
|
||||
export VAULT_PASSPHRASE=bench-passphrase
|
||||
|
||||
# measure LABEL CMD...: one timed run, printing "seconds rss_kib threads".
|
||||
measure() {
|
||||
local out=$work/time.$$ peak=0 pid t
|
||||
/usr/bin/time -f '%e %M' -o "$out" "$@" >/dev/null 2>&1 &
|
||||
pid=$!
|
||||
while kill -0 $pid 2>/dev/null; do
|
||||
for t in /proc/[0-9]*/status; do
|
||||
if grep -q "^PPid:[[:space:]]*$pid$" "$t" 2>/dev/null; then
|
||||
n=$(awk '/^Threads:/{print $2}' "$t" 2>/dev/null || echo 0)
|
||||
((n > peak)) && peak=$n
|
||||
fi
|
||||
done
|
||||
sleep 0.05
|
||||
done
|
||||
wait $pid
|
||||
echo "$(cat "$out") $peak"
|
||||
rm -f -- "$out"
|
||||
}
|
||||
|
||||
summarise() { # lines of "secs rss threads" -> median min max, max rss, max threads
|
||||
sort -n | awk '{t[NR]=$1; if($2>r)r=$2; if($3>th)th=$3} END {printf "%8.2f s (min %.2f, max %.2f) rss %6.0f MiB threads %d\n", t[int((NR+1)/2)], t[1], t[NR], r/1024, th}'
|
||||
}
|
||||
|
||||
fresh() { # fresh key and vault at $1
|
||||
rm -rf -- "$1"
|
||||
mkdir -p -- "$1"
|
||||
export VAULT_KEY=$1/key VAULT_DIR=$1/store
|
||||
"$bin" genkey 2>/dev/null
|
||||
"$bin" init 2>/dev/null
|
||||
}
|
||||
|
||||
for impl in go rust; do
|
||||
bin=$work/vault-$impl
|
||||
echo "== $impl ($(stat -c %s "$bin") bytes)"
|
||||
# Writes: a fresh vault for every run.
|
||||
for step in add-big add-small; do
|
||||
results=""
|
||||
for run in $(seq 1 "$runs"); do
|
||||
fresh "$work/run-$impl"
|
||||
case $step in
|
||||
add-big) r=$(measure "$bin" add "$data/big") ;;
|
||||
add-small) r=$(measure "$bin" add "$data/small") ;;
|
||||
esac
|
||||
results+="$r"$'\n'
|
||||
done
|
||||
printf ' %-10s ' "$step"
|
||||
printf '%s' "$results" | grep . | summarise
|
||||
done
|
||||
# Reads: one populated vault, read repeatedly.
|
||||
v=$work/run-$impl
|
||||
fresh "$v"
|
||||
"$bin" add "$data/big" 2>/dev/null
|
||||
"$bin" add "$data/small" 2>/dev/null
|
||||
for step in get-big list get-small; do
|
||||
results=""
|
||||
for run in $(seq 1 "$runs"); do
|
||||
rm -f -- "$v/big.out" "$v/small.out"
|
||||
case $step in
|
||||
get-big) r=$(measure "$bin" get big -o "$v/big.out") ;;
|
||||
list) r=$(measure "$bin" list) ;;
|
||||
get-small) r=$(measure "$bin" get "small/f$(printf "%0${#small}d" 1)" -o "$v/small.out") ;;
|
||||
esac
|
||||
results+="$r"$'\n'
|
||||
done
|
||||
printf ' %-10s ' "$step"
|
||||
printf '%s' "$results" | grep . | summarise
|
||||
done
|
||||
rm -rf -- "$work/run-$impl"
|
||||
done
|
||||