Rewrite vault in Rust (format-compatible with 0.5) #1
@@ -0,0 +1,111 @@
|
||||
name: ci
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
# Oracle-backed interoperability tests must run, not skip, in CI.
|
||||
VAULT_REQUIRE_ORACLE: "1"
|
||||
|
||||
jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- run: cargo fmt --check
|
||||
- run: cargo clippy --locked --all-targets --all-features -- -D warnings
|
||||
|
||||
test:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: [ubuntu-24.04, ubuntu-24.04-arm, macos-15, macos-15-intel, windows-2025]
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
# The Go toolchain builds the test-only oracle; vault itself needs no Go.
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: "1.26.x"
|
||||
cache-dependency-path: oracle/go/go.sum
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- name: test
|
||||
shell: bash
|
||||
run: |
|
||||
if [[ $RUNNER_OS == Windows ]]; then unset VAULT_REQUIRE_ORACLE; fi
|
||||
cargo test --locked --all-targets --all-features
|
||||
- run: cargo test --locked --doc
|
||||
- run: cargo build --locked --release
|
||||
- name: oracle self-test
|
||||
if: runner.os == 'Linux'
|
||||
working-directory: oracle/go
|
||||
run: go test ./...
|
||||
|
||||
cross:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-go@v6
|
||||
with:
|
||||
go-version: "1.26.x"
|
||||
cache-dependency-path: oracle/go/go.sum
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- name: zig and cargo-zigbuild
|
||||
run: |
|
||||
curl -sSLo zig.tar.xz https://ziglang.org/download/0.16.0/zig-x86_64-linux-0.16.0.tar.xz
|
||||
echo "70e49664a74374b48b51e6f3fdfbf437f6395d42509050588bd49abe52ba3d00 zig.tar.xz" | sha256sum -c
|
||||
mkdir -p "$HOME/zig" && tar -xJf zig.tar.xz -C "$HOME/zig" --strip-components=1
|
||||
echo "$HOME/zig" >>"$GITHUB_PATH"
|
||||
cargo install cargo-zigbuild --version 0.23.4 --locked
|
||||
cargo install cross --version 0.2.5 --locked
|
||||
- name: targets
|
||||
run: |
|
||||
rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl i686-unknown-linux-musl \
|
||||
arm-unknown-linux-musleabihf armv7-unknown-linux-musleabihf riscv64gc-unknown-linux-musl \
|
||||
loongarch64-unknown-linux-musl powerpc64le-unknown-linux-musl \
|
||||
s390x-unknown-linux-gnu x86_64-pc-windows-gnu i686-pc-windows-gnu aarch64-pc-windows-gnullvm \
|
||||
x86_64-unknown-freebsd i686-unknown-freebsd
|
||||
- name: build
|
||||
run: |
|
||||
# Everything but macOS, which the test job builds on a Mac.
|
||||
TARGETS="linux/amd64 linux/arm64 linux/386 linux/armv6 linux/armv7 linux/riscv64 linux/loong64 \
|
||||
linux/ppc64le linux/s390x windows/amd64 windows/386 windows/arm64 freebsd/amd64 freebsd/386" ./build.sh
|
||||
- name: run builds under qemu, cross-checked with the Go oracle
|
||||
run: |
|
||||
sudo apt-get update -q && sudo apt-get install -yq qemu-user libc6-s390x-cross
|
||||
(cd oracle/go && go build -o ../../target/vault-go .)
|
||||
for t in "arm64 qemu-aarch64" "armv7 qemu-arm" "riscv64 qemu-riscv64" "ppc64le qemu-ppc64le" \
|
||||
"s390x qemu-s390x -L /usr/s390x-linux-gnu"; do
|
||||
EMU="${t#* }" scripts/smoke.sh "dist/vault-linux-${t%% *}" target/vault-go
|
||||
done
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: dist
|
||||
path: dist/
|
||||
|
||||
supply-chain:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup show active-toolchain || rustup toolchain install
|
||||
- run: cargo install cargo-deny --version 0.20.2 --locked
|
||||
- run: cargo deny --locked check
|
||||
|
||||
fuzz:
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- run: rustup toolchain install nightly-2026-10-01 --profile minimal
|
||||
- run: cargo +nightly-2026-10-01 install cargo-fuzz --version 0.13.2 --locked
|
||||
- name: a short run of every target
|
||||
working-directory: fuzz
|
||||
run: |
|
||||
for t in $(cargo +nightly-2026-10-01 fuzz list); do
|
||||
cargo +nightly-2026-10-01 fuzz run "$t" -- -max_total_time=60
|
||||
done
|
||||
@@ -6,3 +6,8 @@
|
||||
/docs/guide/version.tex
|
||||
/docs/guide/vault-guide.pdf
|
||||
/docs/guide/*.log
|
||||
/target
|
||||
fuzz/target
|
||||
fuzz/corpus
|
||||
fuzz/artifacts
|
||||
fuzz/coverage
|
||||
|
||||
@@ -4,71 +4,93 @@ This file provides guidance to Claude Code when working with this repository.
|
||||
|
||||
## Overview
|
||||
|
||||
vault is a post-quantum encrypted file vault: one `package main` at the repo
|
||||
root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
|
||||
vault is a post-quantum encrypted file vault: one Rust package (`Cargo.toml`
|
||||
at the root, toolchain pinned in `rust-toolchain.toml`) that builds a single
|
||||
`vault` binary. The library (`src/lib.rs`) exists so tests and benchmarks can
|
||||
reach the same code; `src/main.rs` only calls `cli::main`.
|
||||
|
||||
- `main.go` - CLI: `genkey`, `init`, `add` (file, directory
|
||||
or URL), `migrate` (add, then delete originals), `mirror`, `absorb` (tar
|
||||
on stdin, `absorb.go`), `ls` (one folder level), `list` (recursive),
|
||||
`get` (`-r` for a folder), `mv`/`cp` (files or folders; `transfers`
|
||||
resolves names like mv(1), copies share objects), `rm` (`-r` for a
|
||||
folder, asks unless `-f`), `unlock`/`lock`.
|
||||
- `agent*.go` - `vault unlock` re-executes itself as a hidden `__agent`
|
||||
process holding the suite ID and seed (pipe in, mlocked, undumpable) and
|
||||
serving them on a Unix socket in a 0700 per-user dir until a deadline
|
||||
(fixed, or with `--idle` reset on each use). `keyIdentity` asks it first
|
||||
unless `$VAULT_PASSPHRASE` is set.
|
||||
- `keyfile.go` - key file with two passphrase slots (Argon2id +
|
||||
XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second
|
||||
slot is either a decoy identity or random bytes; slot order is random.
|
||||
- `store.go` - on-disk vault: `records/` holds fixed-size encrypted catalogue
|
||||
records, `objects/` holds file bodies cut into 4 MiB chunks (last one
|
||||
bucket-padded) whose IDs are HMAC'd from a per-record seed. Both are
|
||||
sharded into 256 `xx/` subdirectories by ID prefix. Records a key cannot
|
||||
open are skipped, not treated as errors.
|
||||
- `lock*.go` - vault lock on `DIR/.lock`: fcntl on unix, LockFileEx on
|
||||
Windows, an exclusive lock file elsewhere. Writers lock exclusively,
|
||||
readers shared, taken after the passphrase prompt.
|
||||
- `docs/guide` - LaTeX user guide (`build-pdf.sh` via Tectonic; `shots.py`
|
||||
re-shoots screenshots from the real binary). Keep it in step with the CLI.
|
||||
- `packaging/repo.sh [TAG]` - builds dixon-vault .deb/.rpm/Arch packages
|
||||
from a tag with nfpm and publishes them, signed, to the apt/dnf/pacman
|
||||
repo at repo.dixon.cx (same script, other project block, in ../mapsize).
|
||||
- `build.sh` - cross-compiles every `go tool dist list` target except
|
||||
iOS and android/386, amd64, arm (cgo-only).
|
||||
- `internal/pqcrypt` - file encryption (header + STREAM ChaCha20-Poly1305),
|
||||
suite registry, hybrid FrodoKEM-640 + ML-KEM-768 default suite.
|
||||
- `internal/format`, `internal/stream`, `internal/term` - header encoding,
|
||||
chunked payload encryption, terminal passphrase prompts.
|
||||
- `src/cli.rs` - commands and Go-`flag`-compatible parsing: `genkey`,
|
||||
`init`, `add` (file, directory or URL), `migrate`, `mirror`, `absorb`,
|
||||
`ls`/`list`, `get` (`-r`), `mv`/`cp` (`transfers` resolves names like
|
||||
mv(1)), `rm` (`-r`, asks unless `-f`), `unlock`/`lock`, hidden `__agent`.
|
||||
- `src/agent.rs` - the unlock agent: `unlock` re-executes the binary as
|
||||
`__agent` (passphrase variables stripped, secret over a pipe); it hardens
|
||||
itself (`platform::memory`) before reading the key, serves it on a Unix
|
||||
socket in a 0700 per-user dir until a deadline (fixed or `--idle`).
|
||||
Wire-compatible with the Go agent. Not available on Windows.
|
||||
- `src/keyfile.rs` - key file: two Argon2id/XChaCha20-Poly1305 slots, decoy
|
||||
or random bytes in the second, random order, both always tried.
|
||||
- `src/crypto/` - `frodo.rs` (FrodoKEM-640-SHAKE ported from CIRCL v1.6.4),
|
||||
`kem.rs` (hybrid suite 0x0001 with RustCrypto ml-kem, `Identity`),
|
||||
`header.rs` (Fury header format), `fury.rs` (file encryption), `stream.rs`
|
||||
(STREAM, including random access).
|
||||
- `src/store/` - on-disk vault: `records/`, `objects/` sharded by ID prefix,
|
||||
catalogue, add/remove/copy/extract, filler; `types.rs` holds the validated
|
||||
types (`VaultName`, `Id`, `ObjectSeed`, `Record`) and record JSON.
|
||||
- `src/pipeline.rs` - the one bounded worker pool and the eight reusable
|
||||
object buffers; coordinators run on the caller's thread.
|
||||
- `src/platform/` - all `unsafe` and OS specifics: `fs.rs` (no-clobber
|
||||
publish, `Dir` confined extraction, epoch times, dir sync), `lock.rs`
|
||||
(fcntl / LockFileEx / lock file), `memory.rs` (mlock, undumpable),
|
||||
`term.rs` (prompts).
|
||||
- `src/net/` - `add URL` and `mirror` with limits; `url.rs` (Go `net/url`
|
||||
semantics), `html.rs` (tokenizer and link rewriting).
|
||||
- `src/absorb.rs` - tar (optionally gzipped) from stdin, streamed in.
|
||||
- `src/secret.rs`, `src/rng.rs`, `src/util.rs`, `src/error.rs` - secret
|
||||
buffers, randomness (seeded only under `cfg(test)`), Go-compatible
|
||||
parsing/printing helpers, the error type.
|
||||
- `oracle/go` - the frozen Go implementation (test-only reference, see
|
||||
`oracle/README.md`); `oracle/gen-vectors.sh` writes `tests/vectors`.
|
||||
- `tests/` - CLI and interoperability tests against the real binaries;
|
||||
`fuzz/` - cargo-fuzz targets; `benches/micro.rs` - micro-benchmarks.
|
||||
- `docs/` - man pages, `COMPATIBILITY.md`, `MIGRATION.md`, `PLATFORMS.md`,
|
||||
`BENCHMARKS.md`, and the LaTeX guide (`docs/guide`, `build-pdf.sh` via
|
||||
Tectonic; `shots.py` re-shoots screenshots from the real binary). Keep
|
||||
them in step with the CLI.
|
||||
- `build.sh` - cross-builds every supported target (see `PLATFORMS.md`).
|
||||
- `packaging/package.sh` builds and inspects dixon-vault packages locally;
|
||||
`packaging/repo.sh [TAG]` signs and publishes them to repo.dixon.cx.
|
||||
|
||||
## Commands
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go build .
|
||||
./build.sh # all platforms, into ./dist
|
||||
TARGETS="linux/amd64" ./build.sh # a subset
|
||||
cargo test --locked # needs go on PATH for interop tests
|
||||
cargo clippy --locked --all-targets --all-features -- -D warnings
|
||||
cargo fmt --check
|
||||
cargo build --release --locked
|
||||
./build.sh # all buildable targets, into ./dist
|
||||
TARGETS="linux/amd64" ./build.sh # a subset
|
||||
cargo deny --locked check # advisories and licences
|
||||
(cd fuzz && cargo +nightly fuzz run header)
|
||||
```
|
||||
|
||||
End-to-end check:
|
||||
|
||||
```sh
|
||||
export VAULT_PASSPHRASE=test VAULT_KEY=$PWD/k VAULT_DIR=$PWD/s
|
||||
./vault genkey && ./vault init && ./vault add README.md docs && ./vault list
|
||||
target/release/vault genkey && target/release/vault init && target/release/vault add README.md docs && target/release/vault list
|
||||
```
|
||||
|
||||
## Invariants
|
||||
|
||||
- The on-disk formats are frozen and shared with the Go version; check any
|
||||
change against `docs/COMPATIBILITY.md` and the oracle vectors.
|
||||
- Never write a seed or private key unencrypted (the agent gets it by pipe).
|
||||
- Keep suite IDs, stanza tags, HKDF info strings and `fileKeySize` (32) stable.
|
||||
- Keep suite IDs, stanza tags, HKDF info strings and the 32-byte file key stable.
|
||||
- Every ciphertext's and shard directory's mtime is pinned to the epoch;
|
||||
records stay one fixed size; objects stay at most 4 MiB and bucket-padded.
|
||||
These hide metadata from the storage host.
|
||||
- An object's plaintext starts with its own ID, binding it to its record
|
||||
and its position in the file.
|
||||
- `get` publishes output only after every object has authenticated.
|
||||
- `get` publishes output only after every object has authenticated, with a
|
||||
no-replace primitive; never fall back to check-then-rename.
|
||||
- Decoy deniability: both key slots stay the same size and are both always
|
||||
tried; `init` always writes filler; foreign records are skipped silently.
|
||||
- Every command that changes the vault holds the exclusive lock.
|
||||
- Keep dependencies pure Go and CGO_ENABLED=0 builds working.
|
||||
- Every command that changes the vault holds the exclusive lock (POSIX
|
||||
fcntl locks, compatible with Go's).
|
||||
- Crash-safe order: objects before records, new records before removing old
|
||||
ones, record removal (synced) before object removal.
|
||||
- Secrets live in `SecretArray`/`SecretVec`/`LockedBuf`: no Clone, Debug or
|
||||
serialisation. Production randomness is the OS only.
|
||||
- `unsafe` only in `src/platform`, each block with a SAFETY comment.
|
||||
- Diagnostics go to standard error, lower-case, no trailing full stops.
|
||||
|
||||
Generated
+1320
File diff suppressed because it is too large
Load Diff
+57
@@ -0,0 +1,57 @@
|
||||
[package]
|
||||
name = "vault"
|
||||
version = "0.6.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.99"
|
||||
license = "GPL-3.0-or-later"
|
||||
description = "Post-quantum encrypted file vault with a decoy passphrase"
|
||||
homepage = "https://dixon.cx/vault"
|
||||
repository = "https://github.com/andydixon/vault"
|
||||
publish = false
|
||||
|
||||
[dependencies]
|
||||
argon2 = { version = "0.6", default-features = false, features = ["alloc", "zeroize", "parallel"] }
|
||||
chacha20poly1305 = { version = "0.11", default-features = false, features = ["alloc", "zeroize"] }
|
||||
hkdf = "0.13"
|
||||
hmac = { version = "0.13", features = ["zeroize"] }
|
||||
sha2 = { version = "0.11", default-features = false, features = ["zeroize"] }
|
||||
sha3 = { version = "0.11", default-features = false, features = ["zeroize"] }
|
||||
ml-kem = { version = "0.3.2", features = ["zeroize", "hazmat"] }
|
||||
zeroize = { version = "1.9", features = ["derive", "serde"] }
|
||||
getrandom = "0.4"
|
||||
base64 = "0.23"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
jiff = "0.2"
|
||||
tar = { version = "0.4.46", default-features = false }
|
||||
flate2 = { version = "1.1", default-features = false, features = ["rust_backend"] }
|
||||
ureq = { version = "3.4", default-features = false, features = ["rustls", "platform-verifier", "gzip"] }
|
||||
|
||||
[dev-dependencies]
|
||||
aes = "0.9"
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
rustix = { version = "1.1.5", features = ["fs", "process", "mm", "termios", "event", "net", "stdio", "param"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
windows-sys = { version = "0.61", features = ["Win32_Foundation", "Win32_Storage_FileSystem", "Win32_System_Console", "Win32_System_IO", "Win32_System_Memory", "Win32_Security"] }
|
||||
|
||||
# Tests run the real KEMs and Argon2id; unoptimised they are far too slow.
|
||||
[profile.dev]
|
||||
opt-level = 1
|
||||
|
||||
[profile.dev.package."*"]
|
||||
opt-level = 3
|
||||
|
||||
[profile.release]
|
||||
codegen-units = 1
|
||||
lto = true
|
||||
strip = true
|
||||
|
||||
[[bench]]
|
||||
name = "micro"
|
||||
harness = false
|
||||
|
||||
[[bench]]
|
||||
name = "aead_probe"
|
||||
harness = false
|
||||
@@ -23,16 +23,36 @@ Releases are packaged and published with `packaging/repo.sh [TAG]`.
|
||||
|
||||
## Build
|
||||
|
||||
vault is written in Rust (the toolchain is pinned in `rust-toolchain.toml`;
|
||||
`rustup` installs it on first use). Releases up to 0.5 were Go; the Go code
|
||||
lives on, frozen, in `oracle/go` as the test reference. Nothing at run time
|
||||
needs Go.
|
||||
|
||||
```sh
|
||||
go build . # produces ./vault
|
||||
go test ./...
|
||||
./build.sh # every OS and CPU architecture, into ./dist
|
||||
cargo build --release --locked # produces target/release/vault
|
||||
cargo test --locked # unit, CLI and Go-interoperability tests
|
||||
./build.sh # every target this host can build, into ./dist
|
||||
TARGETS="linux/amd64 windows/amd64" ./build.sh
|
||||
```
|
||||
|
||||
`build.sh` cross-compiles static binaries for every target in
|
||||
`go tool dist list` except iOS and 32-bit or x86 Android, which need cgo. It builds 32-bit ARM Linux for
|
||||
both armv6 and armv7, and writes `dist/SHA256SUMS`. Set `TARGETS` to build a
|
||||
subset, for example `TARGETS="linux/amd64 darwin/arm64" ./build.sh`.
|
||||
`build.sh` cross-compiles with `cargo-zigbuild` and `zig` (Linux, statically
|
||||
linked with musl, and Windows), on a Mac for macOS, and with `cross` and
|
||||
Docker for FreeBSD. The release covers Linux, Windows, macOS and FreeBSD. It names binaries as before
|
||||
(`vault-linux-arm64`, `vault-windows-amd64.exe`, …), builds 32-bit ARM Linux
|
||||
for both armv6 and armv7, and writes `dist/SHA256SUMS`. `docs/PLATFORMS.md`
|
||||
lists every target, how it is built and tested, and the Go targets Rust
|
||||
cannot build.
|
||||
|
||||
The interoperability tests build the Go oracle when `go` is installed and
|
||||
are skipped (and say so) otherwise; set `VAULT_REQUIRE_ORACLE=1` to make a
|
||||
missing oracle an error, as CI does. `oracle/gen-vectors.sh` regenerates the
|
||||
Go test vectors. Packages are built and inspected locally, unsigned, with
|
||||
`packaging/package.sh`, and published with `packaging/repo.sh [TAG]`.
|
||||
|
||||
Further reading: `SECURITY.md` (what is and is not protected, memory and
|
||||
filesystem guarantees), `docs/COMPATIBILITY.md` (the formats),
|
||||
`docs/MIGRATION.md` (what changed from the Go version) and
|
||||
`docs/BENCHMARKS.md`.
|
||||
|
||||
## User guide
|
||||
|
||||
@@ -82,12 +102,13 @@ vault lock # forget it now
|
||||
Commands that change the vault take an exclusive lock on `DIR/.lock`, and
|
||||
commands that read it take a shared one. A second process waits, and says
|
||||
so, until the first finishes. The operating system releases the lock if a
|
||||
process dies. Plan 9 and WebAssembly have no file locks, so there the lock
|
||||
file itself is the lock. After a crash on those platforms you remove it by
|
||||
hand; the error message names it.
|
||||
process dies; the locks are the same ones the Go version used, so old and
|
||||
new versions exclude each other.
|
||||
|
||||
Adding a name that already exists replaces it. `get` refuses to overwrite an
|
||||
existing file and writes nothing until the whole object has authenticated.
|
||||
Adding a name that already exists replaces it. `get` never replaces an
|
||||
existing file (or symlink) and writes nothing until the whole file has
|
||||
authenticated. `add URL` and `mirror` have size and time limits; see
|
||||
`vault(1)`.
|
||||
|
||||
`--key PATH` and `--vault DIR` work with every command. They default to
|
||||
`$VAULT_KEY` and `$VAULT_DIR`, then to `~/.vault/key` and `~/.vault/store`.
|
||||
@@ -103,7 +124,8 @@ over a pipe and never touches the disk; the helper's memory is locked
|
||||
against swapping, it cannot be core-dumped (or, on Linux, debugged), and it
|
||||
answers only on a socket in a directory private to your user. While it is
|
||||
unlocked, anything running as you can use the vault, decoy or not, so lock
|
||||
before handing the machine over.
|
||||
before handing the machine over. If the helper cannot protect its memory it
|
||||
refuses the key. `unlock` is not available on Windows.
|
||||
|
||||
## Decoy passphrase
|
||||
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
# Security notes
|
||||
|
||||
What vault protects, how, and where the protection stops. See
|
||||
`docs/COMPATIBILITY.md` for the exact formats.
|
||||
|
||||
## What is protected
|
||||
|
||||
- **Contents, names, sizes, modes and times** of stored files, against
|
||||
anyone who reads the vault directory without the passphrase: everything is
|
||||
inside Fury files sealed to a hybrid FrodoKEM-640 + ML-KEM-768 key, so the
|
||||
file key stays secret unless both KEMs are broken.
|
||||
- **Structure**: records are one fixed size; objects are at most 4 MiB and
|
||||
padded to power-of-two buckets; object names derive from a secret per-file
|
||||
seed, so nothing on disk links a file's objects; every ciphertext and shard
|
||||
directory has its timestamps pinned to the epoch; all 256 shards exist from
|
||||
the start; `init` adds filler sealed to a discarded key.
|
||||
- **Integrity**: every byte that is acted on is authenticated (header MAC,
|
||||
STREAM tags with a final-chunk flag, no trailing data); an object starts
|
||||
with its own ID, so objects cannot be swapped, reordered or replayed under
|
||||
another record; `get` publishes output only after the whole file, padding
|
||||
and end of file included, has authenticated.
|
||||
- **Deniability**: the key file always has two equal slots, one possibly
|
||||
random; both are always tried; decoy and real files are indistinguishable
|
||||
from filler to the other passphrase.
|
||||
|
||||
## What is not protected
|
||||
|
||||
- The number and total size of ciphertexts, and when the directory changes,
|
||||
are visible to whoever stores it.
|
||||
- A passphrase is only as strong as it is: Argon2id (3 passes, 64 MiB, 4
|
||||
lanes) slows guessing, it does not prevent it.
|
||||
- Anyone who can run code as you while the vault is unlocked (or watch you
|
||||
type the passphrase) can read everything.
|
||||
- Nothing here hides that you use vault.
|
||||
|
||||
## Secrets in memory
|
||||
|
||||
Passphrases, seeds, private keys, key-encryption keys and plaintext buffers
|
||||
are held in owned types (`SecretArray`, `SecretVec`, `LockedBuf`) that are
|
||||
not `Clone`, `Copy`, `Debug` or serialisable, have fixed allocations that
|
||||
never reallocate, and are cleared with `zeroize` when dropped — on success,
|
||||
on error and while unwinding from a panic (release builds unwind).
|
||||
|
||||
What cannot be cleared, so is not claimed:
|
||||
|
||||
- copies the compiler leaves in registers or on the stack;
|
||||
- copies inside third-party key objects: `chacha20poly1305` cipher states
|
||||
(the key schedule), `ml-kem`'s `Seed` passed by value (the decapsulation
|
||||
key itself is zeroized on drop with the crate's `zeroize` feature),
|
||||
`hmac`/`hkdf` internal states, `argon2` internals other than its working
|
||||
memory (which vault allocates and clears itself);
|
||||
- the passphrase in `$VAULT_PASSPHRASE` (it is in the process environment
|
||||
for the process lifetime) and in the terminal driver's buffers;
|
||||
- anything after `abort`, `kill -9`, a power cut, or once swapped out;
|
||||
- file data in the OS page cache, and plaintext temporary files on disk
|
||||
(below).
|
||||
|
||||
None of this is erasure from physical memory.
|
||||
|
||||
## The agent
|
||||
|
||||
`vault unlock` re-executes the binary as `vault __agent` with
|
||||
`VAULT_PASSPHRASE` and `VAULT_DECOY_PASSPHRASE` removed from its environment,
|
||||
no secret in its arguments, its working directory `/` and standard error
|
||||
closed. It hands over the 34-byte suite ID and seed through a pipe. Before
|
||||
reading it (straight from the pipe into the locked page, with no
|
||||
intermediate buffer), the agent:
|
||||
|
||||
1. sets the core-dump size limit to zero;
|
||||
2. on Linux, sets `PR_SET_DUMPABLE` to 0 (no core dumps, no same-user
|
||||
`ptrace`, `/proc/PID/mem` or `/proc/PID/environ`) and checks it took;
|
||||
3. maps a private page, `mlock`s it and, on Linux, marks it
|
||||
`MADV_DONTDUMP`, all checked.
|
||||
|
||||
Any failure is reported to `unlock` and the agent exits without the key;
|
||||
there is no fallback to unprotected memory. On other Unix systems (macOS,
|
||||
the BSDs, illumos) steps 1 and 3 apply; there is no portable equivalent of
|
||||
`PR_SET_DUMPABLE`, so another process of the same user may be able to read
|
||||
the agent's memory with a debugger.
|
||||
|
||||
The socket lives in a directory that must be owned by you with mode exactly
|
||||
0700 (refused otherwise); the socket is 0600, checked before readiness. Each
|
||||
request has a two-second deadline; the expiry deadline is enforced with a
|
||||
timed `poll`, so it does not depend on another client arriving, and a key
|
||||
is never handed out after expiry. On exit the agent clears the page and
|
||||
removes the socket only if it is still its own. `unlock` waits at most ten
|
||||
seconds for the agent to report and kills it otherwise.
|
||||
|
||||
Windows: the agent is not supported (`unlock` says so); every command asks
|
||||
for the passphrase.
|
||||
|
||||
## Filesystem safety
|
||||
|
||||
- Output is published with an atomic no-replace primitive
|
||||
(`renameat2(RENAME_NOREPLACE)`, `renameatx_np(RENAME_EXCL)`, `link`+`unlink`,
|
||||
or `MoveFileExW` without replace), so an existing file, symlink (dangling
|
||||
or not) or one created concurrently is never replaced; only vault's own
|
||||
temporary file is removed. There is no check-then-rename fallback.
|
||||
- The temporary output file is mode 0600 until the content has fully
|
||||
authenticated; only then are the recorded mode and times applied.
|
||||
- `get -r` opens each directory below the chosen destination with
|
||||
`O_NOFOLLOW|O_DIRECTORY` relative to its parent, so a symlink planted
|
||||
inside the destination cannot redirect files elsewhere. On Windows, which
|
||||
has no `openat`, each component is checked not to be a link or junction
|
||||
just before use; a concurrent swap between check and use is not excluded.
|
||||
- Writes follow crash-safe order with explicit `fsync` of files and
|
||||
directories (Unix): objects before their record; a new record before the
|
||||
removal of the one it replaces; a record's removal before its objects'.
|
||||
A failure after a record is already in place never deletes its objects.
|
||||
Windows has no directory fsync through the APIs used; NTFS journals
|
||||
metadata.
|
||||
- `migrate` checks an original is still the file it stored (device, inode,
|
||||
size, time; on Windows only size and time) and then deletes it by name; a
|
||||
replacement in the instant between the check and the deletion is not
|
||||
excluded. Paths inside the vault directory are refused outright.
|
||||
- A filesystem with neither `RENAME_NOREPLACE` nor hard links (some FUSE
|
||||
mounts, FAT) cannot publish without risking a replace, so `get` fails
|
||||
there with a message saying so rather than fall back.
|
||||
- The vault lock uses the same POSIX `fcntl` locks as the Go version, so
|
||||
mixed Go and Rust processes exclude each other. Only a read-only
|
||||
filesystem lets a reader proceed without the lock.
|
||||
|
||||
## Plaintext on disk
|
||||
|
||||
- `vault add URL` and `vault mirror` stream non-HTML downloads to a 0600
|
||||
temporary file in `$TMPDIR` before storing them, because the size must be
|
||||
known first. On Unix the file is unlinked as soon as it is created, so it
|
||||
has no name and disappears even if vault is killed; on other systems it
|
||||
is a named file removed on every normal path, including errors. HTML and
|
||||
CSS are rewritten in memory. `vault absorb` stores tar
|
||||
entries straight from the stream with no temporary file.
|
||||
- `vault get` writes plaintext where you ask; `vault migrate` deletes the
|
||||
originals with `unlink`, which is not secure deletion.
|
||||
|
||||
## Network
|
||||
|
||||
Downloads verify TLS certificates (rustls with the platform verifier; it is
|
||||
never disabled), follow at most 10 redirects (http and https only; for
|
||||
`mirror`, only to the same host), and are bounded by connect, response and
|
||||
whole-request deadlines and a byte budget counted after decompression. See
|
||||
`vault(1)` for the defaults and flags.
|
||||
|
||||
## Cryptographic implementations
|
||||
|
||||
FrodoKEM-640-SHAKE is a port of CIRCL v1.6.4's implementation (constant-time
|
||||
in the same places: no secret-dependent branches or indices; the
|
||||
re-encryption check selects the rejection key with a mask). It reproduces
|
||||
CIRCL's NIST KAT digest. ML-KEM-768 (`ml-kem`), ChaCha20-Poly1305 and
|
||||
XChaCha20-Poly1305 (`chacha20poly1305`), Argon2id (`argon2`), HKDF, HMAC,
|
||||
SHA-2 and SHA-3 are RustCrypto crates; TLS is rustls with `ring`. A Rust
|
||||
rewrite is not stronger cryptography: the algorithms and parameters are
|
||||
unchanged.
|
||||
|
||||
## Reporting
|
||||
|
||||
Please report vulnerabilities privately to andy@dixon.cx.
|
||||
@@ -0,0 +1,25 @@
|
||||
use chacha20poly1305::aead::{AeadInOut, KeyInit};
|
||||
use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce};
|
||||
use std::time::Instant;
|
||||
fn main() {
|
||||
let c = ChaCha20Poly1305::new(&Key::from([1u8; 32]));
|
||||
let mut buf = vec![0u8; 64 * 1024];
|
||||
let n = 1024; // 64 MiB
|
||||
let t = Instant::now();
|
||||
for i in 0..n {
|
||||
let mut nb = [0u8; 12];
|
||||
nb[3..11].copy_from_slice(&(i as u64).to_be_bytes());
|
||||
std::hint::black_box(c.encrypt_inout_detached(&Nonce::from(nb), &[], buf.as_mut_slice().into()).unwrap());
|
||||
}
|
||||
let s = t.elapsed().as_secs_f64();
|
||||
println!("raw seal: {:.0} MB/s", (n * 65536) as f64 / s / 1e6);
|
||||
let key = [1u8; 32];
|
||||
let data = vec![0u8; 64 << 20];
|
||||
let t = Instant::now();
|
||||
let mut out = Vec::with_capacity(data.len() + (1 << 20));
|
||||
vault::crypto::stream::seal_all(&key, &data, &mut out);
|
||||
println!("seal_all: {:.0} MB/s", data.len() as f64 / t.elapsed().as_secs_f64() / 1e6);
|
||||
let t = Instant::now();
|
||||
let n = vault::crypto::stream::open_all(&key, &mut out).unwrap();
|
||||
println!("open_all: {:.0} MB/s ({n})", data.len() as f64 / t.elapsed().as_secs_f64() / 1e6);
|
||||
}
|
||||
@@ -0,0 +1,79 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The Rust side of scripts/bench.sh: the same operations as the Go
|
||||
//! oracle's Benchmark* functions, timed as the median of repeated runs.
|
||||
//! Run with `cargo bench --bench micro`.
|
||||
|
||||
use std::hint::black_box;
|
||||
use std::io::{Read, Write};
|
||||
use std::time::{Duration, Instant};
|
||||
use vault::crypto::{fury, kem, stream};
|
||||
|
||||
fn pattern(n: usize, salt: u8) -> Vec<u8> {
|
||||
(0..n).map(|i| (i.wrapping_mul(7) as u8) ^ salt).collect()
|
||||
}
|
||||
|
||||
/// Runs `f` until about two seconds have passed (at least 5 times) and
|
||||
/// prints the median and the spread.
|
||||
fn bench(name: &str, bytes: usize, mut f: impl FnMut()) {
|
||||
f(); // warm up
|
||||
let mut times = Vec::new();
|
||||
let start = Instant::now();
|
||||
while times.len() < 5 || (start.elapsed() < Duration::from_secs(2) && times.len() < 10_000) {
|
||||
let t = Instant::now();
|
||||
f();
|
||||
times.push(t.elapsed());
|
||||
}
|
||||
times.sort();
|
||||
let med = times[times.len() / 2];
|
||||
let (lo, hi) = (times[times.len() / 10], times[times.len() * 9 / 10]);
|
||||
let rate = if bytes > 0 {
|
||||
format!(" {:.0} MB/s", bytes as f64 / med.as_secs_f64() / 1e6)
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
println!(
|
||||
"{name:<24} median {:>10.3} ms (p10 {:.3}, p90 {:.3}, n={}){rate}",
|
||||
med.as_secs_f64() * 1e3,
|
||||
lo.as_secs_f64() * 1e3,
|
||||
hi.as_secs_f64() * 1e3,
|
||||
times.len()
|
||||
);
|
||||
}
|
||||
|
||||
fn main() {
|
||||
let seed = pattern(32, 1);
|
||||
bench("derive", 0, || {
|
||||
black_box(kem::Identity::from_suite_seed(kem::SUITE_ID, &seed).unwrap());
|
||||
});
|
||||
let id = kem::Identity::from_suite_seed(kem::SUITE_ID, &seed).unwrap();
|
||||
let r = id.recipient();
|
||||
let fk: [u8; 32] = pattern(32, 2).try_into().unwrap();
|
||||
bench("wrap", 0, || {
|
||||
black_box(r.wrap(&fk, &mut vault::rng::os()));
|
||||
});
|
||||
let st = r.wrap(&fk, &mut vault::rng::os());
|
||||
bench("unwrap", 0, || {
|
||||
black_box(id.unwrap(std::slice::from_ref(&st)).unwrap());
|
||||
});
|
||||
let pt = pattern(4 << 20, 3);
|
||||
bench("object 4MiB", pt.len(), || {
|
||||
let mut ct = Vec::new();
|
||||
fury::encrypt_to(&mut ct, &r, &pt, &mut vault::rng::os());
|
||||
black_box(fury::decrypt_bounded(&ct[..], &id, pt.len()).unwrap());
|
||||
});
|
||||
let key = [0u8; 32];
|
||||
let big = vec![0u8; 64 << 20];
|
||||
bench("stream 64MiB", big.len(), || {
|
||||
let mut w = stream::StreamWriter::new(&key, Vec::with_capacity(big.len() + (1 << 16)));
|
||||
w.write_all(&big).unwrap();
|
||||
let ct = w.finish().unwrap();
|
||||
let mut out = Vec::with_capacity(big.len());
|
||||
stream::StreamReader::new(&key, &ct[..]).read_to_end(&mut out).unwrap();
|
||||
black_box(out);
|
||||
});
|
||||
let body = vault::keyfile::new_body(b"bench", b"", &mut vault::rng::os());
|
||||
bench("unlock", 0, || {
|
||||
black_box(vault::keyfile::unlock_body(&body, b"bench").unwrap());
|
||||
});
|
||||
}
|
||||
@@ -1,48 +1,135 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build vault for every OS and CPU architecture Go can target without cgo.
|
||||
# Build vault for the supported operating systems and CPU architectures.
|
||||
#
|
||||
# ./build.sh # everything, into ./dist
|
||||
# ./build.sh # every target this host can build, into ./dist
|
||||
# TARGETS="linux/amd64 darwin/arm64" ./build.sh
|
||||
# OUT=/tmp/release ./build.sh
|
||||
# OUT=/tmp/release JOBS=4 ./build.sh
|
||||
#
|
||||
# Binaries are static, stripped and reproducible (CGO_ENABLED=0, -trimpath).
|
||||
# iOS and most Android targets are skipped (they need cgo); see below.
|
||||
# 32-bit ARM Linux is built twice: armv6 (Raspberry Pi Zero/1) and armv7.
|
||||
# SHA256SUMS is written next to the binaries.
|
||||
# Binaries keep the Go-era names, vault-OS-ARCH[.exe], and are stripped
|
||||
# release builds from the pinned toolchain (rust-toolchain.toml) and the
|
||||
# locked dependencies (Cargo.lock), with build paths remapped so the output
|
||||
# does not depend on where the tree lives. Linux binaries are static (musl)
|
||||
# except s390x, which links glibc dynamically; static linking is
|
||||
# checked. SHA256SUMS is written next to the binaries.
|
||||
#
|
||||
# How each target is built, and what the host needs:
|
||||
# zig cargo-zigbuild and zig on PATH (Linux, Windows GNU ABI)
|
||||
# mac a Mac with Xcode (macOS)
|
||||
# cross `cross` and Docker (FreeBSD)
|
||||
# A requested target the host cannot build is reported, the others are still
|
||||
# built, and the script exits non-zero. See docs/PLATFORMS.md for the Go
|
||||
# targets with no Rust equivalent; asking for one is an error.
|
||||
#
|
||||
# JOBS builds several targets at once, each in its own target directory
|
||||
# under target/build-sh (default 1: one at a time, cargo using every CPU).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
root=$PWD
|
||||
|
||||
table() {
|
||||
# os/arch Rust target method
|
||||
cat <<-'EOF'
|
||||
linux/amd64 x86_64-unknown-linux-musl zig
|
||||
linux/arm64 aarch64-unknown-linux-musl zig
|
||||
linux/386 i686-unknown-linux-musl zig
|
||||
linux/armv6 arm-unknown-linux-musleabihf zig
|
||||
linux/armv7 armv7-unknown-linux-musleabihf zig
|
||||
linux/riscv64 riscv64gc-unknown-linux-musl zig
|
||||
linux/loong64 loongarch64-unknown-linux-musl zig
|
||||
linux/ppc64le powerpc64le-unknown-linux-musl zig
|
||||
linux/s390x s390x-unknown-linux-gnu zig
|
||||
windows/amd64 x86_64-pc-windows-gnu zig
|
||||
windows/386 i686-pc-windows-gnu zig
|
||||
windows/arm64 aarch64-pc-windows-gnullvm zig
|
||||
darwin/amd64 x86_64-apple-darwin mac
|
||||
darwin/arm64 aarch64-apple-darwin mac
|
||||
freebsd/amd64 x86_64-unknown-freebsd cross
|
||||
freebsd/386 i686-unknown-freebsd cross
|
||||
EOF
|
||||
}
|
||||
|
||||
# Go-era targets no longer released (the release covers Linux, Windows,
|
||||
# macOS and FreeBSD). See docs/PLATFORMS.md for why each was dropped.
|
||||
unsupported="aix/ppc64 android/arm64 dragonfly/amd64 freebsd/arm freebsd/arm64 freebsd/riscv64
|
||||
illumos/amd64 js/wasm linux/mips linux/mipsle linux/mips64 linux/mips64le linux/ppc64
|
||||
netbsd/386 netbsd/amd64 netbsd/arm netbsd/arm64 openbsd/386 openbsd/amd64 openbsd/arm
|
||||
openbsd/arm64 openbsd/ppc64 openbsd/riscv64 plan9/386 plan9/amd64 plan9/arm solaris/amd64 wasip1/wasm"
|
||||
|
||||
out=${OUT:-dist}
|
||||
# iOS and 32-bit/x86 Android need cgo and a vendor SDK. android/arm64 builds
|
||||
# a static binary that runs in Termux.
|
||||
targets=${TARGETS:-$(go tool dist list | grep -vE '^ios/|^android/(386|amd64|arm)$')}
|
||||
jobs=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}
|
||||
jobs=${JOBS:-1}
|
||||
targets=${TARGETS:-$(table | awk '{print $1}')}
|
||||
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
# Never clean anything but our own output, and never in a directory that
|
||||
# matters: OUT must be a dedicated directory, not /, $HOME or the tree.
|
||||
mkdir -p -- "$out"
|
||||
out=$(cd -- "$out" && pwd -P)
|
||||
case $out in
|
||||
/ | "$(cd ~ && pwd -P)" | "$root") echo "refusing OUT=$out: use a dedicated directory" >&2; exit 2 ;;
|
||||
esac
|
||||
rm -f -- "$out"/vault-* "$out"/SHA256SUMS
|
||||
|
||||
export CARGO_TARGET_DIR_BASE=$root/target/build-sh
|
||||
export RUSTFLAGS="${RUSTFLAGS:-} --remap-path-prefix=$root=vault --remap-path-prefix=${CARGO_HOME:-$HOME/.cargo}=cargo"
|
||||
export OUT_DIR=$out JOBS_N=$jobs HOST_OS=$(uname -s) root
|
||||
logs=$root/target/build-sh-logs
|
||||
mkdir -p -- "$logs"
|
||||
export logs
|
||||
|
||||
build_one() {
|
||||
local os=${1%/*} arch=${1#*/} variants=("") name ext=""
|
||||
[[ $os == windows ]] && ext=.exe
|
||||
[[ $arch == wasm ]] && ext=.wasm
|
||||
[[ $os/$arch == linux/arm ]] && variants=(6 7)
|
||||
for v in "${variants[@]}"; do
|
||||
name=vault-$os-$arch${v:+v$v}$ext
|
||||
if ! GOOS=$os GOARCH=$arch GOARM=${v:-} CGO_ENABLED=0 \
|
||||
go build -trimpath -buildvcs=false -ldflags='-s -w -buildid=' -o "$OUT_DIR/$name" . 2>"$OUT_DIR/$name.log"; then
|
||||
echo "FAILED $name" >&2
|
||||
cat "$OUT_DIR/$name.log" >&2
|
||||
return 1
|
||||
local want=$1 row os triple method name ext="" tdir bin
|
||||
row=$(table | awk -v t="$want" '$1 == t')
|
||||
if [[ -z $row ]]; then
|
||||
if grep -qw -- "$want" <<<"$unsupported"; then
|
||||
echo "UNSUPPORTED $want: not released; see docs/PLATFORMS.md" >&2
|
||||
else
|
||||
echo "UNKNOWN $want" >&2
|
||||
fi
|
||||
rm "$OUT_DIR/$name.log"
|
||||
echo "built $name"
|
||||
done
|
||||
return 1
|
||||
fi
|
||||
read -r _ triple method <<<"$row"
|
||||
os=${want%/*}
|
||||
[[ $os == windows ]] && ext=.exe
|
||||
name=vault-${want/\//-}$ext
|
||||
tdir=$root/target
|
||||
[[ $JOBS_N != 1 ]] && tdir=$CARGO_TARGET_DIR_BASE/$triple
|
||||
case $method in
|
||||
zig)
|
||||
command -v cargo-zigbuild >/dev/null && command -v zig >/dev/null ||
|
||||
{ echo "SKIPPED $want: needs cargo-zigbuild and zig" >&2; return 1; }
|
||||
cmd=(cargo zigbuild) ;;
|
||||
mac)
|
||||
[[ $HOST_OS == Darwin ]] || { echo "SKIPPED $want: needs a Mac with Xcode" >&2; return 1; }
|
||||
cmd=(cargo build) ;;
|
||||
cross)
|
||||
command -v cross >/dev/null && command -v docker >/dev/null ||
|
||||
{ echo "SKIPPED $want: needs cross and docker" >&2; return 1; }
|
||||
cmd=(cross build) ;;
|
||||
esac
|
||||
local flags=$RUSTFLAGS
|
||||
# zig's musl getrandom() never reaches the kernel on riscv64 (it spins);
|
||||
# make the getrandom crate issue the system call itself.
|
||||
[[ $triple == riscv64*-musl ]] && flags+=' --cfg getrandom_backend="linux_raw"'
|
||||
if ! RUSTFLAGS=$flags CARGO_TARGET_DIR=$tdir "${cmd[@]}" --release --locked --target "$triple" >"$logs/$name.log" 2>&1; then
|
||||
echo "FAILED $want ($triple)" >&2
|
||||
tail -20 "$logs/$name.log" >&2
|
||||
return 1
|
||||
fi
|
||||
bin=$tdir/$triple/release/vault$ext
|
||||
cp -- "$bin" "$OUT_DIR/$name"
|
||||
if [[ $triple == *-musl* ]] && command -v file >/dev/null; then
|
||||
file -L "$OUT_DIR/$name" | grep -q 'static' || { echo "FAILED $want: not statically linked" >&2; return 1; }
|
||||
fi
|
||||
echo "built $name ($triple)"
|
||||
}
|
||||
export -f build_one
|
||||
export OUT_DIR=$out
|
||||
export -f build_one table
|
||||
export unsupported
|
||||
|
||||
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}'
|
||||
status=0
|
||||
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}' || status=1
|
||||
|
||||
cd "$out"
|
||||
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
|
||||
echo "$(ls vault-* | wc -l | tr -d ' ') binaries in $out"
|
||||
cd -- "$out"
|
||||
if ls vault-* >/dev/null 2>&1; then
|
||||
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
|
||||
fi
|
||||
echo "$(ls vault-* 2>/dev/null | wc -l | tr -d ' ') binaries in $out"
|
||||
exit $status
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# cargo-deny policy: run `cargo deny --locked check`.
|
||||
|
||||
[graph]
|
||||
all-features = true
|
||||
|
||||
[advisories]
|
||||
# Every advisory is an error unless listed here with a reason.
|
||||
ignore = []
|
||||
|
||||
[licenses]
|
||||
# Licences compatible with distributing vault under GPL-3.0-or-later.
|
||||
allow = [
|
||||
"Apache-2.0",
|
||||
"Apache-2.0 WITH LLVM-exception",
|
||||
"BSD-3-Clause",
|
||||
"CDLA-Permissive-2.0",
|
||||
"GPL-3.0-or-later",
|
||||
"ISC",
|
||||
"MIT",
|
||||
"Unicode-3.0",
|
||||
"Unlicense",
|
||||
"Zlib",
|
||||
]
|
||||
confidence-threshold = 0.9
|
||||
|
||||
[bans]
|
||||
multiple-versions = "warn"
|
||||
wildcards = "deny"
|
||||
|
||||
[sources]
|
||||
unknown-registry = "deny"
|
||||
unknown-git = "deny"
|
||||
allow-registry = ["https://github.com/rust-lang/crates.io-index"]
|
||||
@@ -0,0 +1,85 @@
|
||||
# Benchmarks: Go 0.5 against Rust 0.6
|
||||
|
||||
Measured on 2026-10-08 on one machine: Intel Core i7-7700 (4 cores, 8
|
||||
threads), 31 GiB RAM, ext4 on a two-disk RAID 1 (`/dev/md2`), Linux 7.0.
|
||||
Both binaries were built from this tree: the Go oracle (`oracle/go`, Go
|
||||
1.27.1, `CGO_ENABLED=0`) and the Rust release build (Rust 1.99.0, LTO).
|
||||
|
||||
**Limitation:** the machine was shared. Other users' Docker builds kept the
|
||||
load average between 4 and 12 throughout, so absolute times are noisy.
|
||||
Go and Rust were run alternately under the same conditions, and every
|
||||
figure is a median with its spread, but small differences mean nothing.
|
||||
Page caches were warm (dropping them needs root), so cold-cache numbers
|
||||
were not measured.
|
||||
|
||||
## Primitives
|
||||
|
||||
`cargo bench --bench micro` and
|
||||
`(cd oracle/go && go test -run XXX -bench Oracle -benchtime 2s ./ ./internal/pqcrypt ./internal/stream)`,
|
||||
run alternately twice (load 4.4 to 5.4). Medians in milliseconds; the Rust
|
||||
harness's p10 to p90 range is in brackets.
|
||||
|
||||
| Operation | Go | Rust |
|
||||
| --- | --- | --- |
|
||||
| Derive hybrid identity from seed | 5.6 | 3.0 (2.9 to 4.6) |
|
||||
| Wrap file key (both encapsulations) | 7.4 to 8.0 | 3.1 (3.0 to 4.8) |
|
||||
| Unwrap file key (both decapsulations) | 7.6 to 8.3 | 3.1 (3.0 to 3.6) |
|
||||
| Encrypt and decrypt one 4 MiB object | 19.9 to 20.1 | 19.0 to 19.5 |
|
||||
| STREAM seal and open, 64 MiB, streaming API | 89 to 92 | 187 to 188 |
|
||||
| Unlock key file (Argon2id for both slots) | 121 to 125 | 129 (125 to 147) |
|
||||
|
||||
- The FrodoKEM port is about twice as fast as CIRCL's Go code, mostly from
|
||||
multiplying A row by row instead of walking it by column.
|
||||
- ChaCha20-Poly1305 is slower: RustCrypto reaches about 1.2 GB/s on this CPU
|
||||
(`cargo bench --bench aead_probe`), Go's assembly about twice that. The
|
||||
64 MiB row uses the streaming reader and writer, which copy through
|
||||
64 KiB buffers. The vault itself uses the in-memory path (`seal_all`
|
||||
0.85 GB/s, `open_all` 1.4 GB/s), and per 4 MiB object the two are
|
||||
level because the KEM work is cheaper.
|
||||
- Unlock takes 180 ms with Argon2id lanes computed in turn. Computing the
|
||||
four lanes in parallel (argon2's `parallel` feature) brings it to 129 ms,
|
||||
close to Go. The parameters are unchanged.
|
||||
|
||||
## End to end
|
||||
|
||||
`RUNS=3 BIG_MB=1024 SMALL=500 scripts/bench.sh` (load 8 to 13). Every step
|
||||
includes unlocking the key. The 1 GiB file and the 500 small files (1 to
|
||||
16 KiB each) are each added to a fresh vault. The reads use one vault
|
||||
holding both, plus filler. Times are medians of 3 (min to max).
|
||||
|
||||
| Step | Go | Rust |
|
||||
| --- | --- | --- |
|
||||
| add 1 GiB file | 41.7 s (32.0 to 50.3) | 29.6 s (28.5 to 31.0) |
|
||||
| add 500 small files | 123.9 s (113.3 to 156.2) | 79.0 s (77.3 to 133.3) |
|
||||
| get 1 GiB file | 9.8 s (8.8 to 10.0) | 16.2 s (15.5 to 16.7) |
|
||||
| list (about 530 records) | 4.95 s (4.89 to 5.20) | 0.74 s (0.71 to 0.77) |
|
||||
| get one small file | 5.24 s (4.88 to 6.00) | 0.74 s (0.70 to 0.81) |
|
||||
| Peak RSS | 137 to 140 MiB | 132 MiB |
|
||||
| Peak threads | 18 | 17 |
|
||||
| Binary size | 11.5 MB | 3.5 MB (static musl: 3.4 MB) |
|
||||
|
||||
- **Catalogue reads** (`list`, and every command that looks a name up) are
|
||||
about 6 times faster. Records are decrypted in parallel on the worker pool
|
||||
(bounded at twice the worker count in flight), and each FrodoKEM
|
||||
decapsulation is cheaper.
|
||||
- **Adds** are faster. Both implementations fsync every object, its shard
|
||||
directory and every record. Rust overlaps up to eight objects' encryption
|
||||
and syncs on the pool.
|
||||
- **`get` of a large file is slower, on purpose.** Rust fsyncs the output
|
||||
file before giving it its name, so a crash just after `get` cannot leave
|
||||
the name pointing at an empty or partial file. Go did not. On an idle
|
||||
rerun (load about 8, output on the same disk), Go took 3.6 s and Rust
|
||||
15.5 s. Of Rust's time, 13.3 s was the single `fsync` of 1 GiB of
|
||||
freshly written data (measured with `strace -T`). The other 2 s or so is
|
||||
unlock, decryption and writing. This is the cost of the
|
||||
durability barrier on this RAID 1 array, and it scales with the disk's
|
||||
write speed.
|
||||
- Peak memory is dominated by Argon2id: 64 MiB for each of the two slots,
|
||||
computed at the same time in both implementations. Rust's object pipeline
|
||||
adds at most eight 4 MiB plaintext buffers and their ciphertexts. Peak
|
||||
threads: Rust has 8 workers, 8 Argon2 threads while unlocking, and the
|
||||
main thread.
|
||||
|
||||
Reproduce with `scripts/bench.sh` (it builds both binaries, creates
|
||||
synthetic data in `target/bench`, and prints the tables' numbers) and the
|
||||
two micro-benchmark commands above.
|
||||
@@ -0,0 +1,187 @@
|
||||
# Compatibility specification
|
||||
|
||||
This is the on-disk and on-wire contract between the Go implementation
|
||||
(frozen in `oracle/go`, commit `4c1507c`) and the Rust implementation. It is
|
||||
derived from the Go sources and pinned by vectors the Go oracle generated
|
||||
(`tests/vectors`, regenerated with `oracle/gen-vectors.sh`). The Rust port is
|
||||
an implementation migration, not a format migration: nothing here changes,
|
||||
no data is rekeyed or converted, and no new suite becomes the default.
|
||||
|
||||
Each section names the tests that hold the Rust code to it. "Go" below always
|
||||
means the oracle.
|
||||
|
||||
## 1. Key file
|
||||
|
||||
| Item | Value |
|
||||
| --- | --- |
|
||||
| Encoding | one PEM block, type `VAULT KEY`, standard base64 with padding, 64 columns, as Go's `pem.Encode` |
|
||||
| Body | exactly 198 bytes: an 18-byte header and two 90-byte slots |
|
||||
| Header | `VAULTKEY1`, Argon2id time `3` (u32 BE), memory `65536` KiB (u32 BE), lanes `4` (u8) |
|
||||
| Slot | 16-byte salt, 24-byte XChaCha20-Poly1305 nonce, 34 bytes sealed (u16 BE suite ID, 32-byte master seed), 16-byte tag |
|
||||
| Slot key | Argon2id v1.3 (passphrase bytes as given, no normalisation, salt, t=3, m=64 MiB, p=4, 32 bytes) |
|
||||
| Associated data | the 18-byte header |
|
||||
| Second slot | a decoy identity's slot, or 90 random bytes; which slot is real is random |
|
||||
| Unlock | Argon2id runs for both slots every time (Rust runs them concurrently); the first slot that opens wins; "incorrect passphrase" if neither |
|
||||
|
||||
Accepted framing: optional text before the block (Go's `pem.Decode`), only
|
||||
whitespace after it, header lines inside the block ignored. Files over 64 KiB
|
||||
are refused outright (Go would read them, then reject them).
|
||||
|
||||
Tests: `keyfile::tests::*` (Argon2id vectors, Go-written key files with and
|
||||
without decoy open to the Go-recorded seeds, byte-identical PEM re-encoding,
|
||||
malformed framing, NFC/NFD passphrases differ).
|
||||
|
||||
## 2. Hybrid suite `0x0001`, tag `frodo640shake+mlkem768`
|
||||
|
||||
- Master seed: 32 bytes. Expansion:
|
||||
`HKDF-SHA-256(ikm = seed, salt = none, info = "fury/v1 seed-expand frodo640shake+mlkem768")`,
|
||||
read as 48 bytes of FrodoKEM seed then 64 bytes of ML-KEM seed.
|
||||
- FrodoKEM-640-SHAKE exactly as CIRCL v1.6.4 `kem/frodo/frodo640shake`
|
||||
(the round-3 FrodoKEM, not the ISO/eFrodoKEM variants): key seed
|
||||
`s(16) || seedSE(16) || z(16)`; public key 9616 bytes (`seedA || pack(B)`);
|
||||
private key 19888 bytes (`s || pk || S as u16 LE || H(pk)`); ciphertext 9720
|
||||
bytes; 16-byte shared secret; 16-byte encapsulation seed `mu`; implicit
|
||||
rejection returns `SHAKE128(ct || s)`.
|
||||
- ML-KEM-768 (FIPS 203) with seed `d || z`; encapsulation key 1184 bytes,
|
||||
expanded decapsulation key 2400 bytes, ciphertext 1088 bytes, 32-byte
|
||||
shared secret, implicit rejection.
|
||||
- Public key: FrodoKEM public key `||` ML-KEM encapsulation key (10800 bytes).
|
||||
Private key (for comparison only): FrodoKEM private `||` ML-KEM expanded.
|
||||
- Wrap: fresh, independent randomness for both encapsulations every time.
|
||||
`KEK = HKDF-SHA-256(ikm = ss1 || ss2, salt = none, info = "fury/v1 kek frodo640shake+mlkem768" || ct1 || ct2)`;
|
||||
the 32-byte file key is sealed with ChaCha20-Poly1305 under the KEK and an
|
||||
all-zero 12-byte nonce. The zero nonce is safe only because each KEK is new
|
||||
and used once; the Rust code keeps that sealing function private to the
|
||||
suite.
|
||||
- Stanza: type `frodo640shake+mlkem768`, two arguments
|
||||
`base64(ct1)`, `base64(ct2)`, body = 48-byte sealed file key.
|
||||
- Unwrap: a stanza of another type is "not mine"; wrong argument count,
|
||||
bad base64 or wrong ciphertext lengths are fatal for the file; an AEAD
|
||||
failure (wrong key) is "not mine". The first stanza that opens wins.
|
||||
|
||||
Rust implementation: FrodoKEM is a line-by-line port of CIRCL
|
||||
(`src/crypto/frodo.rs`) because no maintained Rust crate implements CIRCL's
|
||||
variant (RustCrypto `frodo-kem` 0.1 implements the salted ISO version with
|
||||
9752-byte ciphertexts). ML-KEM is RustCrypto `ml-kem` 0.3.2.
|
||||
|
||||
Tests: `crypto::frodo::tests::nist_kat_digest` (reproduces the SHA-256
|
||||
CIRCL checks against the FrodoKEM reference KAT file),
|
||||
`crypto::kem::tests::{frodo_matches_circl, mlkem_matches_circl,
|
||||
hybrid_derivation_matches_go, wrap_matches_go_and_unwraps,
|
||||
fresh_wraps_differ_and_foreign_keys_do_not_match}`.
|
||||
|
||||
## 3. Fury header
|
||||
|
||||
```text
|
||||
fury-encryption.org/v1\n
|
||||
-> TYPE ARG ARG\n per stanza; TYPE and ARGs are 1+ bytes of 0x21-0x7e
|
||||
BODY, 64 base64 columns per line, ending with a line shorter than 64
|
||||
--- MAC\n MAC = base64 of 32 bytes
|
||||
```
|
||||
|
||||
- Base64: standard alphabet, no padding, non-zero trailing bits rejected,
|
||||
CR or LF anywhere in an encoded value rejected.
|
||||
- A body whose encoding fills its last line (the 48-byte wrapped key encodes
|
||||
to exactly 64 characters) is followed by an empty line; serialisation is
|
||||
canonical and Rust reproduces Go's bytes exactly.
|
||||
- Limits: 1 MiB per stanza (all its lines), 16 MiB per header; a line must
|
||||
end in LF; three bytes of look-ahead decide between a stanza and the
|
||||
footer; the payload starts immediately after the footer's LF, with any
|
||||
read-ahead returned first.
|
||||
- MAC: `HMAC-SHA-256(key = HKDF-SHA-256(file key, salt none, info "header"), message = header up to and including "---")`,
|
||||
compared in constant time.
|
||||
- Payload: 16-byte nonce, then STREAM keyed by
|
||||
`HKDF-SHA-256(file key, salt = nonce, info "payload")`.
|
||||
|
||||
Tests: `crypto::fury::tests::{header_vectors_match_go,
|
||||
header_parser_agrees_with_go, decrypts_go_files}` (the parser's accept or
|
||||
reject verdict, and the payload bytes left after the header, match Go on
|
||||
every case in `crypto.json`), `crypto::header::tests::*`,
|
||||
`tests/interop_crypto.rs` (Go decrypts files Rust wrote, and rejects
|
||||
truncated, extended and corrupted ones).
|
||||
|
||||
## 4. STREAM
|
||||
|
||||
- 64 KiB plaintext chunks; ChaCha20-Poly1305; 16-byte tag per chunk.
|
||||
- Nonce: 12 bytes, bytes 3..11 the big-endian chunk index, last byte `1`
|
||||
on the final chunk only, `0` otherwise.
|
||||
- Only the final chunk may be short; it may be empty only when it is the
|
||||
only chunk (empty plaintext). A plaintext that is an exact multiple of
|
||||
64 KiB ends with a full-length final chunk.
|
||||
- Rejected: no final chunk (truncation), an empty final chunk after
|
||||
others, bytes after the final chunk, any chunk failing authentication,
|
||||
a payload length no STREAM encoding can have.
|
||||
- The index never wraps; Rust refuses to go past 2^64 − 1 chunks (Go's
|
||||
counter is 88 bits; neither limit is reachable in practice).
|
||||
- Random access (`StreamReaderAt`) authenticates the final chunk when
|
||||
constructed, then any chunk on demand.
|
||||
|
||||
Tests: `crypto::stream::tests::*` (Go vectors for sizes around every
|
||||
boundary, any write/read step sizes, malformations rejected identically by
|
||||
the streaming, in-memory and random-access readers, concurrency of the
|
||||
random-access reader, counter overflow).
|
||||
|
||||
## 5. Store
|
||||
|
||||
- `DIR/.vault` contains exactly `VAULT 2\n`.
|
||||
- `DIR/records/xx/ID` and `DIR/objects/xx/ID`: `ID` is 16 bytes as
|
||||
lowercase hex, `xx` its first byte; all 256 shards of both exist after
|
||||
`init`. Entries whose names are not exactly that form, or are in the
|
||||
wrong shard, are ignored.
|
||||
- Every ciphertext, every shard and both top directories have access and
|
||||
modification times pinned to the Unix epoch; removals re-pin the shard.
|
||||
- Record: a Fury file whose plaintext is exactly 4096 bytes: a u32 BE JSON
|
||||
length (≤ 4092), the JSON, random padding. JSON is Go's `encoding/json`
|
||||
output for `{"name","seed","size","mode","modified","added"}`: HTML
|
||||
characters and U+2028/2029 escaped, times RFC 3339 with up to nine
|
||||
fractional digits, trailing zeros removed, `Z`. Rust writes it
|
||||
byte-identically.
|
||||
- Validation on read: name valid (below), seed 64 lowercase hex characters,
|
||||
size 0..=1 TiB, both times valid RFC 3339; unknown fields refused.
|
||||
- Names: non-empty UTF-8, ≤ 1024 bytes, already clean (Go `path.Clean`),
|
||||
relative, not `.`/`..`/`../…`, no backslash, no control characters.
|
||||
- Object IDs: `HMAC-SHA-256(key = seed, msg = index as big-endian i64)[:16]`.
|
||||
- Objects: plaintext = own ID (16 bytes), then up to `4 MiB − 16` file
|
||||
bytes, then random padding to the next power of two from 64 KiB to 4 MiB.
|
||||
An empty file still has one object. These sizes are plaintext; the
|
||||
ciphertext adds the header (~14.6 KB) and 16 bytes per 64 KiB.
|
||||
- Reading an object checks: it opens, its first 16 bytes are its ID, its
|
||||
length is exactly the padded size for its part, and the whole STREAM
|
||||
(final chunk, end of file) authenticates.
|
||||
- Records the key cannot open (no stanza matches) are skipped silently;
|
||||
ones that open but are malformed are reported and skipped.
|
||||
- Duplicates (an interrupted replace) are ordered newest `added` first; the
|
||||
next write of that name removes them all.
|
||||
- Filler: `init` writes 8–32 entries (or more, to reach `--filler`) of
|
||||
log-uniform sizes from 1 KiB to `clamp(target/8, 4 MiB, 1 GiB)`, sealed to
|
||||
a throwaway key, through the same code as `add`.
|
||||
|
||||
Tests: `store::types::tests::*` (names, object IDs, padded sizes, object
|
||||
counts, record JSON byte-identical to Go), `store::tests::reads_go_vault`
|
||||
(the oracle's fixture vault: real files, decoy file, foreign record),
|
||||
`store::tests::*`, `tests/cli.rs` lifecycle tests in both directions.
|
||||
|
||||
## 6. Locks
|
||||
|
||||
`DIR/.lock`, created mode 0600. Unix: POSIX `fcntl` record locks over the
|
||||
whole file (`F_SETLK` then `F_SETLKW`), shared for readers, exclusive for
|
||||
writers, released by the kernel when the process dies. Windows:
|
||||
`LockFileEx` on byte 0, length 1. Other systems: an `O_EXCL` lock file.
|
||||
Tests: `tests/cli.rs::lock_*` (Rust waits for Go, Go waits for Rust, both
|
||||
proceed after the holder is killed).
|
||||
|
||||
## 7. Agent
|
||||
|
||||
Socket `$XDG_RUNTIME_DIR` (else `$TMPDIR`, else `/tmp`) `/vault-UID/agent-`
|
||||
followed by the first 8 bytes of SHA-256 of the key file's absolute path
|
||||
(Go `filepath.Abs`, including its use of `$PWD`) in hex. The directory must
|
||||
be the user's own with mode exactly 0700. One request per connection: `g`
|
||||
→ 34 bytes (u16 BE suite ID, seed); `q` → the agent wipes the key and
|
||||
closes. Two-second deadline per request. Tests:
|
||||
`tests/cli.rs::agent_*` (Rust client with Go agent and the reverse).
|
||||
|
||||
## 8. What is deliberately not identical
|
||||
|
||||
See `docs/MIGRATION.md` for the full list of intentional behaviour changes.
|
||||
None of them alters a byte that is written to the vault or the key file
|
||||
beyond what the Go format already allows (fresh randomness, timestamps).
|
||||
@@ -0,0 +1,103 @@
|
||||
# Migrating from the Go implementation
|
||||
|
||||
vault 0.6 is a rewrite in Rust of the Go vault (0.5 and earlier). It is an
|
||||
implementation migration only: key files, vaults, the lock and the unlock
|
||||
agent are the same formats, byte for byte where the format is
|
||||
deterministic (see `COMPATIBILITY.md`). There is nothing to convert. Go and
|
||||
Rust builds can be used on the same vault, even at the same time: they
|
||||
honour each other's locks, and either can use an agent the other started.
|
||||
|
||||
To go back, install 0.5: everything 0.6 writes, 0.5 reads.
|
||||
|
||||
## Intentional behaviour changes
|
||||
|
||||
Safety and correctness:
|
||||
|
||||
- **`get` never replaces anything.** Output is published with an atomic
|
||||
no-replace rename or link, so a file, a symlink (even a dangling one) or a
|
||||
file created while `get` runs is left alone. Go checked first and then
|
||||
renamed, which a concurrent creator could race.
|
||||
- **`get -r` does not follow symlinks inside the output directory.** Each
|
||||
directory below the chosen one is opened without following links; a link
|
||||
or non-directory in the way stops the run. (Windows: checked just before
|
||||
use, see `SECURITY.md`.)
|
||||
- **Extracted files are synced** to disk before they appear under their
|
||||
name, and their directory afterwards.
|
||||
- **`get` needs an atomic no-replace primitive.** On a filesystem with
|
||||
neither `RENAME_NOREPLACE` nor hard links (some FUSE mounts, FAT) it fails
|
||||
with a message saying so, where Go would rename.
|
||||
- **`add`/`migrate` refuse paths inside the vault directory** (Go refused
|
||||
only the directory itself; `migrate store/records` would have deleted the
|
||||
vault's records).
|
||||
- **`absorb` refuses GNU sparse files** (Go expanded them), and honours PAX
|
||||
size records as Go does.
|
||||
- **A source file that changes while being added is not stored.** If it
|
||||
grows, shrinks, is modified or replaced during `add`/`migrate`, the
|
||||
command fails and nothing is stored. Go stored whatever it read (for a
|
||||
file that grew, a truncated copy).
|
||||
- **`migrate` deletes an original only if it is still the file stored**
|
||||
(same inode, size and time), after the record is durable.
|
||||
- **Deletes are durable in order.** Removing a file syncs the removal of its
|
||||
record before deleting its objects, so a crash cannot leave a record
|
||||
pointing at deleted objects. A failed sync after a record is already in
|
||||
place no longer deletes its objects.
|
||||
- **Lock errors are reported.** Go let a reader continue without the lock on
|
||||
any error; Rust does so only on a read-only filesystem.
|
||||
- **Stricter records.** A record must have every field, with exactly the
|
||||
field names Go writes, and nothing after the JSON. Go also accepted
|
||||
differently-cased names, missing fields (as zero) and trailing bytes;
|
||||
Go never writes such records, so no existing vault is affected.
|
||||
- **Key files over 64 KiB** are rejected without reading them whole.
|
||||
|
||||
Agent (`unlock`):
|
||||
|
||||
- The agent **refuses the key if it cannot protect it**: locking its memory,
|
||||
disabling core dumps and (Linux) becoming undumpable must all succeed. Go
|
||||
ignored failures there.
|
||||
- Its environment is stripped of `VAULT_PASSPHRASE` and
|
||||
`VAULT_DECOY_PASSPHRASE`; it runs in `/` with standard error closed.
|
||||
- Expiry no longer waits for the request in progress beyond its deadline,
|
||||
and a key is never handed out after expiry.
|
||||
- `unlock` waits at most ten seconds for the agent to start and kills it on
|
||||
failure; the agent removes its socket only if it is still its own.
|
||||
- **Windows: `unlock` is not available** (Go's agent there had no memory
|
||||
protection). Every command asks for the passphrase.
|
||||
|
||||
Network (`add URL`, `mirror`):
|
||||
|
||||
- Explicit limits, all adjustable: connect 30 s, response headers 60 s,
|
||||
whole request 2 h (`add`) or 5 min (`mirror`, per request), download size
|
||||
16 GiB (`add`) or 4 GiB in total (`mirror`), 10 redirects, 10 000 mirror
|
||||
pages, 16 MiB per HTML or CSS page. Bytes are counted after decompression.
|
||||
New flags `--max-bytes`, `--timeout`, `--max-pages`. Go had no limits.
|
||||
- Redirects go to http or https only; a `mirror` redirect to another host is
|
||||
skipped (Go followed it).
|
||||
- `mirror` resolves links against the final address after redirects and the
|
||||
page's `<base href>`; it parses HTML instead of matching a pattern, so it
|
||||
rewrites only `href`, `src`, `action` and `poster` on real tags, plus
|
||||
`url(...)` in CSS, `<style>` and `style=`, keeping every other byte and
|
||||
the original quoting. Text in scripts and comments is no longer touched.
|
||||
- `mirror` never lets two different URLs in one crawl overwrite each other's
|
||||
file; the second is skipped. It exits with status **3** when it stored
|
||||
some resources but skipped others (Go always exited 0).
|
||||
- Proxies follow `HTTPS_PROXY`/`HTTP_PROXY`/`NO_PROXY` as in Go; local
|
||||
addresses are never proxied.
|
||||
|
||||
Command line:
|
||||
|
||||
- New option `--workers N` (1–256) on every command.
|
||||
- `--key`/`--vault` no longer need a home directory when given explicitly.
|
||||
- Command-line arguments must be valid UTF-8 (Go accepted any bytes, but
|
||||
vault names had to be UTF-8 already).
|
||||
- `ls`/`list` piped into a reader that stops early end quietly with
|
||||
status 141, as Go's did by `SIGPIPE`.
|
||||
|
||||
Platforms: releases now cover Linux, Windows, macOS and FreeBSD (16
|
||||
binaries, down from 43); `PLATFORMS.md` lists what was dropped and why.
|
||||
|
||||
## Unchanged on purpose
|
||||
|
||||
Formats, limits (1024-byte names, 1 TiB files, 4 MiB objects, 4096-byte
|
||||
records, 16 MiB headers, 1 MiB stanzas), Argon2id parameters, the hybrid
|
||||
suite, filler, decoy behaviour, messages and exit statuses (apart from the
|
||||
above), the shape of `ls` and `list` output, and the agent protocol.
|
||||
@@ -0,0 +1,67 @@
|
||||
# Platforms
|
||||
|
||||
The Go version was built for every target of `go tool dist list` that needs
|
||||
no cgo: 43 binaries. The Rust release is narrower by decision (Linux,
|
||||
Windows, macOS, FreeBSD): 16 binaries. This is the target-by-target record:
|
||||
how each is built, and how far it has actually been verified. "Runtime" means
|
||||
a binary was executed; "build" means it only compiled and linked.
|
||||
|
||||
Tooling used for verification on the development machine (x86-64 Linux):
|
||||
the pinned Rust 1.99.0, zig 0.16.0 with cargo-zigbuild 0.23.4, qemu-user
|
||||
10.2.1 (Ubuntu package, unpacked locally, no system changes), glibc cross
|
||||
sysroots from Ubuntu's `libc6-*-cross` packages, and the Go oracle built for
|
||||
x86-64. `scripts/smoke.sh` is the runtime check: create a key and vault, store
|
||||
files across object boundaries, read them back, then have the Go oracle read
|
||||
what that binary wrote and the binary read what the oracle wrote.
|
||||
|
||||
## Released
|
||||
|
||||
The release covers Linux, Windows, macOS and FreeBSD (a decision made during
|
||||
the port: the Go release's more exotic targets are no longer built).
|
||||
|
||||
| Go name | Rust target | Linking | Built by | Verified |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| linux/amd64 | x86_64-unknown-linux-musl | static | zig | runtime: full test suite (glibc host build) and smoke test of the static binary |
|
||||
| linux/arm64 | aarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-aarch64, cross-checked with the Go oracle |
|
||||
| linux/386 | i686-unknown-linux-musl | static | zig | runtime: smoke test under qemu-i386, cross-checked |
|
||||
| linux/armv6 | arm-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
|
||||
| linux/armv7 | armv7-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
|
||||
| linux/riscv64 | riscv64gc-unknown-linux-musl | static | zig | runtime: smoke test and crypto unit tests under qemu-riscv64, cross-checked (note 1) |
|
||||
| linux/loong64 | loongarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-loongarch64, cross-checked |
|
||||
| linux/ppc64le | powerpc64le-unknown-linux-musl | static | zig | runtime: smoke test under qemu-ppc64le, cross-checked |
|
||||
| linux/s390x | s390x-unknown-linux-gnu | glibc, dynamic | zig | runtime: smoke test under qemu-s390x (big-endian) with an Ubuntu glibc sysroot, cross-checked |
|
||||
| windows/amd64 | x86_64-pc-windows-gnu | static CRT | zig | build; tests cross-compile cleanly; runtime in CI only (windows-2025 job, not yet run) |
|
||||
| windows/386 | i686-pc-windows-gnu | static CRT | zig | build only |
|
||||
| windows/arm64 | aarch64-pc-windows-gnullvm | static CRT | zig | build only |
|
||||
| darwin/amd64 | x86_64-apple-darwin | system | a Mac (Xcode) | CI only (macos-15-intel job, not yet run) |
|
||||
| darwin/arm64 | aarch64-apple-darwin | system | a Mac (Xcode) | CI only (macos-15 job, not yet run) |
|
||||
| freebsd/amd64 | x86_64-unknown-freebsd | system | cross 0.2.5 (Docker) | build only (no emulator) |
|
||||
| freebsd/386 | i686-unknown-freebsd | system | cross 0.2.5 (Docker) | build only |
|
||||
|
||||
Feature notes: the `unlock` agent works on Linux, macOS and FreeBSD;
|
||||
Windows asks for the passphrase every time. Full agent hardening
|
||||
(`PR_SET_DUMPABLE`) is Linux-only; elsewhere the agent locks its memory and
|
||||
disables core dumps. See `SECURITY.md`.
|
||||
|
||||
1. zig's bundled musl `getrandom()` never reaches the kernel on riscv64 (the
|
||||
process spins). `build.sh` and `packaging/package.sh` build that target
|
||||
with `--cfg getrandom_backend="linux_raw"`, which makes the `getrandom`
|
||||
crate issue the system call itself; with that flag the binary passes.
|
||||
|
||||
## No longer released
|
||||
|
||||
| Go name | Why |
|
||||
| --- | --- |
|
||||
| android/arm64 | was a copy of the static Linux arm64 binary; that binary still runs in Termux, but `add URL`/`mirror` need `/etc/resolv.conf` |
|
||||
| linux/ppc64 (big-endian) | zig links it for the ELFv2 loader that BE distributions lack; built with `cross` instead, even a hello-world crashes at start-up under qemu-ppc64, so it cannot be verified |
|
||||
| linux/mips, mipsle, mips64, mips64le | Rust tier 3 (no stable std) since 1.72 |
|
||||
| netbsd/amd64 | builds only with a newer sysroot than cross 0.2.5 provides (std needs `libexecinfo`); netbsd/386, arm, arm64 are tier 3 |
|
||||
| openbsd/* | Rust tier 3; OpenBSD packages Rust itself, so a native build there would work |
|
||||
| freebsd/arm, arm64, riscv64 | Rust tier 3 |
|
||||
| dragonfly/amd64, aix/ppc64 | Rust tier 3 |
|
||||
| illumos/amd64, solaris/amd64 | outside the chosen matrix (illumos builds with cross; Solaris has no cross 0.2.5 image) |
|
||||
| wasip1/wasm | outside the chosen matrix (it builds; no agent, no prompts) |
|
||||
| js/wasm | no filesystem in the browser target |
|
||||
| plan9/* | no Rust target |
|
||||
|
||||
Asking `build.sh` for any of these reports it and fails.
|
||||
+6
-3
@@ -7,7 +7,9 @@ sandbox path and file owner are shown as /home/sam and "sam sam", key lines
|
||||
are highlighted in amber for the reader, and "moment" frames show a prompt
|
||||
with a cursor, exactly as vault prints it before you type.
|
||||
|
||||
Needs: go, python3, and Chrome or Chromium (set CHROME=/path if not found).
|
||||
Needs: cargo (with the pinned toolchain), cargo-zigbuild and zig for the
|
||||
build screenshot, python3, and Chrome or Chromium (set CHROME=/path if not
|
||||
found).
|
||||
Output: docs/guide/img/*.png
|
||||
"""
|
||||
|
||||
@@ -75,7 +77,8 @@ class Sandbox:
|
||||
self.bin = os.path.join(self.root, "bin")
|
||||
os.makedirs(self.home)
|
||||
os.makedirs(self.bin)
|
||||
subprocess.run(["go", "build", "-o", os.path.join(self.bin, "vault"), "."], cwd=REPO, check=True)
|
||||
subprocess.run(["cargo", "build", "--release", "--locked", "--quiet"], cwd=REPO, check=True)
|
||||
shutil.copy2(os.path.join(REPO, "target", "release", "vault"), os.path.join(self.bin, "vault"))
|
||||
|
||||
def env(self, extra=None):
|
||||
e = {
|
||||
@@ -406,7 +409,7 @@ def main():
|
||||
|
||||
# building, into a temporary OUT so an existing dist/ is left alone
|
||||
rel = tempfile.mkdtemp(prefix="vault-release-")
|
||||
targets = "linux/amd64 linux/arm darwin/arm64 windows/amd64"
|
||||
targets = "linux/amd64 linux/arm64 linux/armv7 windows/amd64"
|
||||
t = Term(box, title="sam@laptop: ~/src/vault")
|
||||
out = subprocess.run(["./build.sh"], cwd=REPO, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||
text=True, env=dict(os.environ, TARGETS=targets, OUT=rel)).stdout
|
||||
|
||||
+35
-20
@@ -221,15 +221,17 @@ Linux PC or server & almost everything & vault-linux-amd64 \\
|
||||
Linux on ARM64 & Pi 4/5 (64-bit OS), Graviton, Ampere & vault-linux-arm64 \\
|
||||
Raspberry Pi, 32-bit & Pi 2, 3, 4 on a 32-bit OS & vault-linux-armv7 \\
|
||||
Raspberry Pi Zero / 1 & the tiny older ones & vault-linux-armv6 \\
|
||||
Android & in Termux & vault-android-arm64 \\
|
||||
FreeBSD, OpenBSD, NetBSD & amd64, arm64, 386, arm\ldots & vault-freebsd-amd64 etc. \\
|
||||
Something exotic & RISC-V, POWER, s390x, MIPS, LoongArch & see \cmd{dist/} \\
|
||||
Android & in Termux: the Linux ARM64 file & vault-linux-arm64 \\
|
||||
FreeBSD & amd64 or 386 & vault-freebsd-amd64 etc. \\
|
||||
Something exotic & RISC-V, POWER (little-endian), s390x, LoongArch & see \cmd{dist/} \\
|
||||
\end{vtable}
|
||||
|
||||
\begin{squirrel}
|
||||
The full release covers 43 platform builds, including AIX, Solaris, illumos,
|
||||
Plan~9 and even WebAssembly. Will you ever need \file{vault-plan9-arm}?
|
||||
Almost certainly not. Is it pleasing that it exists? Extremely.
|
||||
The release covers sixteen builds for Linux, Windows, macOS and FreeBSD,
|
||||
from Windows on ARM to IBM mainframes. The older Go builds reached further,
|
||||
to Plan~9 and WebAssembly; \file{docs/PLATFORMS.md} says why those went. Will you ever need
|
||||
\file{vault-linux-s390x}? Almost certainly not. Is it pleasing that it
|
||||
exists? Extremely.
|
||||
\end{squirrel}
|
||||
|
||||
\begin{headsup}
|
||||
@@ -287,8 +289,8 @@ so Gatekeeper quarantines it. Once you've checked the SHA-256, clear the flag:
|
||||
|
||||
\subsection{Android (Termux)}
|
||||
\begin{shell}
|
||||
$ curl -LO https://.../vault-android-arm64
|
||||
$ chmod +x vault-android-arm64 && mv vault-android-arm64 $PREFIX/bin/vault
|
||||
$ curl -LO https://.../vault-linux-arm64
|
||||
$ chmod +x vault-linux-arm64 && mv vault-linux-arm64 $PREFIX/bin/vault
|
||||
\end{shell}
|
||||
|
||||
\section{Did it work?}
|
||||
@@ -299,24 +301,28 @@ Run \cmd{vault help}. You should see the full command list:
|
||||
|
||||
\section{Building it yourself}
|
||||
|
||||
Prefer to build from source? You need Go 1.26.6 or newer. Then:
|
||||
Prefer to build from source? You need Rust, through \cmd{rustup}, which
|
||||
fetches the exact compiler version the project pins. Then:
|
||||
|
||||
\begin{shell}
|
||||
$ go build . # just your platform, produces ./vault
|
||||
$ ./build.sh # every platform, into ./dist, with SHA256SUMS
|
||||
$ cargo build --release --locked # just your platform: target/release/vault
|
||||
$ ./build.sh # every platform it can, into ./dist, with SHA256SUMS
|
||||
\end{shell}
|
||||
|
||||
\cmd{build.sh} cross-compiles every target Go supports without C libraries.
|
||||
\cmd{build.sh} cross-compiles Linux and Windows with \cmd{cargo-zigbuild}
|
||||
and \cmd{zig}, macOS on a Mac, and FreeBSD with \cmd{cross} and Docker; it says which targets your machine can't build.
|
||||
32-bit ARM Linux gets two builds, armv6 and armv7. Want just a few? Set
|
||||
\env{TARGETS}. Want them somewhere else? Set \env{OUT}.
|
||||
|
||||
\shot{build}{A partial build, followed by checking every binary against SHA256SUMS}
|
||||
|
||||
\begin{whyso}
|
||||
The binaries are \emph{static} and \emph{reproducible}: no C code, no system
|
||||
libraries, file paths stripped, build IDs zeroed. Two people building the same
|
||||
commit get byte-identical files, so you can check a release against your own
|
||||
build.
|
||||
The Linux binaries are \emph{static}: they need no system libraries at all.
|
||||
Every build uses the pinned compiler and the locked versions of every
|
||||
dependency, with build paths stripped, so the result doesn't depend on whose
|
||||
machine or which folder built it. Built with the same compiler for the same
|
||||
target, the same commit should give the same file, so you can check a
|
||||
release against your own build.
|
||||
\end{whyso}
|
||||
|
||||
\begin{pocket}
|
||||
@@ -324,7 +330,7 @@ build.
|
||||
\item One file per platform. Rename it \cmd{vault} and put it on your \env{PATH}.
|
||||
\item Always check the SHA-256 against \file{SHA256SUMS}.
|
||||
\item macOS: clear the quarantine flag after checking.
|
||||
\item \cmd{./build.sh} builds all 43 platform binaries from source.
|
||||
\item \cmd{./build.sh} builds every platform your machine can, from source.
|
||||
\end{pocketlist}
|
||||
\end{pocket}
|
||||
|
||||
@@ -1071,8 +1077,8 @@ who's slow at typing never blocks you.
|
||||
\begin{whyso}
|
||||
\textbf{What if vault crashes while holding the lock?} On Linux, macOS, the
|
||||
BSDs and Windows, the operating system drops the lock the moment the process
|
||||
dies. Nothing to clean up. On Plan~9 and WebAssembly, which have no file
|
||||
locks, vault uses a \file{.lock} file instead. If one's ever left behind,
|
||||
dies. Nothing to clean up. Older Go builds of vault use the very same locks,
|
||||
so mixing versions is safe. If one's ever left behind,
|
||||
vault tells you after a minute exactly which file to delete.
|
||||
\end{whyso}
|
||||
|
||||
@@ -1326,7 +1332,16 @@ could not read passphrase (set VAULT\_PASSPHRASE\ldots) & No terminal to ask on,
|
||||
agent did not start: \ldots & \cmd{unlock} couldn't start its helper. The reason follows; commands still work, asking for the passphrase. \\
|
||||
nothing was unlocked & \cmd{vault lock} found nothing to lock. Not an error. \\
|
||||
waiting for another vault process to finish & Not an error. Another command is writing; this one continues when it's done. \\
|
||||
vault still locked after a minute\ldots & Plan~9 and WebAssembly only. If nothing else is running, delete the file it names. \\
|
||||
cannot lock the vault: \ldots & The lock file couldn't be opened or locked; the reason follows. Check the vault folder's permissions. \\
|
||||
\ldots\ changed while being added & The file was being written while vault read it. Nothing was stored; try again when it's finished. \\
|
||||
\ldots\ changed after it was stored; not deleting it & \cmd{migrate} stored the file, but it changed before it could be deleted. The original is kept; the vault has the version it read. \\
|
||||
\ldots\ already exists & \cmd{get} never overwrites, not even a symlink. Pick another name with \cmd{-o}. \\
|
||||
\ldots\ refusing to extract through it & \cmd{get -r} found a symlink (or a file) where a folder should be inside the output folder, and stopped rather than follow it. \\
|
||||
cannot lock agent memory \ldots, cannot make the agent undumpable \ldots & \cmd{unlock}'s helper couldn't protect the key in memory, so it refused it. Commands still work, asking for the passphrase. \\
|
||||
vault unlock is not supported on this platform\ldots & Windows: every command asks for the passphrase. \\
|
||||
download exceeds \ldots\ bytes & A download passed its size limit. Raise it with \cmd{--max-bytes} if you meant it. \\
|
||||
redirected to another host: \ldots & \cmd{mirror} only follows redirects within the site it is copying. \\
|
||||
mirror incomplete: stored \ldots, skipped \ldots & Some pages failed; the reasons were printed as they happened. Exit status 3. \\
|
||||
skipping unreadable record \ldots & A catalogue entry is damaged. Everything else still works; restore \file{records/} from backup to recover that file. \\
|
||||
object does not match its catalogue record & The encrypted contents were swapped or damaged. Nothing was written. Restore \file{objects/} from backup. \\
|
||||
object size does not match its catalogue record & Same as above: damage or tampering, caught before any output appeared. \\
|
||||
|
||||
+21
-2
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-ADD 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-ADD 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-add \- encrypt files, directories or URLs into the vault
|
||||
.SH SYNOPSIS
|
||||
@@ -7,6 +7,10 @@ vault-add \- encrypt files, directories or URLs into the vault
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RB [ \-\-max\-bytes
|
||||
.IR SIZE ]
|
||||
.RB [ \-\-timeout
|
||||
.IR DURATION ]
|
||||
.IR FILE | DIR | URL
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
@@ -17,7 +21,14 @@ and the key are skipped. A stored name that already exists is replaced.
|
||||
.PP
|
||||
An \fBhttp://\fR or \fBhttps://\fR \fIURL\fR is downloaded with a Chrome user
|
||||
agent and stored under the last element of its final (post-redirect) path, or
|
||||
the host name if there is none. Only a 200 response is accepted.
|
||||
the host name if there is none. Only a 200 response is accepted. Certificates
|
||||
are always verified; at most 10 redirects are followed, to http or https
|
||||
only. The download is staged in a private temporary file (it must be
|
||||
complete before it can be stored), which is removed afterwards whatever
|
||||
happens; it is plaintext on disk meanwhile.
|
||||
.PP
|
||||
A file that changes while it is being added (it grows, shrinks, or is
|
||||
replaced) is not stored, and the command fails.
|
||||
.PP
|
||||
\fIFOLDER\fR places the result inside a folder in the vault. File contents
|
||||
are cut into 4 MiB chunks, the last bucket-padded.
|
||||
@@ -28,6 +39,14 @@ Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.BI \-\-max\-bytes " SIZE"
|
||||
Largest download accepted, counted after decompression, such as 500M or 50G.
|
||||
Default 16G; at most 1T.
|
||||
.TP
|
||||
.BI \-\-timeout " DURATION"
|
||||
Deadline for the whole download, such as 30m. Default 2h. Connecting must
|
||||
take under 30 seconds and the response headers must arrive within a minute.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
|
||||
+9
-4
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-GET 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-GET 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-get \- decrypt a stored file
|
||||
.SH SYNOPSIS
|
||||
@@ -13,13 +13,18 @@ vault-get \- decrypt a stored file
|
||||
.I NAME
|
||||
.SH DESCRIPTION
|
||||
Decrypts \fINAME\fR. Nothing appears at the output path until every object has
|
||||
authenticated. The stored mode and modification time are restored. It refuses
|
||||
to overwrite an existing file. Takes a shared lock.
|
||||
authenticated, padding and end of file included; until then the output is a
|
||||
private temporary file in the same directory, removed on failure. The stored
|
||||
mode and modification time are restored. It never replaces anything already
|
||||
at the output path, including a symbolic link (even a dangling one) or a
|
||||
file created while it runs. Takes a shared lock.
|
||||
.PP
|
||||
With \fB\-r\fR, \fINAME\fR is a folder, and every file under it is
|
||||
decrypted into the output directory, keeping the folder structure. Each file
|
||||
is checked as above; a clash stops the run, leaving the files already
|
||||
written.
|
||||
written. Directories are created as needed, mode 0700; a symbolic link (or
|
||||
anything but a directory) where a directory belongs inside the output
|
||||
directory stops the run rather than being followed.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-o " OUTPUT"
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-MIGRATE 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-MIGRATE 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-migrate \- move files into the vault, deleting the originals
|
||||
.SH SYNOPSIS
|
||||
@@ -13,7 +13,10 @@ vault-migrate \- move files into the vault, deleting the originals
|
||||
.SH DESCRIPTION
|
||||
Works like
|
||||
.BR vault\-add (1)
|
||||
but deletes each original once it is stored, then any directories left empty.
|
||||
but deletes each original once it is stored (and its record is on disk),
|
||||
then any directories left empty. An original is deleted only if it is still
|
||||
the same file, unchanged, that was stored; otherwise it is kept and the
|
||||
command fails.
|
||||
Directories still holding skipped files stay. A URL cannot be migrated.
|
||||
.PP
|
||||
Before storing anything it lists the vault names that would be overwritten and
|
||||
|
||||
+60
-17
@@ -1,4 +1,4 @@
|
||||
.TH VAULT-MIRROR 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT-MIRROR 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-mirror \- mirror a website into an encrypted vault
|
||||
.SH SYNOPSIS
|
||||
@@ -7,6 +7,12 @@ vault-mirror \- mirror a website into an encrypted vault
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.RB [ \-\-max\-pages
|
||||
.IR N ]
|
||||
.RB [ \-\-max\-bytes
|
||||
.IR SIZE ]
|
||||
.RB [ \-\-timeout
|
||||
.IR DURATION ]
|
||||
.I URL
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
@@ -20,21 +26,30 @@ and stores every page and asset it finds in the vault, beneath
|
||||
if no folder is given).
|
||||
.PP
|
||||
The crawl is single threaded: one request at a time, in breadth-first order.
|
||||
It follows only links on the same host as
|
||||
It follows only links on the same host (and port) as
|
||||
.IR URL ,
|
||||
and requests are sent with a Chrome user agent string.
|
||||
and only redirects to that host; a redirect elsewhere skips the page.
|
||||
Requests are sent with a Chrome user agent string, and certificates are
|
||||
always verified.
|
||||
.PP
|
||||
Links in HTML
|
||||
Links in HTML start tags
|
||||
.RB ( href ,
|
||||
.BR src ,
|
||||
.BR action
|
||||
and
|
||||
.B poster
|
||||
attributes) and in CSS
|
||||
.RB ( url(...) )
|
||||
that point into the mirror are rewritten to relative paths, so the copy
|
||||
browses correctly once fetched out of the vault. Links to other hosts, and
|
||||
fragments, are left as they were.
|
||||
attributes) and CSS
|
||||
.RB ( url(...)
|
||||
in style sheets,
|
||||
.B <style>
|
||||
elements and
|
||||
.B style
|
||||
attributes) that point into the mirror are rewritten to relative paths, so
|
||||
the copy browses correctly once fetched out of the vault. Relative links are
|
||||
resolved against the page's final address and its
|
||||
.B <base href>
|
||||
if it has one. Links to other hosts, script and comment text, and every
|
||||
other byte of the page are left as they were. Fragments are kept.
|
||||
.PP
|
||||
Pages are named after their URL path. A path that is empty or ends in
|
||||
.B /
|
||||
@@ -42,19 +57,39 @@ becomes
|
||||
.BR index.html ;
|
||||
a query string is appended as
|
||||
.BI _ query
|
||||
(URL-escaped). Anything already stored under the same name is replaced.
|
||||
(URL-escaped). A name already stored by an earlier run is replaced, but if
|
||||
two different URLs in one crawl would get the same name, the second is
|
||||
skipped rather than overwriting the first.
|
||||
.PP
|
||||
A page that fails to download, or answers with anything but
|
||||
A page that fails to download, answers with anything but
|
||||
.BR "200 OK" ,
|
||||
is reported on standard error and skipped; the crawl carries on.
|
||||
The vault is locked exclusively for the whole run.
|
||||
or exceeds a limit is reported on standard error as
|
||||
.BI "skipping " URL ": " reason
|
||||
and skipped; the crawl carries on. HTML and CSS are rewritten in memory;
|
||||
other files are staged in a private temporary file (plaintext on disk
|
||||
meanwhile) that is always removed. The vault is locked exclusively for the
|
||||
whole run.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-max\-pages " N"
|
||||
Pages and assets to fetch at most. Default 10000.
|
||||
.TP
|
||||
.BI \-\-max\-bytes " SIZE"
|
||||
Total bytes to download at most, counted after decompression. Default 4G.
|
||||
.TP
|
||||
.BI \-\-timeout " DURATION"
|
||||
Deadline for each request. Default 5m.
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.PP
|
||||
A single HTML or CSS file larger than 16 MiB is skipped.
|
||||
.SH EXIT STATUS
|
||||
0 when everything found was stored, 3 when some of it was skipped, 1 on
|
||||
an error that stops the crawl.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
@@ -75,11 +110,19 @@ vault mirror https://example.com/ archive/2026
|
||||
vault get archive/2026/example.com/index.html
|
||||
.RE
|
||||
.SH LIMITATIONS
|
||||
Links are found by pattern matching, not by parsing HTML: URLs assembled by
|
||||
scripts, and
|
||||
URLs assembled by scripts, and
|
||||
.B srcset
|
||||
lists, are neither followed nor rewritten. There is no depth limit, rate
|
||||
limit or robots.txt handling; only mirror sites you may copy.
|
||||
lists, are neither followed nor rewritten. In attribute values only numeric
|
||||
character references and
|
||||
.BR & ,
|
||||
.BR < ,
|
||||
.BR > ,
|
||||
.BR " ,
|
||||
.B '
|
||||
and
|
||||
.B
|
||||
are decoded. There is no rate limit or robots.txt handling; only mirror
|
||||
sites you may copy.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1)
|
||||
.SH AUTHOR
|
||||
|
||||
@@ -32,6 +32,14 @@ memory is locked against swapping and excluded from core dumps; on Linux the
|
||||
helper is also not debuggable. It answers only on a Unix socket inside
|
||||
\fI$XDG_RUNTIME_DIR/vault-UID\fR (or the temporary directory), a directory
|
||||
that must belong to this user with mode 0700. Rebooting ends it too.
|
||||
If the helper cannot lock its memory, disable core dumps or (on Linux) make
|
||||
itself undebuggable, it refuses the key and
|
||||
.B unlock
|
||||
fails, saying why. The helper is the same program, started with the
|
||||
passphrase variables removed from its environment. Helpers started by
|
||||
earlier (Go) versions of vault are compatible and can be used and locked.
|
||||
.PP
|
||||
Not available on Windows, where every command asks for the passphrase.
|
||||
.PP
|
||||
While the key is unlocked, any program running as this user can use it, and
|
||||
it is whichever key the passphrase opened, real or decoy. Lock before
|
||||
|
||||
+23
-2
@@ -1,4 +1,4 @@
|
||||
.TH VAULT 1 "September 2026" "vault" "User Commands"
|
||||
.TH VAULT 1 "October 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault \- post-quantum encrypted file vault
|
||||
.SH SYNOPSIS
|
||||
@@ -77,6 +77,11 @@ Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.TP
|
||||
.BI \-\-workers " N"
|
||||
Threads for encryption, decryption and file I/O, 1 to 256. Default: the
|
||||
number of CPUs, at most 8. Up to eight 4 MiB objects are in flight at once
|
||||
whatever the count.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_KEY
|
||||
@@ -92,6 +97,19 @@ Passphrase, instead of prompting on the terminal or using a key remembered by
|
||||
.B VAULT_DECOY_PASSPHRASE
|
||||
Decoy passphrase used by
|
||||
.BR "vault genkey" .
|
||||
.TP
|
||||
.BR HTTPS_PROXY ", " HTTP_PROXY ", " NO_PROXY
|
||||
Proxies for
|
||||
.B add
|
||||
.I URL
|
||||
and
|
||||
.BR mirror ,
|
||||
as other programs use them; local addresses are never proxied.
|
||||
.TP
|
||||
.BR XDG_RUNTIME_DIR ", " TMPDIR
|
||||
Where
|
||||
.BR vault\-unlock (1)
|
||||
keeps its socket, and where downloads are staged.
|
||||
.SH LOCKING
|
||||
Commands that change the vault hold an exclusive lock on
|
||||
.IR DIR/.lock ;
|
||||
@@ -106,7 +124,10 @@ and
|
||||
fills each vault with unreadable filler, so hidden files look like filler to
|
||||
anyone holding only the decoy passphrase.
|
||||
.SH EXIT STATUS
|
||||
0 on success; 1 on error (diagnostics on standard error); 2 when run with no command.
|
||||
0 on success; 1 on error (diagnostics on standard error); 2 when run with no
|
||||
command; 3 when
|
||||
.BR vault\-mirror (1)
|
||||
stored some resources but had to skip others.
|
||||
.SH EXAMPLES
|
||||
.nf
|
||||
vault genkey
|
||||
|
||||
Generated
+1287
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
[package]
|
||||
name = "vault-fuzz"
|
||||
version = "0.0.0"
|
||||
publish = false
|
||||
edition = "2024"
|
||||
|
||||
[package.metadata]
|
||||
cargo-fuzz = true
|
||||
|
||||
[dependencies]
|
||||
libfuzzer-sys = "0.4"
|
||||
tar = { version = "0.4.46", default-features = false }
|
||||
vault = { path = ".." }
|
||||
|
||||
# Not part of the main package's workspace.
|
||||
[workspace]
|
||||
|
||||
[profile.release]
|
||||
debug = 1
|
||||
|
||||
[[bin]]
|
||||
name = "header"
|
||||
path = "fuzz_targets/header.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "stream"
|
||||
path = "fuzz_targets/stream.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "keyfile"
|
||||
path = "fuzz_targets/keyfile.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "record"
|
||||
path = "fuzz_targets/record.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "names"
|
||||
path = "fuzz_targets/names.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "html"
|
||||
path = "fuzz_targets/html.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
|
||||
[[bin]]
|
||||
name = "tar"
|
||||
path = "fuzz_targets/tar.rs"
|
||||
test = false
|
||||
doc = false
|
||||
bench = false
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The Fury header parser never panics, and accepts only canonical headers:
|
||||
//! whatever it accepts re-serialises to exactly the bytes it consumed.
|
||||
#![no_main]
|
||||
use std::io::Read;
|
||||
use vault::crypto::header;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Ok((h, mut rest)) = header::parse(data) {
|
||||
let mut tail = Vec::new();
|
||||
rest.read_to_end(&mut tail).unwrap();
|
||||
let consumed = &data[..data.len() - tail.len()];
|
||||
assert_eq!(h.marshal(), consumed, "non-canonical header accepted");
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The mirror's HTML and CSS rewriters never panic, and rewriting nothing
|
||||
//! returns the input byte for byte.
|
||||
#![no_main]
|
||||
use vault::net::html;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let _ = html::tokenize(data);
|
||||
assert_eq!(html::rewrite_html(data, &mut |_| None, &mut |_| None), data);
|
||||
assert_eq!(html::rewrite_css(data, false, &mut |_| None), data);
|
||||
let _ = html::rewrite_html(data, &mut |_| None, &mut |u| Some(format!("x/{}", u.len())));
|
||||
if let Ok(s) = std::str::from_utf8(data) {
|
||||
let _ = vault::net::Url::parse(s).map(|u| u.join("../a?b#c"));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,11 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! PEM decoding never panics, and decoding what it encoded is the identity.
|
||||
#![no_main]
|
||||
use vault::keyfile::{decode_pem, encode_pem, parse_body};
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Some(body) = decode_pem(data) {
|
||||
let _ = parse_body(&body);
|
||||
assert_eq!(decode_pem(encode_pem(&body).as_bytes()), Some(body));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Path validation: a valid vault name is clean, relative and confined; the
|
||||
//! tar name check only ever yields valid-shaped relative names.
|
||||
#![no_main]
|
||||
use vault::store::VaultName;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let Ok(s) = std::str::from_utf8(data) else { return };
|
||||
if let Some(n) = VaultName::new(s) {
|
||||
let n = n.as_str();
|
||||
assert_eq!(vault::util::clean(n), n);
|
||||
assert!(!n.starts_with('/') && !n.contains('\\') && !n.split('/').any(|c| c == ".." || c == "." || c.is_empty()));
|
||||
assert!(n.len() <= 1024);
|
||||
}
|
||||
if let Some(n) = vault::absorb::safe_name(s) {
|
||||
assert!(!n.is_empty() && !n.starts_with('/') && !n.split('/').any(|c| c == ".."));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,15 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Record JSON parsing never panics; an accepted record re-encodes (Go's
|
||||
//! JSON) to something that parses back to the same values.
|
||||
#![no_main]
|
||||
use vault::store::{Id, Record};
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
if let Ok(r) = Record::from_json(data, Id([0; 16])) {
|
||||
let Some(j) = r.to_json() else { return };
|
||||
let r2 = Record::from_json(&j, Id([0; 16])).expect("own encoding parses");
|
||||
assert_eq!(r.name, r2.name);
|
||||
assert_eq!((r.size, r.mode, r.modified, r.added), (r2.size, r2.mode, r2.modified, r2.added));
|
||||
assert!(r.seed.same_as(&r2.seed));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! The streaming, in-memory and random-access STREAM readers agree on every
|
||||
//! input: the same plaintext or all reject. Inputs are mostly forgeries;
|
||||
//! the first byte selects a mode that starts from a valid stream instead.
|
||||
#![no_main]
|
||||
use std::io::{Read, Write};
|
||||
use vault::crypto::stream::{StreamReader, StreamReaderAt, StreamWriter, open_all};
|
||||
|
||||
const KEY: [u8; 32] = [7; 32];
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let Some((&mode, rest)) = data.split_first() else { return };
|
||||
let input = if mode & 1 == 1 {
|
||||
// A valid stream of the given plaintext, then a mutation.
|
||||
let mut w = StreamWriter::new(&KEY, Vec::new());
|
||||
let n = rest.len().min(200_000);
|
||||
w.write_all(&rest[..n]).unwrap();
|
||||
let mut c = w.finish().unwrap();
|
||||
if mode & 2 == 2 && !c.is_empty() {
|
||||
let i = (mode as usize * 7919) % c.len();
|
||||
c[i] ^= mode | 1;
|
||||
}
|
||||
if mode & 4 == 4 {
|
||||
c.truncate(c.len().saturating_sub(mode as usize % 40));
|
||||
}
|
||||
c
|
||||
} else {
|
||||
rest.to_vec()
|
||||
};
|
||||
let mut a = Vec::new();
|
||||
let streaming = StreamReader::new(&KEY, &input[..]).read_to_end(&mut a).map(|_| a);
|
||||
let mut b = input.clone();
|
||||
let whole = open_all(&KEY, &mut b).map(|n| b[..n].to_vec());
|
||||
assert_eq!(streaming.is_ok(), whole.is_ok());
|
||||
if let (Ok(x), Ok(y)) = (&streaming, &whole) {
|
||||
assert_eq!(x, y);
|
||||
}
|
||||
if let Ok(r) = StreamReaderAt::new(&KEY, &input[..], input.len() as u64) {
|
||||
let mut out = vec![0u8; r.size() as usize];
|
||||
let got = r.read_at(&mut out, 0).map(|n| out[..n].to_vec());
|
||||
assert_eq!(got.is_ok(), whole.is_ok());
|
||||
if let (Ok(x), Ok(y)) = (got, whole) {
|
||||
assert_eq!(x, y);
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
//! Walking arbitrary tar input as absorb does never panics, and every name
|
||||
//! it would store is confined.
|
||||
#![no_main]
|
||||
use std::io::Read;
|
||||
|
||||
libfuzzer_sys::fuzz_target!(|data: &[u8]| {
|
||||
let mut a = tar::Archive::new(data);
|
||||
let Ok(entries) = a.entries() else { return };
|
||||
for e in entries.take(64) {
|
||||
let Ok(mut e) = e else { return };
|
||||
let name = e.path_bytes().into_owned();
|
||||
if let Some(n) = std::str::from_utf8(&name).ok().and_then(vault::absorb::safe_name) {
|
||||
assert!(!n.split('/').any(|c| c == ".."));
|
||||
}
|
||||
let mut sink = Vec::new();
|
||||
let _ = e.by_ref().take(1 << 20).read_to_end(&mut sink);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,46 @@
|
||||
# Go oracle (test-only)
|
||||
|
||||
`oracle/go` is a frozen copy of the Go implementation of vault, taken from
|
||||
commit `4c1507c` ("Add unlock --idle, and packaging/repo.sh for
|
||||
repo.dixon.cx"), the last Go release on `master`. It is kept as the
|
||||
reference that the Rust implementation is checked against. It is **not**
|
||||
part of the shipped product: release builds, packages and the `vault`
|
||||
binary contain no Go code and need no Go toolchain.
|
||||
|
||||
Provenance:
|
||||
|
||||
- All 32 Go sources and `go.mod`/`go.sum` under `oracle/go` are
|
||||
byte-identical to the same paths at the repository root at `4c1507c`.
|
||||
- The only additions are test files whose names contain `oracle`
|
||||
(`*oracle_vectors_test.go`, `oracle_decrypt_test.go`,
|
||||
`*oracle_bench_test.go`). They call the frozen code and record what it
|
||||
computes; they skip themselves unless `VAULT_ORACLE_OUT` or
|
||||
`VAULT_ORACLE_DECRYPT` is set (benchmarks run only with `-bench`), and add
|
||||
nothing to the oracle binary.
|
||||
- Dependencies are pinned by `go.mod`/`go.sum`: CIRCL v1.6.4,
|
||||
golang.org/x/crypto v0.53.0, x/sys v0.46.0, x/term v0.44.0. The module
|
||||
needs Go 1.26.6 or newer.
|
||||
|
||||
Check the copy is still faithful:
|
||||
|
||||
```sh
|
||||
for f in $(git ls-tree -r --name-only 4c1507c | grep -E '\.go$|^go\.(mod|sum)$'); do
|
||||
cmp <(git show 4c1507c:$f) oracle/go/$f || echo "DIFFERS: $f"
|
||||
done
|
||||
```
|
||||
|
||||
Use:
|
||||
|
||||
```sh
|
||||
oracle/gen-vectors.sh # rewrite tests/vectors from the oracle
|
||||
(cd oracle/go && go build -o ../../target/vault-go .) # build the oracle CLI
|
||||
(cd oracle/go && go test ./...) # its own test-suite
|
||||
```
|
||||
|
||||
`cargo test` builds the oracle binary itself for the interoperability tests
|
||||
when `go` is on `PATH` (or uses `$VAULT_GO_ORACLE`), and reports them as
|
||||
skipped otherwise.
|
||||
|
||||
All passphrases, seeds and plaintexts in the vectors are synthetic and
|
||||
public (`oracle-real passphrase`, `oracle-decoy passphrase`, fixed byte
|
||||
patterns). Never commit real keys or real vault contents.
|
||||
Executable
+14
@@ -0,0 +1,14 @@
|
||||
#!/usr/bin/env bash
|
||||
# Regenerate the Go-oracle vectors and fixtures in tests/vectors.
|
||||
#
|
||||
# Everything is computed by the frozen Go implementation in oracle/go (see
|
||||
# oracle/README.md); nothing comes from the Rust code under test. Key files
|
||||
# and ciphertexts use fresh randomness, so regenerating changes them; the
|
||||
# deterministic vectors (*.json fields derived from fixed seeds) do not.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
out=$PWD/tests/vectors
|
||||
mkdir -p "$out"
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream)
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./)
|
||||
echo "vectors written to $out"
|
||||
@@ -0,0 +1,57 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func BenchmarkOracleDerive(b *testing.B) {
|
||||
seed := pattern(32, 1)
|
||||
for b.Loop() {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, seed)
|
||||
id.Close()
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOracleWrap(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
r := id.Recipient()
|
||||
fk := pattern(32, 2)
|
||||
for b.Loop() {
|
||||
r.Wrap(fk)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkOracleUnwrap(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
st, _ := id.Recipient().Wrap(pattern(32, 2))
|
||||
for b.Loop() {
|
||||
if _, err := id.Unwrap(st); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Whole-file encryption and decryption of 4 MiB: one vault object.
|
||||
func BenchmarkOracleObject4MiB(b *testing.B) {
|
||||
id, _ := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
pt := pattern(4<<20, 3)
|
||||
var ct bytes.Buffer
|
||||
b.SetBytes(int64(len(pt)))
|
||||
for b.Loop() {
|
||||
ct.Reset()
|
||||
w, _ := Encrypt(&ct, id.Recipient())
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
r, err := Decrypt(bytes.NewReader(ct.Bytes()), id)
|
||||
if err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
io.Copy(io.Discard, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only: decrypts files another implementation wrote, for the Rust
|
||||
// port's interoperability tests. Skips unless VAULT_ORACLE_DECRYPT is set.
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// TestOracleDecrypt opens every DIR/pattern-SIZE.fury with the identity of
|
||||
// master seed pattern(32, 1) and checks it holds pattern(SIZE, 0x5a), and
|
||||
// that every DIR/bad-*.fury is rejected.
|
||||
func TestOracleDecrypt(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_DECRYPT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_DECRYPT")
|
||||
}
|
||||
id, err := NewPQIdentityFromSeed(0x0001, pattern(32, 1))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
files, _ := filepath.Glob(filepath.Join(dir, "*.fury"))
|
||||
if len(files) == 0 {
|
||||
t.Fatal("no files")
|
||||
}
|
||||
for _, f := range files {
|
||||
data, _ := os.ReadFile(f)
|
||||
r, err := Decrypt(bytes.NewReader(data), id)
|
||||
var got []byte
|
||||
if err == nil {
|
||||
got, err = io.ReadAll(r)
|
||||
}
|
||||
base := filepath.Base(f)
|
||||
if strings.HasPrefix(base, "bad-") {
|
||||
if err == nil {
|
||||
t.Errorf("%s: accepted", base)
|
||||
}
|
||||
continue
|
||||
}
|
||||
size, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(base, "pattern-"), ".fury"))
|
||||
if err != nil || !bytes.Equal(got, pattern(size, 0x5a)) {
|
||||
t.Errorf("%s: %v", base, err)
|
||||
}
|
||||
t.Logf("%s ok", base)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,298 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port. It adds nothing to the
|
||||
// production code: it only calls the frozen implementation and records what
|
||||
// it computes. Run through oracle/gen-vectors.sh.
|
||||
|
||||
package pqcrypt
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/andydixon/vault/internal/format"
|
||||
"github.com/cloudflare/circl/kem"
|
||||
"github.com/cloudflare/circl/kem/frodo/frodo640shake"
|
||||
"github.com/cloudflare/circl/kem/mlkem/mlkem768"
|
||||
"golang.org/x/crypto/hkdf"
|
||||
)
|
||||
|
||||
type kemVector struct {
|
||||
Seed string `json:"seed"`
|
||||
PubSHA256 string `json:"pub_sha256"`
|
||||
PrivSHA256 string `json:"priv_sha256"`
|
||||
EncapSeed string `json:"encap_seed"`
|
||||
CtSHA256 string `json:"ct_sha256"`
|
||||
SharedSecret string `json:"ss"`
|
||||
RejectFlipIdx int `json:"reject_flip_index"`
|
||||
RejectSS string `json:"reject_ss"`
|
||||
}
|
||||
|
||||
type hybridVector struct {
|
||||
MasterSeed string `json:"master_seed"`
|
||||
FrodoSeed string `json:"frodo_seed"`
|
||||
MLKEMSeed string `json:"mlkem_seed"`
|
||||
PubSHA256 string `json:"pub_sha256"`
|
||||
PrivSHA256 string `json:"priv_sha256"`
|
||||
PubPrefix string `json:"pub_prefix"`
|
||||
}
|
||||
|
||||
type wrapVector struct {
|
||||
MasterSeed string `json:"master_seed"`
|
||||
FrodoEncap string `json:"frodo_encap_seed"`
|
||||
MLKEMEncap string `json:"mlkem_encap_seed"`
|
||||
FileKey string `json:"file_key"`
|
||||
KEK string `json:"kek"`
|
||||
Header string `json:"header_hex"` // full serialised header, MAC line included
|
||||
MAC string `json:"mac"`
|
||||
Nonce string `json:"nonce"`
|
||||
PayloadKey string `json:"payload_key"`
|
||||
StanzaArg0 string `json:"stanza_arg0_sha256"`
|
||||
StanzaBody string `json:"stanza_body"`
|
||||
}
|
||||
|
||||
type headerCase struct {
|
||||
Input string `json:"input_hex"`
|
||||
Accept bool `json:"accept"`
|
||||
Rest string `json:"rest_hex,omitempty"` // payload bytes after the header
|
||||
}
|
||||
|
||||
func sum(b []byte) string { s := sha256.Sum256(b); return hex.EncodeToString(s[:]) }
|
||||
|
||||
func pattern(n int, salt byte) []byte {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = byte(i*7) ^ salt
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func kemVectors(t *testing.T, s kem.Scheme, n int) []kemVector {
|
||||
var out []kemVector
|
||||
for i := range n {
|
||||
seed := pattern(s.SeedSize(), byte(i*31))
|
||||
pk, sk := s.DeriveKeyPair(seed)
|
||||
pkb, _ := pk.MarshalBinary()
|
||||
skb, _ := sk.MarshalBinary()
|
||||
es := pattern(s.EncapsulationSeedSize(), byte(i*17+5))
|
||||
ct, ss, err := s.EncapsulateDeterministically(pk, es)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ss2, err := s.Decapsulate(sk, ct)
|
||||
if err != nil || !bytes.Equal(ss, ss2) {
|
||||
t.Fatal("decapsulation mismatch")
|
||||
}
|
||||
flip := (i * 997) % len(ct)
|
||||
bad := append([]byte(nil), ct...)
|
||||
bad[flip] ^= 0x01
|
||||
rss, err := s.Decapsulate(sk, bad)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out = append(out, kemVector{
|
||||
Seed: hex.EncodeToString(seed), PubSHA256: sum(pkb), PrivSHA256: sum(skb),
|
||||
EncapSeed: hex.EncodeToString(es), CtSHA256: sum(ct), SharedSecret: hex.EncodeToString(ss),
|
||||
RejectFlipIdx: flip, RejectSS: hex.EncodeToString(rss),
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
suite := frodoMLKEM768{}
|
||||
frodo, ml := frodo640shake.Scheme(), mlkem768.Scheme()
|
||||
|
||||
var hybrids []hybridVector
|
||||
for i := range 6 {
|
||||
master := pattern(32, byte(i*53+1))
|
||||
if i == 5 {
|
||||
master = bytes.Repeat([]byte{0xff}, 32)
|
||||
}
|
||||
r := hkdf.New(sha256.New, master, nil, []byte("fury/v1 seed-expand "+suite.tag()))
|
||||
fs := make([]byte, frodo.SeedSize())
|
||||
ms := make([]byte, ml.SeedSize())
|
||||
io.ReadFull(r, fs)
|
||||
io.ReadFull(r, ms)
|
||||
pub, priv, err := suite.deriveKeypair(master)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hybrids = append(hybrids, hybridVector{
|
||||
MasterSeed: hex.EncodeToString(master), FrodoSeed: hex.EncodeToString(fs), MLKEMSeed: hex.EncodeToString(ms),
|
||||
PubSHA256: sum(pub), PrivSHA256: sum(priv), PubPrefix: hex.EncodeToString(pub[:64]),
|
||||
})
|
||||
}
|
||||
|
||||
// A wrap with controlled encapsulation randomness, built from the real
|
||||
// deriveKEK, aeadEncrypt, header marshalling, headerMAC and streamKey.
|
||||
var wraps []wrapVector
|
||||
for i := range 3 {
|
||||
master := pattern(32, byte(i*53+1))
|
||||
id, err := NewPQIdentityFromSeed(suite.id(), master)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pub := id.Recipient().Bytes()
|
||||
fpk, _ := frodo.UnmarshalBinaryPublicKey(pub[:frodo.PublicKeySize()])
|
||||
mpk, _ := ml.UnmarshalBinaryPublicKey(pub[frodo.PublicKeySize():])
|
||||
fe, me := pattern(16, byte(i+100)), pattern(32, byte(i+150))
|
||||
ct1, ss1, _ := frodo.EncapsulateDeterministically(fpk, fe)
|
||||
ct2, ss2, _ := ml.EncapsulateDeterministically(mpk, me)
|
||||
kek := suite.deriveKEK(ss1, ss2, ct1, ct2)
|
||||
fileKey := pattern(32, byte(i+200))
|
||||
body, _ := aeadEncrypt(kek, fileKey)
|
||||
st := &format.Stanza{Type: suite.tag(), Args: []string{format.EncodeToString(ct1), format.EncodeToString(ct2)}, Body: body}
|
||||
hdr := &format.Header{Recipients: []*format.Stanza{st}}
|
||||
mac, err := headerMAC(fileKey, hdr)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hdr.MAC = mac
|
||||
var hb bytes.Buffer
|
||||
hdr.Marshal(&hb)
|
||||
// The real identity must open what was built.
|
||||
got, err := id.Unwrap([]*Stanza{(*Stanza)(st)})
|
||||
if err != nil || !bytes.Equal(got, fileKey) {
|
||||
t.Fatal("identity did not unwrap the vector")
|
||||
}
|
||||
nonce := pattern(16, byte(i+250))
|
||||
wraps = append(wraps, wrapVector{
|
||||
MasterSeed: hex.EncodeToString(master), FrodoEncap: hex.EncodeToString(fe), MLKEMEncap: hex.EncodeToString(me),
|
||||
FileKey: hex.EncodeToString(fileKey), KEK: hex.EncodeToString(kek), Header: hex.EncodeToString(hb.Bytes()),
|
||||
MAC: hex.EncodeToString(mac), Nonce: hex.EncodeToString(nonce), PayloadKey: hex.EncodeToString(streamKey(fileKey, nonce)),
|
||||
StanzaArg0: sum([]byte(st.Args[0])), StanzaBody: hex.EncodeToString(body),
|
||||
})
|
||||
}
|
||||
|
||||
// Whole files from the real Encrypt, with fresh randomness, for the
|
||||
// first hybrid identity. Plaintexts are pattern(size, 0x5a).
|
||||
id, _ := NewPQIdentityFromSeed(suite.id(), pattern(32, 1))
|
||||
fdir := filepath.Join(dir, "fury")
|
||||
os.MkdirAll(fdir, 0o755)
|
||||
for _, size := range []int{0, 1, 65535, 65536, 65537, 131072, 200000} {
|
||||
var buf bytes.Buffer
|
||||
w, err := Encrypt(&buf, id.Recipient())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
w.Write(pattern(size, 0x5a))
|
||||
if err := w.Close(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
os.WriteFile(filepath.Join(fdir, "pattern-"+itoa(size)+".fury"), buf.Bytes(), 0o644)
|
||||
}
|
||||
|
||||
// Header parser verdicts on valid and malformed headers.
|
||||
var cases []headerCase
|
||||
valid, _ := hex.DecodeString(wraps[0].Header)
|
||||
add := func(b []byte) {
|
||||
h, payload, err := format.Parse(bytes.NewReader(b))
|
||||
c := headerCase{Input: hex.EncodeToString(b), Accept: err == nil}
|
||||
if err == nil {
|
||||
rest, _ := io.ReadAll(payload)
|
||||
c.Rest = hex.EncodeToString(rest)
|
||||
var again bytes.Buffer
|
||||
h.Marshal(&again)
|
||||
}
|
||||
cases = append(cases, c)
|
||||
}
|
||||
for _, m := range headerMutations(valid) {
|
||||
add(m)
|
||||
}
|
||||
|
||||
write(t, filepath.Join(dir, "crypto.json"), map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, internal/pqcrypt/oracle_vectors_test.go",
|
||||
"frodo640shake": kemVectors(t, frodo, 8),
|
||||
"mlkem768": kemVectors(t, ml, 8),
|
||||
"hybrid": hybrids,
|
||||
"wrap": wraps,
|
||||
"header_cases": cases,
|
||||
})
|
||||
}
|
||||
|
||||
func itoa(n int) string { return string(appendInt(nil, n)) }
|
||||
|
||||
func appendInt(b []byte, n int) []byte {
|
||||
if n >= 10 {
|
||||
b = appendInt(b, n/10)
|
||||
}
|
||||
return append(b, byte('0'+n%10))
|
||||
}
|
||||
|
||||
// headerMutations returns valid headers plus a range of malformed ones.
|
||||
func headerMutations(valid []byte) [][]byte {
|
||||
s := string(valid)
|
||||
introEnd := bytes.IndexByte(valid, '\n') + 1
|
||||
footer := bytes.LastIndex(valid, []byte("\n---")) + 1
|
||||
lines := bytes.SplitAfter(valid, []byte("\n"))
|
||||
join := func(ls [][]byte) []byte { return bytes.Join(ls, nil) }
|
||||
var out [][]byte
|
||||
push := func(b []byte) { out = append(out, append([]byte(nil), b...)) }
|
||||
push(valid)
|
||||
push(append(append([]byte(nil), valid...), []byte("payload\x00\x01")...))
|
||||
push([]byte(""))
|
||||
push([]byte("fury-encryption.org/v1\n--- " + s[len(s)-44:])) // no stanzas
|
||||
push([]byte("fury-encryption.org/v2\n" + s[introEnd:])) // wrong intro
|
||||
push([]byte("fury-encryption.org/v1" + s[introEnd:])) // intro without newline
|
||||
push(valid[:footer]) // no footer
|
||||
push(valid[:len(valid)-1]) // footer without newline
|
||||
push(append(append([]byte(nil), valid[:len(valid)-1]...), '=', '\n')) // padded MAC
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:])) // double space
|
||||
push([]byte(s[:footer] + "---x " + s[footer+4:])) // bad footer prefix
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-2] + "\n")) // short MAC
|
||||
push([]byte(s[:footer] + "--- " + s[footer+4:len(s)-1] + " x\n"))
|
||||
// Stanza line edits.
|
||||
stanza := lines[1]
|
||||
for _, edit := range []string{"-> \n", "->\n", "-> x\n", "-> x\x7f\n", "-> x é\n", "-x a\n", "-> x y\n", "-> x\r\n"} {
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls[1] = []byte(edit)
|
||||
push(join(ls))
|
||||
_ = stanza
|
||||
}
|
||||
// Body edits: the body is the two lines after the stanza line.
|
||||
body := 2
|
||||
for _, edit := range []string{"\n", "AAAA\n", "AAAA=\n", "A\n", "AB\n", "AAB\n", "AA==\n", " AAAA\n", "AAAA\r\n",
|
||||
string(bytes.Repeat([]byte("A"), 64)) + "\n", string(bytes.Repeat([]byte("A"), 68)) + "\n", string(bytes.Repeat([]byte("A"), 65)) + "\n"} {
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls[body] = []byte(edit)
|
||||
push(join(ls))
|
||||
}
|
||||
{
|
||||
// Drop the empty line that ends a 48-byte body: the stanza then runs
|
||||
// into the footer.
|
||||
ls := append([][]byte(nil), lines...)
|
||||
ls = append(ls[:3], ls[4:]...)
|
||||
push(join(ls))
|
||||
}
|
||||
{
|
||||
// A second, foreign stanza ahead of the real one.
|
||||
ls := append([][]byte(nil), lines[:1]...)
|
||||
ls = append(ls, []byte("-> other-suite arg\n"), []byte("AAAA\n"))
|
||||
ls = append(ls, lines[1:]...)
|
||||
push(join(ls))
|
||||
}
|
||||
// The 1 MiB stanza and 16 MiB header limits are exercised by the Rust
|
||||
// tests directly; vectors that size would bloat the repository.
|
||||
return out
|
||||
}
|
||||
|
||||
func write(t *testing.T, path string, v any) {
|
||||
b, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package stream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// STREAM seal and open of 64 MiB, one goroutine (the sequential path).
|
||||
func BenchmarkOracleStream64MiB(b *testing.B) {
|
||||
key := make([]byte, 32)
|
||||
pt := make([]byte, 64<<20)
|
||||
var ct bytes.Buffer
|
||||
b.SetBytes(int64(len(pt)))
|
||||
for b.Loop() {
|
||||
ct.Reset()
|
||||
w, _ := NewEncryptWriter(key, &ct)
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
r, _ := NewDecryptReader(key, bytes.NewReader(ct.Bytes()))
|
||||
io.Copy(io.Discard, r)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port; see oracle/gen-vectors.sh.
|
||||
|
||||
package stream
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
key := make([]byte, 32)
|
||||
for i := range key {
|
||||
key[i] = byte(i + 1)
|
||||
}
|
||||
type vec struct {
|
||||
Size int `json:"size"`
|
||||
CtLen int `json:"ct_len"`
|
||||
CtSHA256 string `json:"ct_sha256"`
|
||||
CtHex string `json:"ct_hex,omitempty"`
|
||||
}
|
||||
var vs []vec
|
||||
for _, size := range []int{0, 1, 15, 16, 17, ChunkSize - 1, ChunkSize, ChunkSize + 1, 2 * ChunkSize, 2*ChunkSize + 1, 3*ChunkSize - 16, 300000} {
|
||||
pt := make([]byte, size)
|
||||
for i := range pt {
|
||||
pt[i] = byte(i*13 + 7)
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
w, _ := NewEncryptWriter(key, &buf)
|
||||
w.Write(pt)
|
||||
w.Close()
|
||||
// The reader-side encryptor must agree.
|
||||
r, _ := NewEncryptReader(key, bytes.NewReader(pt))
|
||||
alt, _ := io.ReadAll(r)
|
||||
if !bytes.Equal(alt, buf.Bytes()) {
|
||||
t.Fatal("EncryptReader disagrees with EncryptWriter")
|
||||
}
|
||||
s := sha256.Sum256(buf.Bytes())
|
||||
v := vec{Size: size, CtLen: buf.Len(), CtSHA256: hex.EncodeToString(s[:])}
|
||||
if size <= 17 {
|
||||
v.CtHex = hex.EncodeToString(buf.Bytes())
|
||||
}
|
||||
vs = append(vs, v)
|
||||
}
|
||||
b, _ := json.MarshalIndent(map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, internal/stream/oracle_vectors_test.go",
|
||||
"key": hex.EncodeToString(key),
|
||||
"plaintext": "byte(i*13+7)",
|
||||
"vectors": vs,
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "stream.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,100 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port's URL handling and mirror
|
||||
// link mapping; see oracle/gen-vectors.sh.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOracleNetVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
type parsed struct {
|
||||
Input string `json:"input"`
|
||||
Err bool `json:"err"`
|
||||
String string `json:"string,omitempty"`
|
||||
Scheme string `json:"scheme,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
RawQuery string `json:"raw_query,omitempty"`
|
||||
Fragment string `json:"fragment,omitempty"`
|
||||
Local string `json:"local_path,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
describe := func(in string, u *url.URL, err error) parsed {
|
||||
if err != nil {
|
||||
return parsed{Input: in, Err: true}
|
||||
}
|
||||
return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path,
|
||||
RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)}
|
||||
}
|
||||
refs := []string{
|
||||
"", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080",
|
||||
"?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A",
|
||||
"https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..",
|
||||
"a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f",
|
||||
"http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only",
|
||||
}
|
||||
bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"}
|
||||
var parses []parsed
|
||||
for _, r := range refs {
|
||||
u, err := url.Parse(r)
|
||||
parses = append(parses, describe(r, u, err))
|
||||
}
|
||||
type resolved struct {
|
||||
Base string `json:"base"`
|
||||
Ref string `json:"ref"`
|
||||
Result parsed `json:"result"`
|
||||
}
|
||||
var resolves []resolved
|
||||
for _, b := range bases {
|
||||
bu, err := url.Parse(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range refs {
|
||||
u, err := bu.Parse(r)
|
||||
resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)})
|
||||
}
|
||||
}
|
||||
rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"},
|
||||
{"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}}
|
||||
var relOut [][3]string
|
||||
for _, p := range rels {
|
||||
relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])})
|
||||
}
|
||||
escapes := map[string]string{}
|
||||
for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} {
|
||||
escapes[s] = url.QueryEscape(s)
|
||||
}
|
||||
times := map[string]any{}
|
||||
for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} {
|
||||
if tm, err := http.ParseTime(s); err == nil {
|
||||
times[s] = tm.Unix()
|
||||
} else {
|
||||
times[s] = nil
|
||||
}
|
||||
}
|
||||
b, _ := json.MarshalIndent(map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go",
|
||||
"parse": parses,
|
||||
"resolve": resolves,
|
||||
"relative": relOut,
|
||||
"query_escape": escapes,
|
||||
"http_time": times,
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only benchmarks for comparing with the Rust port (scripts/bench.sh).
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Unlocking a key file: Argon2id for both slots.
|
||||
func BenchmarkOracleUnlock(b *testing.B) {
|
||||
p := filepath.Join(b.TempDir(), "key")
|
||||
if err := writeKeyFile(p, "bench", ""); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
for b.Loop() {
|
||||
if _, err := unlockKeyFile(p, "bench"); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,208 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector and fixture generator for the Rust port. It calls the
|
||||
// frozen implementation and records what it computes; it changes nothing in
|
||||
// the vault binary. Run through oracle/gen-vectors.sh.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/andydixon/vault/internal/pqcrypt"
|
||||
"golang.org/x/crypto/argon2"
|
||||
)
|
||||
|
||||
// Synthetic, public test passphrases. Never use them for anything real.
|
||||
const (
|
||||
oracleReal = "oracle-real passphrase"
|
||||
oracleDecoy = "oracle-decoy passphrase"
|
||||
)
|
||||
|
||||
func TestOracleVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
out := map[string]any{"provenance": "oracle/go at 4c1507c, oracle_vectors_test.go"}
|
||||
|
||||
// Key files.
|
||||
keys := map[string]any{}
|
||||
for _, k := range []struct{ name, real, decoy string }{
|
||||
{"key-with-decoy.pem", oracleReal, oracleDecoy},
|
||||
{"key-no-decoy.pem", oracleReal, ""},
|
||||
} {
|
||||
p := filepath.Join(dir, k.name)
|
||||
os.Remove(p)
|
||||
if err := writeKeyFile(p, k.real, k.decoy); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry := map[string]any{}
|
||||
for label, pass := range map[string]string{"real": k.real, "decoy": k.decoy} {
|
||||
if pass == "" {
|
||||
continue
|
||||
}
|
||||
id, err := unlockKeyFile(p, pass)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entry[label] = map[string]any{"passphrase": pass, "suite": id.SuiteID(), "seed": hex.EncodeToString(id.Seed())}
|
||||
}
|
||||
keys[k.name] = entry
|
||||
}
|
||||
out["keys"] = keys
|
||||
|
||||
var argon []map[string]string
|
||||
for i, pass := range []string{"", "a", "correct horse", "pässwörd ü", "x\x00y"} {
|
||||
salt := make([]byte, 16)
|
||||
for j := range salt {
|
||||
salt[j] = byte(i*16 + j)
|
||||
}
|
||||
k := argon2.IDKey([]byte(pass), salt, argonTime, argonMemory, argonThreads, 32)
|
||||
argon = append(argon, map[string]string{"passphrase_hex": hex.EncodeToString([]byte(pass)), "salt": hex.EncodeToString(salt), "key": hex.EncodeToString(k)})
|
||||
}
|
||||
out["argon2id"] = argon
|
||||
out["key_header_hex"] = hex.EncodeToString(keyHeader())
|
||||
|
||||
// Object IDs and sizes.
|
||||
seed := "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
|
||||
var ids []string
|
||||
for i := range int64(4) {
|
||||
ids = append(ids, objectID(seed, i))
|
||||
}
|
||||
out["object_ids"] = map[string]any{"seed": seed, "ids": ids, "big_index": objectID(seed, 1<<40), "big_index_value": int64(1 << 40)}
|
||||
padded := map[string]int64{}
|
||||
counts := map[string]int64{}
|
||||
for _, n := range []int64{0, 1, 16, 65535, 65536, 65537, 1 << 20, chunkSize - 1, chunkSize, chunkData - 1, chunkData, chunkData + 1, 2 * chunkData, fileLimit} {
|
||||
padded[fmt.Sprint(n)] = paddedSize(n)
|
||||
counts[fmt.Sprint(n)] = objectCount(n)
|
||||
}
|
||||
out["padded_size"] = padded
|
||||
out["object_count"] = counts
|
||||
|
||||
names := map[string]bool{}
|
||||
for _, n := range []string{"", "a", "a/b", "/a", "a/", "a//b", "./a", "a/./b", "../a", "a/..", "..", ".", "a/../b", "a\\b", "a\x00b",
|
||||
"a\nb", "a\x7fb", "a\u0085b", "ü/ñ", "\xff", "a b", " a", "...", "a/...", "..a", "a/.b", string(make([]byte, 1024)), longName(1024), longName(1025), "ab", "
"} {
|
||||
names[hex.EncodeToString([]byte(n))] = validName(n)
|
||||
}
|
||||
out["valid_name_hex"] = names
|
||||
|
||||
sizes := map[string]any{}
|
||||
for _, s := range []string{"0", "512", "2k", "500M", "2G", "1TB", "", "G", "-1", "2X", "1.5G", "2T", "+5", "1KB", "1MK", "1kb", "B", "1T", "1099511627776", "1099511627777", " 1", "0x10", "1024G", "1025G", "-0", "1bb"} {
|
||||
n, err := parseSize(s)
|
||||
if err != nil {
|
||||
sizes[s] = nil
|
||||
} else {
|
||||
sizes[s] = n
|
||||
}
|
||||
}
|
||||
out["parse_size"] = sizes
|
||||
|
||||
human := map[string]string{}
|
||||
for _, n := range []int64{0, 1, 1023, 1024, 1025, 1536, 1101, 10239, 10240, 1048575, 1048576, 1 << 30, 1<<40 - 1, 1 << 40, 1<<62 + 12345, 1<<63 - 1, 1075, 1126, 1177, 1638, 1689} {
|
||||
human[fmt.Sprint(n)] = humanSize(n)
|
||||
}
|
||||
out["human_size"] = human
|
||||
|
||||
clean := map[string]string{}
|
||||
for _, p := range []string{"", ".", "..", "/", "//", "a/b/../c", "/../a", "a/./b/", "../../a", "a//b", "./", "/a/b/..", "a/b/../../..", "abc/def/ghi/../.."} {
|
||||
clean[p] = path.Clean(p)
|
||||
}
|
||||
out["path_clean"] = clean
|
||||
|
||||
durs := map[string]any{}
|
||||
for _, d := range []string{"5m", "1h", "90s", "1.5h", "300ms", "1h30m", "0", "-5m", "", "5", "1d", "2h45m30.5s", "1us", "1µs", "1ns", ".5s", "5.s", "+5m", "9223372036854775807ns", "9223372036854775808ns"} {
|
||||
v, err := time.ParseDuration(d)
|
||||
if err != nil {
|
||||
durs[d] = nil
|
||||
} else {
|
||||
durs[d] = map[string]any{"ns": int64(v), "string": v.String()}
|
||||
}
|
||||
}
|
||||
out["durations"] = durs
|
||||
|
||||
// Record JSON, exactly as written.
|
||||
var recs []map[string]any
|
||||
for _, r := range []record{
|
||||
{Name: "plain.txt", Seed: seed, Size: 5, Mode: 0o644, Modified: time.Unix(1700000000, 123456789).UTC(), Added: time.Unix(1700000001, 0).UTC()},
|
||||
{Name: "<a&b>/
'\"\\x", Seed: seed, Size: 0, Mode: 0o755, Modified: time.Unix(0, 0).UTC(), Added: time.Unix(1, 100).UTC()},
|
||||
{Name: "ü/日本語", Seed: seed, Size: 1 << 40, Mode: 0o600, Modified: time.Unix(-1, 500000000).UTC(), Added: time.Unix(253402300799, 999999999).UTC()},
|
||||
{Name: "tab\there", Seed: seed, Size: 1, Mode: 0, Modified: time.Unix(1234567890, 120000000).UTC(), Added: time.Unix(1234567890, 1000).UTC()},
|
||||
} {
|
||||
b, err := json.Marshal(&r)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
recs = append(recs, map[string]any{"json": string(b), "name": r.Name, "modified_unix_nano": r.Modified.UnixNano(), "added_unix_nano": r.Added.UnixNano()})
|
||||
}
|
||||
out["records"] = recs
|
||||
|
||||
writeMiniVault(t, filepath.Join(dir, "mini-vault"), filepath.Join(dir, "key-with-decoy.pem"))
|
||||
|
||||
b, _ := json.MarshalIndent(out, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "store.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func longName(n int) string {
|
||||
b := make([]byte, n)
|
||||
for i := range b {
|
||||
b[i] = 'a' + byte(i%26)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// writeMiniVault builds a small vault through the real store code, without
|
||||
// init's filler (which would make the fixture megabytes): three real files,
|
||||
// one decoy file and one record sealed to a throwaway key.
|
||||
func writeMiniVault(t *testing.T, root, keyPath string) {
|
||||
os.RemoveAll(root)
|
||||
for _, d := range []string{recordsDir, objectsDir} {
|
||||
for i := range 256 {
|
||||
if err := os.MkdirAll(filepath.Join(root, d, fmt.Sprintf("%02x", i)), 0o700); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
os.WriteFile(filepath.Join(root, markerName), []byte(markerContent), 0o600)
|
||||
src := filepath.Join(t.TempDir(), "src")
|
||||
put := func(s *store, name string, body []byte, mode os.FileMode, mtime time.Time) {
|
||||
os.WriteFile(src, body, 0o600)
|
||||
os.Chmod(src, mode)
|
||||
os.Chtimes(src, mtime, mtime)
|
||||
cat, err := s.catalogue()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.add(cat, src, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
real, _ := unlockKeyFile(keyPath, oracleReal)
|
||||
decoy, _ := unlockKeyFile(keyPath, oracleDecoy)
|
||||
throwaway, _ := pqcrypt.GeneratePQIdentity()
|
||||
rs := &store{root: root, id: real}
|
||||
boundary := make([]byte, minPadding-idSize)
|
||||
for i := range boundary {
|
||||
boundary[i] = byte(i * 7)
|
||||
}
|
||||
put(rs, "hello.txt", []byte("hello"), 0o644, time.Unix(1700000000, 123456789))
|
||||
put(rs, "dir/empty", nil, 0o600, time.Unix(1600000000, 0))
|
||||
put(rs, "dir/ünïcode.bin", boundary, 0o755, time.Unix(1500000000, 999999999))
|
||||
put(&store{root: root, id: decoy}, "shopping.txt", []byte("milk\n"), 0o600, time.Unix(1700000000, 0))
|
||||
put(&store{root: root, id: throwaway}, "foreign", []byte("not yours"), 0o600, time.Unix(1700000000, 0))
|
||||
for _, d := range []string{recordsDir, objectsDir} {
|
||||
shards, _ := filepath.Glob(filepath.Join(root, d, "??"))
|
||||
for _, p := range append(shards, filepath.Join(root, d)) {
|
||||
os.Chtimes(p, epoch, epoch)
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+140
@@ -0,0 +1,140 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build the dixon-vault .deb, .rpm and Arch packages locally and show what
|
||||
# is in them. Nothing is signed or published: packaging/repo.sh does that,
|
||||
# reusing this script's build step.
|
||||
#
|
||||
# packaging/package.sh # from the working tree, into dist/packages
|
||||
# packaging/package.sh v0.6 # from a tag (a clean export of it)
|
||||
# OUT=/tmp/pkgs ARCHS="amd64 arm64" packaging/package.sh
|
||||
#
|
||||
# Needs: the pinned Rust toolchain with cargo-zigbuild and zig (the static
|
||||
# Linux binaries are cross-built), and go, only to run the pinned nfpm
|
||||
# packager. dpkg-deb, rpm and zstd are used for inspection when present.
|
||||
# None of these is needed to run vault itself.
|
||||
#
|
||||
# ARCHS package architectures (default below)
|
||||
# RPM_SIGNING_KEY armoured private key file to sign the .rpm with
|
||||
# (repo.sh sets it; unset here means unsigned)
|
||||
set -euo pipefail
|
||||
|
||||
# ---- project ---------------------------------------------------------------
|
||||
NAME=vault # the command
|
||||
# The package: plain "vault" is HashiCorp Vault in Arch's [extra] and in
|
||||
# HashiCorp's own apt and dnf repositories, which would win upgrades. Both
|
||||
# install /usr/bin/vault, so they conflict.
|
||||
PACKAGE=dixon-vault
|
||||
CONFLICTS=vault
|
||||
SUMMARY="Post-quantum encrypted file vault with a decoy passphrase"
|
||||
HOMEPAGE=https://dixon.cx/vault
|
||||
LICENSE=GPL-3.0-or-later
|
||||
MANPAGES=docs/*.1
|
||||
DOCS="README.md LICENSE SECURITY.md"
|
||||
# Package architecture -> Rust target (all static musl binaries).
|
||||
triple() {
|
||||
case $1 in
|
||||
amd64) echo x86_64-unknown-linux-musl ;;
|
||||
arm64) echo aarch64-unknown-linux-musl ;;
|
||||
arm) echo armv7-unknown-linux-musleabihf ;;
|
||||
386) echo i686-unknown-linux-musl ;;
|
||||
riscv64) echo riscv64gc-unknown-linux-musl ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
# ----------------------------------------------------------------------------
|
||||
|
||||
ARCHS=${ARCHS:-"amd64 arm64 arm 386 riscv64"}
|
||||
NFPM="go run github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.41.1"
|
||||
|
||||
say() { echo "package: $*" >&2; }
|
||||
die() { say "$*"; exit 1; }
|
||||
|
||||
# build_packages SRC OUTDIR VERSION: binaries, then one .deb, .rpm and
|
||||
# .pkg.tar.zst per architecture, into OUTDIR.
|
||||
build_packages() {
|
||||
local src=$1 outdir=$2 version=$3 arch nfarch t work
|
||||
for tool in cargo cargo-zigbuild zig go; do
|
||||
command -v $tool >/dev/null || die "needs $tool"
|
||||
done
|
||||
work=$(mktemp -d)
|
||||
mkdir -p "$outdir"
|
||||
for arch in $ARCHS; do
|
||||
t=$(triple "$arch") || die "no Rust target for $arch"
|
||||
nfarch=$arch
|
||||
[[ $arch == arm ]] && nfarch=arm7
|
||||
say "building $NAME for $arch ($t)"
|
||||
flags="--remap-path-prefix=$src=vault"
|
||||
# zig's musl getrandom() never reaches the kernel on riscv64; have
|
||||
# the getrandom crate issue the system call itself.
|
||||
[[ $t == riscv64*-musl ]] && flags+=' --cfg getrandom_backend="linux_raw"'
|
||||
(cd "$src" && RUSTFLAGS=$flags cargo zigbuild --quiet --release --locked --target "$t") || die "build for $arch failed"
|
||||
install -D -m 0755 "$src/target/$t/release/$NAME" "$work/bin-$arch/$NAME"
|
||||
{
|
||||
cat <<-EOF
|
||||
name: $PACKAGE
|
||||
arch: $nfarch
|
||||
platform: linux
|
||||
version: $version
|
||||
release: 1
|
||||
maintainer: Andy Dixon <andy@dixon.cx>
|
||||
description: $SUMMARY
|
||||
homepage: $HOMEPAGE
|
||||
license: $LICENSE
|
||||
conflicts: [${CONFLICTS:-}]
|
||||
contents:
|
||||
- src: $work/bin-$arch/$NAME
|
||||
dst: /usr/bin/$NAME
|
||||
file_info: {mode: 0755}
|
||||
EOF
|
||||
for m in $(cd "$src" && ls $MANPAGES); do
|
||||
printf ' - src: %s\n dst: /usr/share/man/man1/%s\n file_info: {mode: 0644}\n' \
|
||||
"$src/$m" "$(basename "$m")"
|
||||
done
|
||||
for d in $DOCS; do
|
||||
printf ' - src: %s\n dst: /usr/share/doc/%s/%s\n file_info: {mode: 0644}\n' \
|
||||
"$src/$d" "$PACKAGE" "$d"
|
||||
done
|
||||
if [[ -n ${RPM_SIGNING_KEY:-} ]]; then
|
||||
printf 'rpm:\n signature:\n key_file: %s\n' "$RPM_SIGNING_KEY"
|
||||
fi
|
||||
} >"$work/nfpm-$arch.yaml"
|
||||
for fmt in deb rpm archlinux; do
|
||||
$NFPM pkg -f "$work/nfpm-$arch.yaml" -p $fmt -t "$outdir/" >/dev/null
|
||||
done
|
||||
done
|
||||
rm -rf -- "$work"
|
||||
}
|
||||
|
||||
# inspect_packages DIR: list each package's metadata and files.
|
||||
inspect_packages() {
|
||||
local p
|
||||
for p in "$1"/*; do
|
||||
echo "== $(basename "$p")"
|
||||
case $p in
|
||||
*.deb) if command -v dpkg-deb >/dev/null; then dpkg-deb -f "$p" Package Version Architecture Conflicts; dpkg-deb -c "$p"; fi ;;
|
||||
*.rpm) if command -v rpm >/dev/null; then rpm -qpi "$p" 2>/dev/null | head -12; rpm -qplv "$p"; fi ;;
|
||||
*.pkg.tar.zst) if command -v zstd >/dev/null; then zstd -dcq "$p" | tar -tvf -; fi ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
# When run (not sourced by repo.sh): build from the tree or a tag, inspect.
|
||||
if [[ ${BASH_SOURCE[0]} == "$0" ]]; then
|
||||
cd "$(git rev-parse --show-toplevel)"
|
||||
out=${OUT:-dist/packages}
|
||||
mkdir -p -- "$out"
|
||||
out=$(cd -- "$out" && pwd -P)
|
||||
rm -f -- "$out"/*.deb "$out"/*.rpm "$out"/*.pkg.tar.zst
|
||||
if [[ $# -gt 0 ]]; then
|
||||
git rev-parse -q --verify "refs/tags/$1" >/dev/null || die "no tag $1"
|
||||
src=$(mktemp -d)
|
||||
trap 'rm -rf -- "$src"' EXIT
|
||||
git archive "$1" | tar -x -C "$src"
|
||||
version=${1#v}
|
||||
else
|
||||
src=$PWD
|
||||
version=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import sys,json; print(json.load(sys.stdin)["packages"][0]["version"])')
|
||||
fi
|
||||
build_packages "$src" "$out" "$version"
|
||||
inspect_packages "$out"
|
||||
say "$(ls "$out" | wc -l) packages in $out (unsigned, not published)"
|
||||
fi
|
||||
+15
-61
@@ -10,37 +10,25 @@
|
||||
# Every index is rebuilt over all the packages in the repository, so other
|
||||
# projects publishing there with their copy of this script are kept.
|
||||
#
|
||||
# Needs: go, docker, gpg, apt-ftparchive (apt-utils), flock.
|
||||
# Needs: the pinned Rust toolchain with cargo-zigbuild and zig, go (to run
|
||||
# nfpm), docker, gpg, apt-ftparchive (apt-utils), flock.
|
||||
# REPO_DIR where the repository lives (default below)
|
||||
# REPO_GNUPGHOME GnuPG home holding the signing key (default below)
|
||||
# ARCHS Go architectures to package (default below)
|
||||
# ARCHS architectures to package (see packaging/package.sh)
|
||||
#
|
||||
# To build and inspect packages without signing or publishing, run
|
||||
# packaging/package.sh instead.
|
||||
#
|
||||
# The same script, with its own project block, is in ../mapsize/packaging.
|
||||
set -euo pipefail
|
||||
|
||||
# ---- project ---------------------------------------------------------------
|
||||
NAME=vault # the command
|
||||
# The package: plain "vault" is HashiCorp Vault in Arch's [extra] and in
|
||||
# HashiCorp's own apt and dnf repositories, which would win upgrades. Both
|
||||
# install /usr/bin/vault, so they conflict.
|
||||
PACKAGE=dixon-vault
|
||||
CONFLICTS=vault
|
||||
SUMMARY="Post-quantum encrypted file vault with a decoy passphrase"
|
||||
HOMEPAGE=https://dixon.cx/vault
|
||||
LICENSE=GPL-3.0-or-later
|
||||
MANPAGES=docs/*.1
|
||||
DOCS="README.md LICENSE"
|
||||
# build OUT: build the binary to OUT, in the source tree, with GOOS, GOARCH
|
||||
# and GOARM set and VERSION the release.
|
||||
build() {
|
||||
CGO_ENABLED=0 go build -trimpath -buildvcs=false -ldflags='-s -w -buildid=' -o "$1" .
|
||||
}
|
||||
# ----------------------------------------------------------------------------
|
||||
# The project (package name, contents, architectures) and the build live in
|
||||
# packaging/package.sh, which can also be run on its own to build and inspect
|
||||
# packages without signing or publishing anything.
|
||||
. "$(dirname "$0")/package.sh"
|
||||
|
||||
REPO_DIR=${REPO_DIR:-/home/andy/domains/repo.dixon.cx/public_html}
|
||||
REPO_GNUPGHOME=${REPO_GNUPGHOME:-$HOME/.config/repo.dixon.cx/gnupg}
|
||||
ARCHS=${ARCHS:-"amd64 arm64 arm 386 riscv64"}
|
||||
NFPM="go run github.com/goreleaser/nfpm/v2/cmd/nfpm@v2.41.1"
|
||||
SUITE=stable
|
||||
COMPONENT=main
|
||||
ARCH_REPO=dixon # the pacman repository name: [dixon]
|
||||
@@ -48,7 +36,7 @@ ARCH_REPO=dixon # the pacman repository name: [dixon]
|
||||
say() { echo "repo: $*" >&2; }
|
||||
die() { say "$*"; exit 1; }
|
||||
|
||||
for tool in go docker gpg apt-ftparchive flock; do
|
||||
for tool in cargo cargo-zigbuild zig go docker gpg apt-ftparchive flock; do
|
||||
command -v $tool >/dev/null || die "needs $tool"
|
||||
done
|
||||
[[ -d $REPO_DIR ]] || die "$REPO_DIR does not exist"
|
||||
@@ -64,47 +52,13 @@ export VERSION
|
||||
|
||||
WORK=$(mktemp -d)
|
||||
trap 'rm -rf "$WORK"' EXIT
|
||||
git archive "$TAG" | tar -x -C "$WORK" --one-top-level=src
|
||||
mkdir -p "$WORK/src"
|
||||
git archive "$TAG" | tar -x -C "$WORK/src"
|
||||
gpg --batch --armor --export-secret-keys "$KEY" >"$WORK/signing.asc"
|
||||
|
||||
# ---- build -----------------------------------------------------------------
|
||||
# ---- build ----------------------------------------------------------------
|
||||
say "building $PACKAGE $VERSION for $ARCHS"
|
||||
mkdir -p "$WORK/out"
|
||||
for arch in $ARCHS; do
|
||||
goarm= nfarch=$arch
|
||||
[[ $arch == arm ]] && goarm=7 nfarch=arm7
|
||||
(cd "$WORK/src" && GOOS=linux GOARCH=$arch GOARM=$goarm build "$WORK/bin-$arch/$NAME")
|
||||
{
|
||||
cat <<-EOF
|
||||
name: $PACKAGE
|
||||
arch: $nfarch
|
||||
platform: linux
|
||||
version: $VERSION
|
||||
release: 1
|
||||
maintainer: Andy Dixon <andy@dixon.cx>
|
||||
description: $SUMMARY
|
||||
homepage: $HOMEPAGE
|
||||
license: $LICENSE
|
||||
conflicts: [${CONFLICTS:-}]
|
||||
contents:
|
||||
- src: $WORK/bin-$arch/$NAME
|
||||
dst: /usr/bin/$NAME
|
||||
file_info: {mode: 0755}
|
||||
EOF
|
||||
for m in $(cd "$WORK/src" && ls $MANPAGES); do
|
||||
printf ' - src: %s\n dst: /usr/share/man/man1/%s\n file_info: {mode: 0644}\n' \
|
||||
"$WORK/src/$m" "$(basename "$m")"
|
||||
done
|
||||
for d in $DOCS; do
|
||||
printf ' - src: %s\n dst: /usr/share/doc/%s/%s\n file_info: {mode: 0644}\n' \
|
||||
"$WORK/src/$d" "$PACKAGE" "$d"
|
||||
done
|
||||
printf 'rpm:\n signature:\n key_file: %s\n' "$WORK/signing.asc"
|
||||
} >"$WORK/nfpm-$arch.yaml"
|
||||
for fmt in deb rpm archlinux; do
|
||||
$NFPM pkg -f "$WORK/nfpm-$arch.yaml" -p $fmt -t "$WORK/out/" >/dev/null
|
||||
done
|
||||
done
|
||||
RPM_SIGNING_KEY=$WORK/signing.asc build_packages "$WORK/src" "$WORK/out" "$VERSION"
|
||||
|
||||
# ---- publish ---------------------------------------------------------------
|
||||
# One publisher at a time: the indexes cover every project's packages.
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
[toolchain]
|
||||
channel = "1.99.0"
|
||||
components = ["rustfmt", "clippy"]
|
||||
profile = "minimal"
|
||||
@@ -0,0 +1 @@
|
||||
max_width = 140
|
||||
Executable
+113
@@ -0,0 +1,113 @@
|
||||
#!/usr/bin/env bash
|
||||
# Compare the Go oracle and the Rust build on this machine, end to end.
|
||||
#
|
||||
# scripts/bench.sh [WORKDIR] (default: target/bench, on the repo's disk)
|
||||
# RUNS=5 BIG_MB=2048 SMALL=5000 scripts/bench.sh
|
||||
#
|
||||
# For each implementation, in a fresh key and vault on the same filesystem:
|
||||
# add one BIG_MB file, and add SMALL small files (1-16 KiB) as a folder, each
|
||||
# into a fresh vault; then, in one vault holding both, get the big file,
|
||||
# list the catalogue (filler plus SMALL+1 records), and get one small file. Each step is timed RUNS times (median, min, max), with peak RSS and
|
||||
# peak thread count. Every step includes unlocking the key (two Argon2id
|
||||
# runs). Both implementations fsync every object and record, so `add`
|
||||
# timings include durable writes. Page caches are warm: dropping them needs
|
||||
# root, so cold-cache numbers are not measured here.
|
||||
#
|
||||
# Needs: go (to build the oracle), cargo, /usr/bin/time (GNU).
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
root=$PWD
|
||||
work=${1:-$root/target/bench}
|
||||
runs=${RUNS:-3}
|
||||
big_mb=${BIG_MB:-1024}
|
||||
small=${SMALL:-500}
|
||||
|
||||
mkdir -p -- "$work"
|
||||
work=$(cd -- "$work" && pwd -P)
|
||||
[[ $work == "$root" || $work == / ]] && { echo "refusing WORKDIR=$work" >&2; exit 2; }
|
||||
|
||||
cargo build --release --locked --quiet
|
||||
(cd oracle/go && CGO_ENABLED=0 go build -trimpath -o "$work/vault-go" .)
|
||||
cp target/release/vault "$work/vault-rust"
|
||||
|
||||
data=$work/data
|
||||
if [[ ! -f $data/big || $(stat -c %s "$data/big") != $((big_mb << 20)) ]]; then
|
||||
rm -rf -- "$data"
|
||||
mkdir -p -- "$data/small"
|
||||
head -c $((big_mb << 20)) /dev/urandom >"$data/big"
|
||||
for i in $(seq -w 1 "$small"); do
|
||||
head -c $((1024 + RANDOM % 15360)) /dev/urandom >"$data/small/f$i"
|
||||
done
|
||||
fi
|
||||
|
||||
export VAULT_PASSPHRASE=bench-passphrase
|
||||
|
||||
# measure LABEL CMD...: one timed run, printing "seconds rss_kib threads".
|
||||
measure() {
|
||||
local out=$work/time.$$ peak=0 pid t
|
||||
/usr/bin/time -f '%e %M' -o "$out" "$@" >/dev/null 2>&1 &
|
||||
pid=$!
|
||||
while kill -0 $pid 2>/dev/null; do
|
||||
for t in /proc/[0-9]*/status; do
|
||||
if grep -q "^PPid:[[:space:]]*$pid$" "$t" 2>/dev/null; then
|
||||
n=$(awk '/^Threads:/{print $2}' "$t" 2>/dev/null || echo 0)
|
||||
((n > peak)) && peak=$n
|
||||
fi
|
||||
done
|
||||
sleep 0.05
|
||||
done
|
||||
wait $pid
|
||||
echo "$(cat "$out") $peak"
|
||||
rm -f -- "$out"
|
||||
}
|
||||
|
||||
summarise() { # lines of "secs rss threads" -> median min max, max rss, max threads
|
||||
sort -n | awk '{t[NR]=$1; if($2>r)r=$2; if($3>th)th=$3} END {printf "%8.2f s (min %.2f, max %.2f) rss %6.0f MiB threads %d\n", t[int((NR+1)/2)], t[1], t[NR], r/1024, th}'
|
||||
}
|
||||
|
||||
fresh() { # fresh key and vault at $1
|
||||
rm -rf -- "$1"
|
||||
mkdir -p -- "$1"
|
||||
export VAULT_KEY=$1/key VAULT_DIR=$1/store
|
||||
"$bin" genkey 2>/dev/null
|
||||
"$bin" init 2>/dev/null
|
||||
}
|
||||
|
||||
for impl in go rust; do
|
||||
bin=$work/vault-$impl
|
||||
echo "== $impl ($(stat -c %s "$bin") bytes)"
|
||||
# Writes: a fresh vault for every run.
|
||||
for step in add-big add-small; do
|
||||
results=""
|
||||
for run in $(seq 1 "$runs"); do
|
||||
fresh "$work/run-$impl"
|
||||
case $step in
|
||||
add-big) r=$(measure "$bin" add "$data/big") ;;
|
||||
add-small) r=$(measure "$bin" add "$data/small") ;;
|
||||
esac
|
||||
results+="$r"$'\n'
|
||||
done
|
||||
printf ' %-10s ' "$step"
|
||||
printf '%s' "$results" | grep . | summarise
|
||||
done
|
||||
# Reads: one populated vault, read repeatedly.
|
||||
v=$work/run-$impl
|
||||
fresh "$v"
|
||||
"$bin" add "$data/big" 2>/dev/null
|
||||
"$bin" add "$data/small" 2>/dev/null
|
||||
for step in get-big list get-small; do
|
||||
results=""
|
||||
for run in $(seq 1 "$runs"); do
|
||||
rm -f -- "$v/big.out" "$v/small.out"
|
||||
case $step in
|
||||
get-big) r=$(measure "$bin" get big -o "$v/big.out") ;;
|
||||
list) r=$(measure "$bin" list) ;;
|
||||
get-small) r=$(measure "$bin" get "small/f$(printf "%0${#small}d" 1)" -o "$v/small.out") ;;
|
||||
esac
|
||||
results+="$r"$'\n'
|
||||
done
|
||||
printf ' %-10s ' "$step"
|
||||
printf '%s' "$results" | grep . | summarise
|
||||
done
|
||||
rm -rf -- "$work/run-$impl"
|
||||
done
|
||||
Executable
+43
@@ -0,0 +1,43 @@
|
||||
#!/usr/bin/env bash
|
||||
# Runtime smoke test of one vault binary, optionally under an emulator:
|
||||
# create a key and vault, store files across object boundaries, read them
|
||||
# back, then (if a reference binary is given) check the reference can read
|
||||
# what this binary wrote and the other way round.
|
||||
#
|
||||
# scripts/smoke.sh BINARY [REFERENCE_BINARY]
|
||||
# EMU="qemu-aarch64" scripts/smoke.sh dist/vault-linux-arm64 target/release/vault
|
||||
#
|
||||
# Uses a private temporary directory, removed afterwards. The passphrase is
|
||||
# a public test value.
|
||||
set -euo pipefail
|
||||
bin=$(realpath "$1")
|
||||
ref=${2:+$(realpath "$2")}
|
||||
emu=${EMU:-}
|
||||
work=$(mktemp -d "${TMPDIR:-/tmp}/vault-smoke.XXXXXX")
|
||||
trap 'rm -rf -- "$work"' EXIT
|
||||
cd "$work"
|
||||
export VAULT_PASSPHRASE=smoke-test-passphrase VAULT_KEY=$work/key VAULT_DIR=$work/store
|
||||
export XDG_RUNTIME_DIR=$work/run
|
||||
mkdir -m 700 run
|
||||
v() { $emu "$bin" "$@"; }
|
||||
|
||||
v genkey 2>/dev/null
|
||||
v init 2>/dev/null
|
||||
head -c $((4 * 1024 * 1024 + 123)) /dev/urandom >big
|
||||
printf 'small\n' >small
|
||||
: >empty
|
||||
v add big 2>/dev/null
|
||||
v add small dir 2>/dev/null
|
||||
v add empty dir 2>/dev/null
|
||||
[[ $(v list | wc -l) == 3 ]] || { echo "FAIL list"; exit 1; }
|
||||
v get big -o big.out 2>/dev/null && cmp big big.out
|
||||
v get dir/small -o small.out 2>/dev/null && cmp small small.out
|
||||
if [[ -n $ref ]]; then
|
||||
"$ref" get big -o big.ref 2>/dev/null && cmp big big.ref
|
||||
printf 'from the reference\n' >r.txt
|
||||
"$ref" add r.txt 2>/dev/null
|
||||
v get r.txt -o r.out 2>/dev/null && cmp r.txt r.out
|
||||
fi
|
||||
v rm -rf dir 2>/dev/null
|
||||
[[ $(v list | wc -l) == $([[ -n $ref ]] && echo 2 || echo 1) ]] || { echo "FAIL rm"; exit 1; }
|
||||
echo "ok $(basename "$bin")${emu:+ (under $emu)}"
|
||||
+325
@@ -0,0 +1,325 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! `vault absorb`: add every regular file of a tar archive (optionally
|
||||
//! gzipped, possibly in several gzip members) read from a stream, each
|
||||
//! streamed straight into the vault so no plaintext touches the disk.
|
||||
//! Other entry types, and names that would escape the destination folder,
|
||||
//! are reported and skipped, as in the Go version.
|
||||
|
||||
use crate::error::{Error, Kind, Result};
|
||||
use crate::store::{Catalogue, Store, VaultName};
|
||||
use crate::util::{Instant, clean, join};
|
||||
use std::io::{BufRead, Read};
|
||||
|
||||
pub fn absorb(
|
||||
store: &Store,
|
||||
cat: &mut Catalogue,
|
||||
mut input: impl BufRead,
|
||||
dest: &str,
|
||||
report: &mut dyn FnMut(&VaultName, bool),
|
||||
) -> Result<()> {
|
||||
let gz = input
|
||||
.fill_buf()
|
||||
.map_err(|e| Error::new(format!("reading archive: {}", crate::util::io_reason(&e))))?
|
||||
.starts_with(&[0x1f, 0x8b]);
|
||||
let r: Box<dyn Read> = if gz {
|
||||
Box::new(flate2::bufread::MultiGzDecoder::new(input))
|
||||
} else {
|
||||
Box::new(input)
|
||||
};
|
||||
let mut archive = tar::Archive::new(r);
|
||||
let entries = archive
|
||||
.entries()
|
||||
.map_err(|e| Error::new(format!("reading archive: {}", crate::util::io_reason(&e))))?;
|
||||
let mut n = 0u64;
|
||||
for entry in entries {
|
||||
let mut entry = entry.map_err(|e| Error::new(format!("reading archive: {}", crate::util::io_reason(&e))))?;
|
||||
let raw = entry.path_bytes().into_owned();
|
||||
let shown = String::from_utf8_lossy(&raw).into_owned();
|
||||
let kind = entry.header().entry_type();
|
||||
if !kind.is_file() {
|
||||
if !kind.is_dir() {
|
||||
eprintln!("skipping {shown}: not a regular file");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let Ok(text) = std::str::from_utf8(&raw) else {
|
||||
crate::bail!("invalid vault path {shown:?}");
|
||||
};
|
||||
let Some(rel) = safe_name(text) else {
|
||||
eprintln!("skipping {shown}: unsafe name");
|
||||
continue;
|
||||
};
|
||||
let name = join(&[dest, &rel]);
|
||||
let name = VaultName::new(&name).ok_or_else(|| Error::new(format!("invalid vault path {name:?}")))?;
|
||||
// The entry's size, with a PAX "size" override applied, as Go's
|
||||
// hdr.Size; the fixed header field alone would truncate files that
|
||||
// Go's archive/tar writes for sizes past 8 GiB.
|
||||
let size = entry.size();
|
||||
if is_gnu_sparse(&mut entry) {
|
||||
eprintln!("skipping {shown}: sparse files are not supported");
|
||||
continue;
|
||||
}
|
||||
let mode = entry.header().mode().unwrap_or(0) & 0o777;
|
||||
let mtime = mtime(&mut entry)?;
|
||||
match store.add_reader(cat, &mut entry, size, mode, mtime, &name) {
|
||||
Ok(a) => report(&name, a.replaced),
|
||||
Err(e) if e.kind == Kind::SourceShort => crate::bail!("{shown}: archive ends early"),
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
n += 1;
|
||||
}
|
||||
if n == 0 {
|
||||
crate::bail!("no files in archive");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Go's check: the name, cleaned as if rooted, must equal the name with a
|
||||
/// leading "./" or "/" removed and cleaned, and contain no backslash.
|
||||
pub fn safe_name(name: &str) -> Option<String> {
|
||||
let rooted = clean(&format!("/{name}"));
|
||||
let n = rooted.strip_prefix('/').unwrap_or(&rooted).to_string();
|
||||
let plain = name.strip_prefix("./").unwrap_or(name);
|
||||
let plain = plain.strip_prefix('/').unwrap_or(plain);
|
||||
if name.contains('\\') || n.is_empty() || n != clean(plain) {
|
||||
return None;
|
||||
}
|
||||
Some(n)
|
||||
}
|
||||
|
||||
/// A PAX-format GNU sparse file, whose data is an encoded map, not the
|
||||
/// file's bytes.
|
||||
fn is_gnu_sparse<R: Read>(entry: &mut tar::Entry<R>) -> bool {
|
||||
if entry.header().entry_type() == tar::EntryType::GNUSparse {
|
||||
return true;
|
||||
}
|
||||
match entry.pax_extensions() {
|
||||
Ok(Some(exts)) => exts.flatten().any(|e| e.key_bytes().starts_with(b"GNU.sparse.")),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// The entry's modification time: the PAX `mtime` (with its fraction) if
|
||||
/// present, else the header's whole seconds.
|
||||
fn mtime<R: Read>(entry: &mut tar::Entry<R>) -> Result<Instant> {
|
||||
if let Ok(Some(exts)) = entry.pax_extensions() {
|
||||
for ext in exts.flatten() {
|
||||
if ext.key_bytes() == b"mtime"
|
||||
&& let Some(t) = std::str::from_utf8(ext.value_bytes()).ok().and_then(parse_pax_time)
|
||||
{
|
||||
return Ok(t);
|
||||
}
|
||||
}
|
||||
}
|
||||
let secs = entry
|
||||
.header()
|
||||
.mtime()
|
||||
.map_err(|e| Error::new(format!("reading archive: {}", crate::util::io_reason(&e))))?;
|
||||
Ok(Instant {
|
||||
secs: secs.min(i64::MAX as u64) as i64,
|
||||
nanos: 0,
|
||||
})
|
||||
}
|
||||
|
||||
/// A PAX time: optional sign, decimal seconds, optional fraction.
|
||||
fn parse_pax_time(s: &str) -> Option<Instant> {
|
||||
let (neg, s) = match s.strip_prefix('-') {
|
||||
Some(r) => (true, r),
|
||||
None => (false, s),
|
||||
};
|
||||
let (whole, frac) = s.split_once('.').unwrap_or((s, ""));
|
||||
if whole.is_empty() || !whole.bytes().all(|c| c.is_ascii_digit()) || !frac.bytes().all(|c| c.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
let secs: i64 = whole.parse().ok()?;
|
||||
let mut nanos: u32 = format!("{:0<9}", &frac[..frac.len().min(9)]).parse().ok()?;
|
||||
let mut secs = if neg { -secs } else { secs };
|
||||
if neg && nanos > 0 {
|
||||
secs -= 1;
|
||||
nanos = 1_000_000_000 - nanos;
|
||||
}
|
||||
Some(Instant { secs, nanos })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::crypto::kem::{Identity, SUITE_ID};
|
||||
use crate::pipeline::Pool;
|
||||
use crate::testutil::TempDir;
|
||||
use std::sync::Arc;
|
||||
|
||||
fn store(tag: &str) -> (TempDir, Store) {
|
||||
let d = TempDir::new(tag);
|
||||
let root = d.0.join("v");
|
||||
Store::init(&root, 0, Arc::new(Pool::new(2))).unwrap();
|
||||
let s = Store::open(
|
||||
&root,
|
||||
Identity::from_suite_seed(SUITE_ID, &[1; 32]).unwrap(),
|
||||
Arc::new(Pool::new(2)),
|
||||
)
|
||||
.unwrap();
|
||||
(d, s)
|
||||
}
|
||||
|
||||
fn header(name: &str, kind: tar::EntryType, size: u64, mode: u32, mtime: u64) -> tar::Header {
|
||||
let mut h = tar::Header::new_gnu();
|
||||
h.set_entry_type(kind);
|
||||
h.set_size(size);
|
||||
h.set_mode(mode);
|
||||
h.set_mtime(mtime);
|
||||
// Raw bytes, so unsafe names reach the reader unaltered.
|
||||
let n = name.as_bytes();
|
||||
h.as_old_mut().name[..n.len()].copy_from_slice(n);
|
||||
h.set_cksum();
|
||||
h
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn absorbs_regular_files_only() {
|
||||
let (d, s) = store("absorb");
|
||||
let mut tw = tar::Builder::new(Vec::new());
|
||||
for (name, kind, body) in [
|
||||
("./d/", tar::EntryType::Directory, &b""[..]),
|
||||
("./d/a.txt", tar::EntryType::Regular, b"hello"),
|
||||
("../escape", tar::EntryType::Regular, b"nope!"),
|
||||
("/abs", tar::EntryType::Regular, b"abs"),
|
||||
("a\\b", tar::EntryType::Regular, b"bs"),
|
||||
("./d/link", tar::EntryType::Symlink, b""),
|
||||
] {
|
||||
let h = header(name, kind, body.len() as u64, 0o600, 1577934245);
|
||||
tw.append(&h, body).unwrap();
|
||||
}
|
||||
let tarball = tw.into_inner().unwrap();
|
||||
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
std::io::Write::write_all(&mut gz, &tarball).unwrap();
|
||||
let gz = gz.finish().unwrap();
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let mut added = Vec::new();
|
||||
absorb(&s, &mut cat, &gz[..], "in", &mut |n, _| added.push(n.to_string())).unwrap();
|
||||
assert_eq!(added, ["in/d/a.txt", "in/abs"]);
|
||||
let r = cat.current("in/d/a.txt").unwrap();
|
||||
assert_eq!(
|
||||
(r.mode, r.modified),
|
||||
(
|
||||
0o600,
|
||||
Instant {
|
||||
secs: 1577934245,
|
||||
nanos: 0
|
||||
}
|
||||
)
|
||||
);
|
||||
let dir = crate::platform::fs::Dir::open(&d.0).unwrap();
|
||||
s.extract(r, &dir, std::ffi::OsStr::new("out")).unwrap();
|
||||
assert_eq!(std::fs::read(d.0.join("out")).unwrap(), b"hello");
|
||||
// Plain tar works too, and an archive with nothing to add fails.
|
||||
absorb(&s, &mut cat, &tarball[..], "", &mut |_, _| {}).unwrap();
|
||||
let mut empty = tar::Builder::new(Vec::new());
|
||||
empty
|
||||
.append(&header("dir/", tar::EntryType::Directory, 0, 0o755, 0), &b""[..])
|
||||
.unwrap();
|
||||
let e = absorb(&s, &mut cat, &empty.into_inner().unwrap()[..], "", &mut |_, _| {}).unwrap_err();
|
||||
assert_eq!(e.msg, "no files in archive");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn truncated_and_malicious_archives() {
|
||||
let (_d, s) = store("absorb-bad");
|
||||
let mut tw = tar::Builder::new(Vec::new());
|
||||
tw.append(&header("big", tar::EntryType::Regular, 100_000, 0o600, 0), &vec![7u8; 100_000][..])
|
||||
.unwrap();
|
||||
let t = tw.into_inner().unwrap();
|
||||
let before = s.catalogue().unwrap().len();
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
let e = absorb(&s, &mut cat, &t[..60_000], "", &mut |_, _| {}).unwrap_err();
|
||||
assert!(e.msg.contains("archive ends early") || e.msg.starts_with("reading archive"), "{e}");
|
||||
assert_eq!(s.catalogue().unwrap().len(), before, "a truncated entry stored something");
|
||||
for junk in [
|
||||
&b"not a tar archive at all, but long enough to fill a header block"[..],
|
||||
&[0x1f, 0x8b, 0, 0][..],
|
||||
&[0u8; 1024][..],
|
||||
] {
|
||||
assert!(absorb(&s, &mut cat, junk, "", &mut |_, _| {}).is_err());
|
||||
}
|
||||
// A corrupted gzip stream fails rather than storing a partial file.
|
||||
let mut gz = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default());
|
||||
std::io::Write::write_all(&mut gz, &t).unwrap();
|
||||
let mut g = gz.finish().unwrap();
|
||||
let mid = g.len() / 2;
|
||||
g[mid] ^= 0xff;
|
||||
assert!(absorb(&s, &mut cat, &g[..], "", &mut |_, _| {}).is_err());
|
||||
assert_eq!(s.catalogue().unwrap().len(), before);
|
||||
}
|
||||
|
||||
/// A PAX "size" record overrides the header's size field, as Go writes
|
||||
/// for files over 8 GiB; the whole body must be stored.
|
||||
#[test]
|
||||
fn pax_size_override() {
|
||||
let (d, s) = store("absorb-pax");
|
||||
let mut t = Vec::new();
|
||||
let rec = |k: &str, v: &str| {
|
||||
let body = format!(" {k}={v}\n");
|
||||
let mut n = body.len() + 1;
|
||||
while format!("{n}{body}").len() != n {
|
||||
n += 1;
|
||||
}
|
||||
format!("{n}{body}")
|
||||
};
|
||||
let pax = rec("size", "11");
|
||||
let mut h = header("PaxHeaders/big", tar::EntryType::XHeader, pax.len() as u64, 0o644, 0);
|
||||
h.set_cksum();
|
||||
t.extend_from_slice(h.as_bytes());
|
||||
let mut p = pax.into_bytes();
|
||||
p.resize(p.len().div_ceil(512) * 512, 0);
|
||||
t.extend_from_slice(&p);
|
||||
let h = header("big.bin", tar::EntryType::Regular, 0, 0o644, 0);
|
||||
t.extend_from_slice(h.as_bytes());
|
||||
let mut body = b"hello world".to_vec();
|
||||
body.resize(512, 0);
|
||||
t.extend_from_slice(&body);
|
||||
t.extend_from_slice(&[0u8; 1024]);
|
||||
let mut cat = s.catalogue().unwrap();
|
||||
absorb(&s, &mut cat, &t[..], "", &mut |_, _| {}).unwrap();
|
||||
let r = cat.current("big.bin").unwrap();
|
||||
assert_eq!(r.size, 11);
|
||||
let dir = crate::platform::fs::Dir::open(&d.0).unwrap();
|
||||
s.extract(r, &dir, std::ffi::OsStr::new("o")).unwrap();
|
||||
assert_eq!(std::fs::read(d.0.join("o")).unwrap(), b"hello world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn names_and_times() {
|
||||
// As in Go, only names that would climb out are refused; others are
|
||||
// cleaned.
|
||||
for (n, want) in [
|
||||
("./a/b", Some("a/b")),
|
||||
("a/../b", Some("b")),
|
||||
("../x", None),
|
||||
("a/../../x", None),
|
||||
("/x", Some("x")),
|
||||
("a//b", Some("a/b")),
|
||||
("a\\b", None),
|
||||
("", None),
|
||||
(".", None),
|
||||
("a/", Some("a")),
|
||||
] {
|
||||
assert_eq!(safe_name(n).as_deref(), want, "{n:?}");
|
||||
}
|
||||
assert_eq!(
|
||||
parse_pax_time("1577934245.5"),
|
||||
Some(Instant {
|
||||
secs: 1577934245,
|
||||
nanos: 500_000_000
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
parse_pax_time("-1.25"),
|
||||
Some(Instant {
|
||||
secs: -2,
|
||||
nanos: 750_000_000
|
||||
})
|
||||
);
|
||||
assert_eq!(parse_pax_time("x"), None);
|
||||
}
|
||||
}
|
||||
+420
@@ -0,0 +1,420 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The key agent, wire-compatible with the Go version: `vault unlock`
|
||||
//! starts a copy of this binary as a hidden `__agent` process, hands it the
|
||||
//! suite ID and master seed over a pipe (never argv, never disk), and the
|
||||
//! agent serves them on a Unix socket in a per-user 0700 directory until a
|
||||
//! deadline, fixed or (with `--idle`) moved on each use, or `vault lock`.
|
||||
//!
|
||||
//! Protocol: the client connects and sends one byte, `g` (get: the agent
|
||||
//! replies with the 2-byte big-endian suite ID and 32-byte seed) or `q`
|
||||
//! (quit: the agent wipes the key and closes). Every request has a
|
||||
//! two-second deadline. The socket is
|
||||
//! `$XDG_RUNTIME_DIR` (else the temp dir) `/vault-UID/agent-` + the first 8
|
||||
//! bytes of SHA-256(absolute key path) in hex, exactly as Go computes it,
|
||||
//! so a Go client can use a Rust agent and the other way round.
|
||||
//!
|
||||
//! Before it accepts the secret, the agent disables core dumps, makes
|
||||
//! itself undumpable (Linux) and locks the page that will hold the secret;
|
||||
//! if any of that fails it reports why and exits without the key.
|
||||
|
||||
use crate::crypto::kem::Identity;
|
||||
use crate::error::{Error, Result};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
pub const AGENT_CMD: &str = "__agent";
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const GET: u8 = b'g';
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const QUIT: u8 = b'q';
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const REPLY: usize = 2 + crate::crypto::kem::SEED_SIZE;
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const TIMEOUT: Duration = Duration::from_secs(2);
|
||||
/// How long `unlock` waits for a starting agent to say it is ready.
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const START_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
#[cfg_attr(not(unix), allow(dead_code))]
|
||||
const SECRET_ENVS: [&str; 2] = ["VAULT_PASSPHRASE", "VAULT_DECOY_PASSPHRASE"];
|
||||
|
||||
/// Go's `filepath.Abs`: lexical, with `os.Getwd`'s preference for `$PWD`.
|
||||
pub fn go_abs(p: &Path) -> std::io::Result<PathBuf> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::ffi::OsStrExt;
|
||||
let joined = if p.is_absolute() { p.to_path_buf() } else { go_getwd()?.join(p) };
|
||||
let s = std::str::from_utf8(joined.as_os_str().as_bytes()).map_err(|_| std::io::Error::other("key path is not valid UTF-8"))?;
|
||||
Ok(PathBuf::from(crate::util::clean(s)))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
std::path::absolute(p)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn go_getwd() -> std::io::Result<PathBuf> {
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let dot = std::fs::metadata(".")?;
|
||||
if let Some(pwd) = std::env::var_os("PWD").map(PathBuf::from)
|
||||
&& pwd.is_absolute()
|
||||
&& let Ok(m) = std::fs::metadata(&pwd)
|
||||
&& m.dev() == dot.dev()
|
||||
&& m.ino() == dot.ino()
|
||||
{
|
||||
return Ok(pwd);
|
||||
}
|
||||
std::env::current_dir()
|
||||
}
|
||||
|
||||
/// The agent socket for the key file at `key`, creating the private
|
||||
/// per-user directory if needed and refusing one that is not private.
|
||||
pub fn socket_path(key: &Path) -> Result<PathBuf> {
|
||||
use sha2::Digest;
|
||||
let abs = go_abs(key).map_err(|e| Error::io("abs", key, &e))?;
|
||||
let base = std::env::var_os("XDG_RUNTIME_DIR")
|
||||
.filter(|v| !v.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(go_temp_dir);
|
||||
let dir = base.join(format!("vault-{}", uid()));
|
||||
match crate::platform::fs::create_private_dir(&dir) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {}
|
||||
Err(e) => return Err(Error::io("mkdir", &dir, &e)),
|
||||
}
|
||||
private_dir(&dir)?;
|
||||
let sum = sha2::Sha256::digest(abs.as_os_str().as_encoded_bytes());
|
||||
Ok(dir.join(format!("agent-{}", crate::store::hex(&sum[..8]))))
|
||||
}
|
||||
|
||||
/// Go's `os.TempDir`: `$TMPDIR` or /tmp on Unix.
|
||||
fn go_temp_dir() -> PathBuf {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
std::env::var_os("TMPDIR")
|
||||
.filter(|v| !v.is_empty())
|
||||
.map(PathBuf::from)
|
||||
.unwrap_or_else(|| PathBuf::from("/tmp"))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
std::env::temp_dir()
|
||||
}
|
||||
}
|
||||
|
||||
fn uid() -> u32 {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
rustix::process::getuid().as_raw()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
// Go's os.Getuid returns -1 on Windows.
|
||||
u32::MAX
|
||||
}
|
||||
}
|
||||
|
||||
/// A real directory, owned by this user, with mode exactly 0700.
|
||||
#[cfg(unix)]
|
||||
fn private_dir(dir: &Path) -> Result<()> {
|
||||
use std::os::unix::fs::{MetadataExt, PermissionsExt};
|
||||
let m = std::fs::symlink_metadata(dir).map_err(|e| Error::io("lstat", dir, &e))?;
|
||||
if !m.is_dir() || m.uid() != uid() || m.permissions().mode() & 0o777 != 0o700 {
|
||||
crate::bail!("{} is not a private directory of this user", dir.display());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn private_dir(_: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
mod imp {
|
||||
use super::*;
|
||||
use crate::platform::memory::{LockedBuf, harden_process};
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
|
||||
fn dial(key: &Path, op: u8) -> Result<UnixStream> {
|
||||
let sock = socket_path(key)?;
|
||||
let c = UnixStream::connect(&sock).map_err(|e| Error::io("dial", &sock, &e))?;
|
||||
c.set_read_timeout(Some(TIMEOUT))?;
|
||||
c.set_write_timeout(Some(TIMEOUT))?;
|
||||
(&c).write_all(&[op])?;
|
||||
Ok(c)
|
||||
}
|
||||
|
||||
/// The identity a running agent holds for `key`.
|
||||
pub fn identity(key: &Path) -> Result<Identity> {
|
||||
let mut c = dial(key, GET)?;
|
||||
let mut buf = crate::secret::SecretArray::<REPLY>::zero();
|
||||
c.read_exact(buf.expose_mut())?;
|
||||
let b = buf.expose();
|
||||
Identity::from_suite_seed(u16::from_be_bytes([b[0], b[1]]), &b[2..]).map_err(Error::from)
|
||||
}
|
||||
|
||||
/// Tells the agent for `key` to forget it; whether one was running.
|
||||
pub fn stop(key: &Path) -> bool {
|
||||
let Ok(mut c) = dial(key, QUIT) else { return false };
|
||||
// Returns once the agent has wiped the key and closed.
|
||||
let _ = std::io::copy(&mut c, &mut std::io::sink());
|
||||
true
|
||||
}
|
||||
|
||||
/// The command that starts an agent: this binary, with passphrase
|
||||
/// variables removed from its environment and nothing secret in argv.
|
||||
pub fn command(sock: &Path, ttl: Duration, idle: bool) -> Result<std::process::Command> {
|
||||
let exe =
|
||||
std::env::current_exe().map_err(|e| Error::new(format!("cannot find this executable: {}", crate::util::io_reason(&e))))?;
|
||||
let mut c = std::process::Command::new(exe);
|
||||
c.arg(AGENT_CMD)
|
||||
.arg(sock)
|
||||
.arg(crate::util::format_duration(ttl))
|
||||
.arg(if idle { "idle" } else { "fixed" });
|
||||
for v in SECRET_ENVS {
|
||||
c.env_remove(v);
|
||||
}
|
||||
c.current_dir("/")
|
||||
.stdin(std::process::Stdio::piped())
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::null());
|
||||
Ok(c)
|
||||
}
|
||||
|
||||
/// Replaces any agent for `key` with one holding `id` for `ttl`.
|
||||
pub fn start(key: &Path, id: &Identity, ttl: Duration, idle: bool) -> Result<()> {
|
||||
let sock = socket_path(key)?;
|
||||
stop(key);
|
||||
let mut child = command(&sock, ttl, idle)?
|
||||
.spawn()
|
||||
.map_err(|e| Error::new(format!("cannot start the agent: {}", crate::util::io_reason(&e))))?;
|
||||
let mut secret = crate::secret::SecretArray::<REPLY>::zero();
|
||||
secret.expose_mut()[..2].copy_from_slice(&id.suite_id().to_be_bytes());
|
||||
secret.expose_mut()[2..].copy_from_slice(id.seed());
|
||||
let mut stdin = child.stdin.take().expect("piped");
|
||||
let wrote = stdin.write_all(secret.expose());
|
||||
drop(secret);
|
||||
drop(stdin);
|
||||
let mut stdout = child.stdout.take().expect("piped");
|
||||
let (tx, rx) = std::sync::mpsc::channel();
|
||||
std::thread::spawn(move || {
|
||||
let mut msg = Vec::new();
|
||||
let _ = stdout.by_ref().take(4096).read_to_end(&mut msg);
|
||||
let _ = tx.send(msg);
|
||||
});
|
||||
// The agent closes its standard output once listening, after saying
|
||||
// "ok" or why not.
|
||||
let msg = rx.recv_timeout(START_TIMEOUT);
|
||||
let fail = |child: &mut std::process::Child, why: String| {
|
||||
let _ = child.kill();
|
||||
let _ = child.wait();
|
||||
Err(Error::new(why))
|
||||
};
|
||||
if let Err(e) = wrote {
|
||||
return fail(
|
||||
&mut child,
|
||||
format!("cannot hand the key to the agent: {}", crate::util::io_reason(&e)),
|
||||
);
|
||||
}
|
||||
match msg {
|
||||
Ok(m) if m == b"ok" => Ok(()),
|
||||
Ok(m) => fail(&mut child, format!("agent did not start: {}", String::from_utf8_lossy(&m).trim())),
|
||||
Err(_) => fail(&mut child, "agent did not start in time".into()),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_fd0(buf: &mut [u8]) -> std::io::Result<()> {
|
||||
let mut done = 0;
|
||||
while done < buf.len() {
|
||||
match rustix::io::read(rustix::stdio::stdin(), &mut buf[done..]) {
|
||||
Ok(0) => return Err(std::io::ErrorKind::UnexpectedEof.into()),
|
||||
Ok(n) => done += n,
|
||||
Err(rustix::io::Errno::INTR) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// The agent process. Returns when it expires or is told to quit.
|
||||
pub fn run(sock: &Path, ttl: &str, mode: &str) {
|
||||
let mut out = std::io::stdout();
|
||||
let report = |out: &mut std::io::Stdout, msg: &str| {
|
||||
let _ = out.write_all(msg.as_bytes());
|
||||
let _ = out.flush();
|
||||
// Closing standard output (by pointing it at /dev/null) tells
|
||||
// `unlock` the outcome is final.
|
||||
if let Ok(null) = rustix::fs::open("/dev/null", rustix::fs::OFlags::WRONLY, rustix::fs::Mode::empty()) {
|
||||
let _ = rustix::stdio::dup2_stdout(&null);
|
||||
}
|
||||
};
|
||||
let ttl = match crate::util::parse_duration(ttl) {
|
||||
Ok(n) if n > 0 && (mode == "fixed" || mode == "idle") => Duration::from_nanos(n as u64),
|
||||
_ => return report(&mut out, "bad duration or mode"),
|
||||
};
|
||||
if let Err(e) = harden_process() {
|
||||
return report(&mut out, &e.to_string());
|
||||
}
|
||||
let mut secret = match LockedBuf::new(REPLY) {
|
||||
Ok(b) => b,
|
||||
Err(e) => return report(&mut out, &e.to_string()),
|
||||
};
|
||||
// Read straight from fd 0 into the locked page: std's buffered
|
||||
// stdin would leave a copy in ordinary heap memory.
|
||||
if read_fd0(secret.as_mut_slice()).is_err() {
|
||||
return report(&mut out, "no key received");
|
||||
}
|
||||
{
|
||||
let s = secret.as_slice();
|
||||
if let Err(e) = Identity::from_suite_seed(u16::from_be_bytes([s[0], s[1]]), &s[2..]) {
|
||||
return report(&mut out, &e);
|
||||
}
|
||||
}
|
||||
// A socket left by an agent that died stops bind. Remove it only if
|
||||
// nothing answers on it, so a concurrent unlock's live agent is not
|
||||
// cut off from `vault lock`.
|
||||
if UnixStream::connect(sock).is_ok() {
|
||||
return report(&mut out, "another agent is already running for this key");
|
||||
}
|
||||
let _ = std::fs::remove_file(sock);
|
||||
let l = match UnixListener::bind(sock) {
|
||||
Ok(l) => l,
|
||||
Err(e) => return report(&mut out, &crate::util::io_msg("listen unix", sock, &e)),
|
||||
};
|
||||
let ours = match std::fs::symlink_metadata(sock) {
|
||||
Ok(m) => m,
|
||||
Err(e) => return report(&mut out, &crate::util::io_msg("stat", sock, &e)),
|
||||
};
|
||||
let cleanup = || {
|
||||
// Remove the socket only if it is still the one this agent made.
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
if let Ok(m) = std::fs::symlink_metadata(sock)
|
||||
&& m.ino() == ours.ino()
|
||||
&& m.dev() == ours.dev()
|
||||
{
|
||||
let _ = std::fs::remove_file(sock);
|
||||
}
|
||||
};
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
if let Err(e) = std::fs::set_permissions(sock, std::fs::Permissions::from_mode(0o600)).and_then(|_| l.set_nonblocking(true)) {
|
||||
cleanup();
|
||||
return report(&mut out, &crate::util::io_msg("chmod", sock, &e));
|
||||
}
|
||||
}
|
||||
report(&mut out, "ok");
|
||||
serve(&l, &mut secret, ttl, mode == "idle");
|
||||
secret.clear();
|
||||
drop(secret);
|
||||
cleanup();
|
||||
}
|
||||
|
||||
/// Serves until the deadline. The listener is polled with a timeout, so
|
||||
/// expiry never waits for another client; each request is bounded by
|
||||
/// two seconds and by the time left.
|
||||
fn serve(l: &UnixListener, secret: &mut LockedBuf, ttl: Duration, idle: bool) {
|
||||
use rustix::event::{PollFd, PollFlags, Timespec, poll};
|
||||
let mut deadline = std::time::Instant::now() + ttl;
|
||||
loop {
|
||||
let left = deadline.saturating_duration_since(std::time::Instant::now());
|
||||
if left.is_zero() {
|
||||
return;
|
||||
}
|
||||
let t = Timespec {
|
||||
tv_sec: left.as_secs() as i64,
|
||||
tv_nsec: left.subsec_nanos() as _,
|
||||
};
|
||||
let mut fds = [PollFd::new(l, PollFlags::IN)];
|
||||
match poll(&mut fds, Some(&t)) {
|
||||
Ok(0) => continue,
|
||||
Ok(_) => {}
|
||||
Err(rustix::io::Errno::INTR) => continue,
|
||||
Err(_) => return,
|
||||
}
|
||||
let Ok((c, _)) = l.accept() else { continue };
|
||||
let left = deadline.saturating_duration_since(std::time::Instant::now());
|
||||
if left.is_zero() {
|
||||
return;
|
||||
}
|
||||
let limit = left.min(TIMEOUT);
|
||||
if c.set_nonblocking(false)
|
||||
.and_then(|_| c.set_read_timeout(Some(limit)))
|
||||
.and_then(|_| c.set_write_timeout(Some(limit)))
|
||||
.is_err()
|
||||
{
|
||||
continue;
|
||||
}
|
||||
let mut op = [0u8; 1];
|
||||
if (&c).read_exact(&mut op).is_err() {
|
||||
continue;
|
||||
}
|
||||
match op[0] {
|
||||
GET => {
|
||||
if std::time::Instant::now() < deadline {
|
||||
let _ = (&c).write_all(secret.as_slice());
|
||||
if idle {
|
||||
deadline = std::time::Instant::now() + ttl;
|
||||
}
|
||||
}
|
||||
}
|
||||
QUIT => {
|
||||
secret.clear();
|
||||
drop(c);
|
||||
return;
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub use imp::{command, identity, run, start, stop};
|
||||
|
||||
#[cfg(not(unix))]
|
||||
mod imp {
|
||||
use super::*;
|
||||
const UNSUPPORTED: &str = "vault unlock is not supported on this platform; use the passphrase each time";
|
||||
|
||||
pub fn identity(_: &Path) -> Result<Identity> {
|
||||
Err(Error::new(UNSUPPORTED))
|
||||
}
|
||||
pub fn stop(_: &Path) -> bool {
|
||||
false
|
||||
}
|
||||
pub fn start(_: &Path, _: &Identity, _: Duration, _: bool) -> Result<()> {
|
||||
Err(Error::new(UNSUPPORTED))
|
||||
}
|
||||
pub fn run(_: &Path, _: &str, _: &str) {}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
pub use imp::{identity, run, start, stop};
|
||||
|
||||
#[cfg(all(test, unix))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn command_strips_secrets_from_the_environment() {
|
||||
let c = command(Path::new("/x/sock"), Duration::from_secs(300), true).unwrap();
|
||||
let removed: Vec<_> = c
|
||||
.get_envs()
|
||||
.filter(|(_, v)| v.is_none())
|
||||
.map(|(k, _)| k.to_string_lossy().into_owned())
|
||||
.collect();
|
||||
for v in SECRET_ENVS {
|
||||
assert!(removed.iter().any(|r| r == v), "{v} not removed");
|
||||
}
|
||||
let args: Vec<_> = c.get_args().map(|a| a.to_string_lossy().into_owned()).collect();
|
||||
assert_eq!(args, [AGENT_CMD, "/x/sock", "5m0s", "idle"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abs_is_lexical() {
|
||||
assert_eq!(go_abs(Path::new("/a/b/../c/./d")).unwrap(), PathBuf::from("/a/c/d"));
|
||||
}
|
||||
}
|
||||
+973
@@ -0,0 +1,973 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The command line, matching the Go version's commands, flags (Go `flag`
|
||||
//! package syntax, flags before, between or after arguments), prompts,
|
||||
//! messages and exit statuses.
|
||||
|
||||
use crate::agent;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::keyfile;
|
||||
use crate::pipeline::{MAX_WORKERS, Pool};
|
||||
use crate::platform::{fs as pfs, lock, term};
|
||||
use crate::secret::SecretVec;
|
||||
use crate::store::{Catalogue, FILE_LIMIT, SourceStamp, Store, VaultName};
|
||||
use crate::util::{self, human_size};
|
||||
use std::ffi::OsString;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
pub const USAGE: &str = "Usage:
|
||||
vault genkey create a new passphrase-protected key
|
||||
vault init [--filler SIZE] create a new, empty vault, with at least
|
||||
SIZE of filler (e.g. 500M, 2G)
|
||||
vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively)
|
||||
or a downloaded URL into the vault,
|
||||
optionally under a folder inside the vault
|
||||
vault migrate FILE|DIR [FOLDER] like add, then delete each original once
|
||||
[--yes] it is stored; asks before overwriting
|
||||
anything already in the vault, unless
|
||||
--yes is given
|
||||
vault mirror URL [FOLDER] crawl a website (same host only, one request
|
||||
at a time) into FOLDER/HOST, rewriting its
|
||||
links to relative paths
|
||||
vault absorb [FOLDER] read a tar archive (optionally gzipped) from
|
||||
standard input and add every file in it,
|
||||
optionally under a folder inside the vault
|
||||
vault ls [FOLDER] list one folder: its files, and its
|
||||
sub-folders (ending in /) with their
|
||||
total size; -r lists everything under it
|
||||
vault list [FOLDER] list every stored file under FOLDER,
|
||||
with full names
|
||||
vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name)
|
||||
vault get -r FOLDER [-o DIR] decrypt a folder and everything under it
|
||||
into DIR (default: the folder's base name)
|
||||
vault mv OLD NEW rename or move a file or folder; NEW
|
||||
ending in / or naming an existing folder
|
||||
moves OLD into it
|
||||
vault cp [-r] OLD NEW copy a file, or with -r a folder, like mv;
|
||||
copies share their encrypted data
|
||||
vault rm NAME delete a stored file
|
||||
vault rm -r FOLDER [-f] delete a folder and everything under it;
|
||||
asks first unless -f (or -rf) is given
|
||||
vault unlock [--for DURATION] remember the passphrase for DURATION
|
||||
[--idle] (default 5m), so commands skip asking;
|
||||
with --idle, DURATION since last used
|
||||
vault lock forget it now
|
||||
|
||||
Options (any command):
|
||||
--key PATH key file (default $VAULT_KEY, else ~/.vault/key)
|
||||
--vault DIR vault directory (default $VAULT_DIR, else ~/.vault/store)
|
||||
--workers N encryption and I/O threads, 1 to 256 (default: CPUs, at
|
||||
most 8)
|
||||
|
||||
Options for add URL and mirror:
|
||||
--max-bytes SIZE largest download (add) or total (mirror); defaults
|
||||
16G and 4G, at most 1T
|
||||
--timeout DURATION deadline for each request; defaults 2h and 5m
|
||||
--max-pages N pages and assets mirror fetches (default 10000)
|
||||
|
||||
The passphrase is read from the terminal, or from $VAULT_PASSPHRASE. After
|
||||
vault unlock, commands use the remembered key instead, unless
|
||||
$VAULT_PASSPHRASE is set.
|
||||
|
||||
Decoy passphrase: genkey offers a second, optional passphrase
|
||||
($VAULT_DECOY_PASSPHRASE when scripted). It unlocks a separate key, so
|
||||
every command run with it sees and changes only the decoy files, and the
|
||||
real files stay invisible. The key file does not reveal whether a decoy
|
||||
exists, and init fills each vault with unreadable filler, so hidden
|
||||
files look like filler to anyone holding only the decoy passphrase.";
|
||||
|
||||
const PASSPHRASE_ENV: &str = "VAULT_PASSPHRASE";
|
||||
const DECOY_ENV: &str = "VAULT_DECOY_PASSPHRASE";
|
||||
/// Exit status when `mirror` stored some resources but skipped others.
|
||||
pub const EXIT_PARTIAL: i32 = 3;
|
||||
|
||||
/// Prints to standard error, ignoring failure (as Go's Fprintf does).
|
||||
macro_rules! say {
|
||||
($($t:tt)*) => {{ let _ = writeln!(std::io::stderr(), $($t)*); }};
|
||||
}
|
||||
|
||||
/// Prints to standard output; a closed pipe ends the program quietly, as
|
||||
/// Go's SIGPIPE on standard output does.
|
||||
fn out(line: &str) {
|
||||
let mut o = std::io::stdout().lock();
|
||||
if let Err(e) = o.write_all(line.as_bytes()).and_then(|_| o.flush())
|
||||
&& e.kind() == std::io::ErrorKind::BrokenPipe
|
||||
{
|
||||
std::process::exit(128 + 13);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Flags {
|
||||
key: Option<String>,
|
||||
vault: Option<String>,
|
||||
output: Option<String>,
|
||||
filler: Option<String>,
|
||||
yes: bool,
|
||||
ttl: Option<i64>,
|
||||
idle: bool,
|
||||
r: bool,
|
||||
f: bool,
|
||||
rf: bool,
|
||||
fr: bool,
|
||||
workers: Option<String>,
|
||||
max_bytes: Option<String>,
|
||||
timeout: Option<i64>,
|
||||
max_pages: Option<String>,
|
||||
}
|
||||
|
||||
enum FlagKind {
|
||||
Str,
|
||||
Bool,
|
||||
Dur,
|
||||
}
|
||||
|
||||
fn flag_kind(name: &str) -> Option<FlagKind> {
|
||||
Some(match name {
|
||||
"key" | "vault" | "o" | "filler" | "workers" | "max-bytes" | "max-pages" => FlagKind::Str,
|
||||
"yes" | "idle" | "r" | "f" | "rf" | "fr" => FlagKind::Bool,
|
||||
"for" | "timeout" => FlagKind::Dur,
|
||||
_ => return None,
|
||||
})
|
||||
}
|
||||
|
||||
fn go_parse_bool(v: &str) -> Option<bool> {
|
||||
match v {
|
||||
"1" | "t" | "T" | "TRUE" | "true" | "True" => Some(true),
|
||||
"0" | "f" | "F" | "FALSE" | "false" | "False" => Some(false),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
impl Flags {
|
||||
fn set(&mut self, name: &str, value: &str) -> std::result::Result<(), String> {
|
||||
match flag_kind(name).expect("known flag") {
|
||||
FlagKind::Bool => {
|
||||
let b = go_parse_bool(value).ok_or_else(|| format!("invalid boolean value {value:?} for -{name}: parse error"))?;
|
||||
*match name {
|
||||
"yes" => &mut self.yes,
|
||||
"idle" => &mut self.idle,
|
||||
"r" => &mut self.r,
|
||||
"f" => &mut self.f,
|
||||
"rf" => &mut self.rf,
|
||||
_ => &mut self.fr,
|
||||
} = b;
|
||||
}
|
||||
FlagKind::Dur => {
|
||||
let d = util::parse_duration(value).map_err(|_| format!("invalid value {value:?} for flag -{name}: parse error"))?;
|
||||
*if name == "for" { &mut self.ttl } else { &mut self.timeout } = Some(d);
|
||||
}
|
||||
FlagKind::Str => {
|
||||
let v = Some(value.to_string());
|
||||
match name {
|
||||
"key" => self.key = v,
|
||||
"vault" => self.vault = v,
|
||||
"o" => self.output = v,
|
||||
"filler" => self.filler = v,
|
||||
"workers" => self.workers = v,
|
||||
"max-bytes" => self.max_bytes = v,
|
||||
_ => self.max_pages = v,
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `flag.FlagSet.Parse`: consumes flags from the front of `args`,
|
||||
/// returning the remaining arguments.
|
||||
fn parse_flags(fl: &mut Flags, mut args: &[String]) -> std::result::Result<Vec<String>, String> {
|
||||
loop {
|
||||
let Some(s) = args.first() else { return Ok(Vec::new()) };
|
||||
let b = s.as_bytes();
|
||||
if b.len() < 2 || b[0] != b'-' {
|
||||
return Ok(args.to_vec());
|
||||
}
|
||||
let mut minuses = 1;
|
||||
if b[1] == b'-' {
|
||||
minuses = 2;
|
||||
if b.len() == 2 {
|
||||
return Ok(args[1..].to_vec());
|
||||
}
|
||||
}
|
||||
let mut name = &s[minuses..];
|
||||
if name.is_empty() || name.starts_with('-') || name.starts_with('=') {
|
||||
return Err(format!("bad flag syntax: {s}"));
|
||||
}
|
||||
args = &args[1..];
|
||||
let mut value = None;
|
||||
if let Some(i) = name[1..].find('=') {
|
||||
value = Some(&name[i + 2..]);
|
||||
name = &name[..i + 1];
|
||||
}
|
||||
let Some(kind) = flag_kind(name) else {
|
||||
if name == "help" || name == "h" {
|
||||
return Err("flag: help requested".into());
|
||||
}
|
||||
return Err(format!("flag provided but not defined: -{name}"));
|
||||
};
|
||||
match (kind, value) {
|
||||
(FlagKind::Bool, None) => fl.set(name, "true")?,
|
||||
(_, Some(v)) => fl.set(name, v)?,
|
||||
(_, None) => {
|
||||
let Some(v) = args.first() else {
|
||||
return Err(format!("flag needs an argument: -{name}"));
|
||||
};
|
||||
fl.set(name, v)?;
|
||||
args = &args[1..];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Flags may come before, between or after positional arguments.
|
||||
fn parse_interleaved(fl: &mut Flags, args: &[String]) -> std::result::Result<Vec<String>, String> {
|
||||
let mut pos = Vec::new();
|
||||
let mut rest = args.to_vec();
|
||||
loop {
|
||||
let left = parse_flags(fl, &rest)?;
|
||||
let Some(first) = left.first() else { return Ok(pos) };
|
||||
pos.push(first.clone());
|
||||
rest = left[1..].to_vec();
|
||||
}
|
||||
}
|
||||
|
||||
fn env_bytes(name: &str) -> Option<Vec<u8>> {
|
||||
let v = std::env::var_os(name)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
Some(std::os::unix::ffi::OsStringExt::into_vec(v))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
Some(v.to_string_lossy().into_owned().into_bytes())
|
||||
}
|
||||
}
|
||||
|
||||
fn read_passphrase(prompt: &str) -> Result<SecretVec> {
|
||||
match env_bytes(PASSPHRASE_ENV) {
|
||||
Some(v) if v.is_empty() => Err(Error::new(format!("{PASSPHRASE_ENV} is set but empty"))),
|
||||
Some(v) => Ok(SecretVec::from_vec(v)),
|
||||
None => term::read_secret(prompt).map_err(|e| {
|
||||
Error::new(format!(
|
||||
"could not read passphrase (set {PASSPHRASE_ENV} for non-interactive use): {}",
|
||||
util::io_reason(&e)
|
||||
))
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn new_passphrase(what: &str) -> Result<SecretVec> {
|
||||
if let Some(v) = env_bytes(PASSPHRASE_ENV).filter(|v| !v.is_empty()) {
|
||||
return Ok(SecretVec::from_vec(v));
|
||||
}
|
||||
let p = read_passphrase(&format!("Enter {what}:"))?;
|
||||
if p.is_empty() {
|
||||
crate::bail!("{what} must not be empty");
|
||||
}
|
||||
let again = read_passphrase(&format!("Confirm {what}:"))?;
|
||||
if !(p == *again.expose()) {
|
||||
crate::bail!("{what}s do not match");
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
/// An optional second passphrase; empty means none.
|
||||
fn decoy_passphrase() -> Result<SecretVec> {
|
||||
if let Some(v) = env_bytes(DECOY_ENV) {
|
||||
return Ok(SecretVec::from_vec(v));
|
||||
}
|
||||
if std::env::var_os(PASSPHRASE_ENV).is_some() {
|
||||
return Ok(SecretVec::from_vec(Vec::new())); // scripted, no decoy
|
||||
}
|
||||
let read = |p: &str| term::read_secret(p).map_err(|e| Error::new(util::io_reason(&e)));
|
||||
let p = read("Enter decoy passphrase (empty for none):")?;
|
||||
if p.is_empty() {
|
||||
return Ok(p);
|
||||
}
|
||||
let again = read("Confirm decoy passphrase:")?;
|
||||
if !(p == *again.expose()) {
|
||||
crate::bail!("decoy passphrases do not match");
|
||||
}
|
||||
Ok(p)
|
||||
}
|
||||
|
||||
fn home() -> Option<PathBuf> {
|
||||
let v = if cfg!(windows) {
|
||||
std::env::var_os("USERPROFILE")
|
||||
} else {
|
||||
std::env::var_os("HOME")
|
||||
};
|
||||
v.filter(|h| !h.is_empty()).map(PathBuf::from)
|
||||
}
|
||||
|
||||
fn default_path(flag: &Option<String>, env: &str, name: &str) -> Result<PathBuf> {
|
||||
if let Some(f) = flag {
|
||||
return Ok(PathBuf::from(f));
|
||||
}
|
||||
if let Some(v) = std::env::var_os(env).filter(|v| !v.is_empty()) {
|
||||
return Ok(PathBuf::from(v));
|
||||
}
|
||||
home()
|
||||
.map(|h| h.join(".vault").join(name))
|
||||
.ok_or_else(|| Error::new(format!("cannot find home directory; set {env}")))
|
||||
}
|
||||
|
||||
/// The key: from a running agent unless a passphrase is in the
|
||||
/// environment, else from the key file with the passphrase.
|
||||
fn key_identity(key: &Path) -> Result<crate::crypto::kem::Identity> {
|
||||
if std::env::var_os(PASSPHRASE_ENV).is_none()
|
||||
&& let Ok(id) = agent::identity(key)
|
||||
{
|
||||
return Ok(id);
|
||||
}
|
||||
let pass = read_passphrase("Enter passphrase:")?;
|
||||
keyfile::unlock(key, pass.expose()).map_err(|e| Error::new(e.to_string()))
|
||||
}
|
||||
|
||||
struct Ctx {
|
||||
flags: Flags,
|
||||
pool: Option<Arc<Pool>>,
|
||||
}
|
||||
|
||||
impl Ctx {
|
||||
fn key(&self) -> Result<PathBuf> {
|
||||
default_path(&self.flags.key, "VAULT_KEY", "key")
|
||||
}
|
||||
|
||||
fn vault_dir(&self) -> Result<PathBuf> {
|
||||
default_path(&self.flags.vault, "VAULT_DIR", "store")
|
||||
}
|
||||
|
||||
fn pool(&mut self) -> Result<Arc<Pool>> {
|
||||
if self.pool.is_none() {
|
||||
let n = match &self.flags.workers {
|
||||
None => Pool::default_workers(),
|
||||
Some(w) => match w.parse::<usize>() {
|
||||
Ok(n) if (1..=MAX_WORKERS).contains(&n) => n,
|
||||
_ => crate::bail!("--workers needs a number from 1 to {MAX_WORKERS}"),
|
||||
},
|
||||
};
|
||||
self.pool = Some(Arc::new(Pool::new(n)));
|
||||
}
|
||||
Ok(Arc::clone(self.pool.as_ref().expect("set")))
|
||||
}
|
||||
|
||||
/// Unlocks the key, opens the vault, then takes the vault lock:
|
||||
/// exclusive for writers, shared for readers.
|
||||
fn open(&mut self, exclusive: bool) -> Result<(Store, lock::Lock)> {
|
||||
let key = self.key()?;
|
||||
let dir = self.vault_dir()?;
|
||||
let id = key_identity(&key)?;
|
||||
let s = Store::open(&dir, id, self.pool()?)?;
|
||||
let l = lock::lock_vault(&dir, exclusive)?;
|
||||
Ok((s, l))
|
||||
}
|
||||
|
||||
fn net_limits(&self, mirror: bool) -> Result<crate::net::Limits> {
|
||||
let mut l = if mirror {
|
||||
crate::net::Limits::mirror_defaults()
|
||||
} else {
|
||||
crate::net::Limits::add_defaults()
|
||||
};
|
||||
if let Some(b) = &self.flags.max_bytes {
|
||||
l.max_bytes = util::parse_size(b, FILE_LIMIT)?;
|
||||
}
|
||||
if let Some(t) = self.flags.timeout {
|
||||
if t <= 0 {
|
||||
crate::bail!("--timeout needs a positive duration, such as 30s or 1h");
|
||||
}
|
||||
l.request = Duration::from_nanos(t as u64);
|
||||
}
|
||||
if let Some(p) = &self.flags.max_pages {
|
||||
l.max_pages = p
|
||||
.parse()
|
||||
.ok()
|
||||
.filter(|&n| n > 0)
|
||||
.ok_or_else(|| Error::new("--max-pages needs a positive number"))?;
|
||||
}
|
||||
Ok(l)
|
||||
}
|
||||
}
|
||||
|
||||
fn report(verb: &str) -> impl FnMut(&VaultName, bool) + '_ {
|
||||
move |n, replaced| say!("{} {n}", if replaced { "replaced" } else { verb })
|
||||
}
|
||||
|
||||
/// Runs the command line; returns the exit status.
|
||||
pub fn main() -> i32 {
|
||||
let args: Vec<OsString> = std::env::args_os().collect();
|
||||
if args.len() < 2 {
|
||||
say!("{USAGE}");
|
||||
return 2;
|
||||
}
|
||||
if args[1] == agent::AGENT_CMD && args.len() == 5 {
|
||||
agent::run(Path::new(&args[2]), &args[3].to_string_lossy(), &args[4].to_string_lossy());
|
||||
return 0;
|
||||
}
|
||||
let mut strs = Vec::with_capacity(args.len());
|
||||
for a in &args[1..] {
|
||||
match a.to_str() {
|
||||
Some(s) => strs.push(s.to_string()),
|
||||
None => {
|
||||
say!("vault: arguments must be valid UTF-8: {}", a.to_string_lossy());
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
match run(&strs[0], &strs[1..]) {
|
||||
Ok(code) => code,
|
||||
Err(e) => {
|
||||
say!("vault: {e}");
|
||||
1
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn run(cmd: &str, args: &[String]) -> Result<i32> {
|
||||
let mut flags = Flags::default();
|
||||
let pos = parse_interleaved(&mut flags, args).map_err(|e| Error::new(format!("{e}\n\n{USAGE}")))?;
|
||||
let want = |min: usize, max: usize| -> Result<()> {
|
||||
if pos.len() < min || pos.len() > max {
|
||||
crate::bail!("wrong number of arguments for {cmd}\n\n{USAGE}");
|
||||
}
|
||||
Ok(())
|
||||
};
|
||||
let mut ctx = Ctx { flags, pool: None };
|
||||
match cmd {
|
||||
"genkey" => {
|
||||
want(0, 0)?;
|
||||
let key = ctx.key()?;
|
||||
if std::fs::metadata(&key).is_ok() {
|
||||
crate::bail!("{} already exists", key.display());
|
||||
}
|
||||
let pass = new_passphrase("passphrase")?;
|
||||
let decoy = decoy_passphrase()?;
|
||||
if pass == *decoy.expose() {
|
||||
crate::bail!("decoy passphrase must differ from the passphrase");
|
||||
}
|
||||
keyfile::write(&key, pass.expose(), decoy.expose()).map_err(|e| Error::new(e.to_string()))?;
|
||||
// One left from an old key at this path would serve it.
|
||||
agent::stop(&key);
|
||||
say!("key written to {}", key.display());
|
||||
}
|
||||
"init" => {
|
||||
want(0, 0)?;
|
||||
let size = util::parse_size(ctx.flags.filler.as_deref().unwrap_or("0"), FILE_LIMIT)?;
|
||||
let dir = ctx.vault_dir()?;
|
||||
Store::init(&dir, size, ctx.pool()?)?;
|
||||
say!("vault created at {}", dir.display());
|
||||
}
|
||||
"add" | "migrate" => {
|
||||
want(1, 2)?;
|
||||
let dest = pos.get(1).map(|d| d.trim_matches('/').to_string()).unwrap_or_default();
|
||||
let migrate = cmd == "migrate";
|
||||
if crate::net::is_url(&pos[0]) {
|
||||
if migrate {
|
||||
crate::bail!("cannot migrate a URL; use add");
|
||||
}
|
||||
let limits = ctx.net_limits(false)?;
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
crate::net::add_url(&s, &mut cat, &pos[0], &dest, &limits, &mut report("added"))?;
|
||||
return Ok(0);
|
||||
}
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
// Never ingest, or delete, the vault, anything inside it, or its key.
|
||||
let vault_dir = ctx.vault_dir()?;
|
||||
if let (Ok(v), Ok(src)) = (std::fs::canonicalize(&vault_dir), std::fs::canonicalize(&pos[0]))
|
||||
&& src != v
|
||||
&& src.starts_with(&v)
|
||||
{
|
||||
crate::bail!("{} is inside the vault", pos[0]);
|
||||
}
|
||||
let skip: Vec<FileId> = [vault_dir, ctx.key()?].iter().filter_map(|p| FileId::of(p, true)).collect();
|
||||
let yes = ctx.flags.yes;
|
||||
add_path(&s, &mut cat, Path::new(&pos[0]), &dest, migrate, &skip, &mut |names| {
|
||||
yes || confirm_overwrite(names)
|
||||
})?;
|
||||
}
|
||||
"mirror" => {
|
||||
want(1, 2)?;
|
||||
if !crate::net::is_url(&pos[0]) {
|
||||
crate::bail!("mirror needs an http:// or https:// URL");
|
||||
}
|
||||
let dest = pos.get(1).map(|d| d.trim_matches('/').to_string()).unwrap_or_default();
|
||||
let limits = ctx.net_limits(true)?;
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
match crate::net::mirror(&s, &mut cat, &pos[0], &dest, &limits, &mut report("added"))? {
|
||||
crate::net::MirrorOutcome::Complete { .. } => {}
|
||||
crate::net::MirrorOutcome::Partial { stored, skipped } => {
|
||||
say!("vault: mirror incomplete: stored {stored}, skipped {skipped}");
|
||||
return Ok(EXIT_PARTIAL);
|
||||
}
|
||||
}
|
||||
}
|
||||
"absorb" => {
|
||||
want(0, 1)?;
|
||||
if stdin_is_terminal() {
|
||||
crate::bail!("absorb reads a tar archive from standard input; pipe one in");
|
||||
}
|
||||
let dest = pos.first().map(|d| d.trim_matches('/').to_string()).unwrap_or_default();
|
||||
// Wait for the archive to start arriving before asking for the
|
||||
// passphrase, so a program feeding the pipe (gpg, ssh) has
|
||||
// finished with the terminal and cannot bury the prompt.
|
||||
let mut input = std::io::BufReader::with_capacity(1 << 16, std::io::stdin().lock());
|
||||
if std::io::BufRead::fill_buf(&mut input).map_or(true, |b| b.is_empty()) {
|
||||
crate::bail!("nothing to absorb on standard input");
|
||||
}
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
crate::absorb::absorb(&s, &mut cat, input, &dest, &mut report("added"))?;
|
||||
}
|
||||
"list" | "ls" => {
|
||||
want(0, 1)?;
|
||||
let prefix = pos.first().map(|d| d.trim_matches('/').to_string()).unwrap_or_default();
|
||||
let (s, _lock) = ctx.open(false)?;
|
||||
let cat = s.catalogue()?;
|
||||
if cmd == "ls" && !ctx.flags.r {
|
||||
for line in ls(&cat, &prefix)? {
|
||||
out(&line);
|
||||
}
|
||||
} else {
|
||||
let names: Vec<String> = if prefix.is_empty() {
|
||||
cat.names().map(String::from).collect()
|
||||
} else {
|
||||
cat.under(&prefix)
|
||||
};
|
||||
for n in names {
|
||||
let r = cat.current(&n).expect("listed");
|
||||
out(&format!("{:>10} {} {n}\n", human_size(r.size), r.modified.local_minute()));
|
||||
}
|
||||
}
|
||||
}
|
||||
"get" => {
|
||||
want(1, 1)?;
|
||||
let out_path = ctx.flags.output.clone().unwrap_or_else(|| util::base(pos[0].trim_end_matches('/')));
|
||||
let (s, _lock) = ctx.open(false)?;
|
||||
let cat = s.catalogue()?;
|
||||
if !ctx.flags.r {
|
||||
let r = cat
|
||||
.current(&pos[0])
|
||||
.ok_or_else(|| Error::new(format!("{}: not in the vault", pos[0])))?;
|
||||
let p = PathBuf::from(&out_path);
|
||||
let Some(fname) = p.file_name() else {
|
||||
crate::bail!("{out_path} already exists");
|
||||
};
|
||||
let parent = p.parent().unwrap_or(Path::new(""));
|
||||
let dir = pfs::Dir::open(parent).map_err(|e| Error::io("open", parent, &e))?;
|
||||
s.extract(r, &dir, fname)?;
|
||||
return Ok(0);
|
||||
}
|
||||
let folder = pos[0].trim_matches('/');
|
||||
let names = cat.under(folder);
|
||||
if folder.is_empty() || names.is_empty() || cat.contains(folder) {
|
||||
crate::bail!("{}: not a folder in the vault", pos[0]);
|
||||
}
|
||||
let root = PathBuf::from(&out_path);
|
||||
pfs::create_private_dir_all(&root).map_err(|e| Error::io("mkdir", &root, &e))?;
|
||||
let top = pfs::Dir::open(&root).map_err(|e| Error::io("open", &root, &e))?;
|
||||
for n in names {
|
||||
let rel = &n[folder.len() + 1..];
|
||||
let parts: Vec<&str> = rel.split('/').collect();
|
||||
let mut dir = None::<pfs::Dir>;
|
||||
for p in &parts[..parts.len() - 1] {
|
||||
let next = dir
|
||||
.as_ref()
|
||||
.unwrap_or(&top)
|
||||
.subdir(p.as_ref())
|
||||
.map_err(|e| Error::new(util::io_reason(&e)))?;
|
||||
dir = Some(next);
|
||||
}
|
||||
let d = dir.as_ref().unwrap_or(&top);
|
||||
s.extract(cat.current(&n).expect("listed"), d, parts[parts.len() - 1].as_ref())?;
|
||||
say!("extracted {}", root.join(rel.replace('/', std::path::MAIN_SEPARATOR_STR)).display());
|
||||
}
|
||||
}
|
||||
"mv" | "rename" | "cp" => {
|
||||
want(2, 2)?;
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
let pairs = transfers(&cat, &pos[0], &pos[1], cmd != "cp" || ctx.flags.r)?;
|
||||
let verb = if cmd == "cp" { "copied" } else { "moved" };
|
||||
s.copy_all(&mut cat, &pairs, cmd != "cp", &mut |from, to| say!("{verb} {from} -> {to}"))?;
|
||||
}
|
||||
"rm" | "delete" => {
|
||||
want(1, 1)?;
|
||||
let f = &ctx.flags;
|
||||
let (recursive, forced) = (f.r || f.rf || f.fr, f.f || f.rf || f.fr);
|
||||
let mut name = pos[0].clone();
|
||||
if recursive {
|
||||
name = name.trim_matches('/').to_string();
|
||||
if name.is_empty() {
|
||||
crate::bail!("refusing to remove the whole vault");
|
||||
}
|
||||
}
|
||||
let (s, _lock) = ctx.open(true)?;
|
||||
let mut cat = s.catalogue()?;
|
||||
let names = if recursive {
|
||||
cat.under(&name)
|
||||
} else if cat.contains(&name) {
|
||||
vec![name.clone()]
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
if names.is_empty() {
|
||||
if forced {
|
||||
return Ok(0);
|
||||
}
|
||||
crate::bail!("{name}: not in the vault");
|
||||
}
|
||||
let size: u64 = names.iter().map(|n| cat.current(n).expect("listed").size).sum();
|
||||
if recursive
|
||||
&& !forced
|
||||
&& !confirm(
|
||||
"these files will be deleted, with no undo:",
|
||||
&names,
|
||||
&format!("delete {} file(s), {}? [y/N]", names.len(), human_size(size)),
|
||||
)
|
||||
{
|
||||
crate::bail!("rm cancelled; nothing changed (-f skips the question)");
|
||||
}
|
||||
s.remove_all(&mut cat, &names, &mut |n| say!("removed {n}"))?;
|
||||
if recursive {
|
||||
say!("removed {} file(s), {}", names.len(), human_size(size));
|
||||
}
|
||||
}
|
||||
"unlock" => {
|
||||
want(0, 0)?;
|
||||
let ttl = ctx.flags.ttl.unwrap_or(5 * 60 * 1_000_000_000);
|
||||
if ttl <= 0 {
|
||||
crate::bail!("--for needs a positive duration, such as 5m or 1h");
|
||||
}
|
||||
let ttl = Duration::from_nanos(ttl as u64);
|
||||
let key = ctx.key()?;
|
||||
let pass = read_passphrase("Enter passphrase:")?;
|
||||
let id = keyfile::unlock(&key, pass.expose()).map_err(|e| Error::new(e.to_string()))?;
|
||||
drop(pass);
|
||||
agent::start(&key, &id, ttl, ctx.flags.idle)?;
|
||||
id.close();
|
||||
if ctx.flags.idle {
|
||||
say!(
|
||||
"unlocked until unused for {}; vault lock forgets the key sooner",
|
||||
util::format_duration(ttl)
|
||||
);
|
||||
} else {
|
||||
let until = util::Instant::from_system(std::time::SystemTime::now() + ttl);
|
||||
say!("unlocked until {}; vault lock forgets the key sooner", until.local_clock());
|
||||
}
|
||||
}
|
||||
"lock" => {
|
||||
want(0, 0)?;
|
||||
if agent::stop(&ctx.key()?) {
|
||||
say!("locked");
|
||||
} else {
|
||||
say!("nothing was unlocked");
|
||||
}
|
||||
}
|
||||
"help" | "-h" | "--help" => out(&format!("{USAGE}\n")),
|
||||
_ => crate::bail!("unknown command {cmd:?}\n\n{USAGE}"),
|
||||
}
|
||||
Ok(0)
|
||||
}
|
||||
|
||||
fn stdin_is_terminal() -> bool {
|
||||
use std::io::IsTerminal;
|
||||
std::io::stdin().is_terminal()
|
||||
}
|
||||
|
||||
/// Works out, like mv(1) and cp(1), what `src` and `dst` mean: each file
|
||||
/// to copy or move, with its new name. `src` is a file, or with
|
||||
/// `recursive` a folder; `dst` ending in / or naming an existing folder
|
||||
/// means into it. No new name may already exist.
|
||||
pub fn transfers(cat: &Catalogue, src: &str, dst: &str, recursive: bool) -> Result<Vec<(String, VaultName)>> {
|
||||
let mut into = dst.is_empty() || dst.ends_with('/');
|
||||
let (src, dst) = (src.trim_matches('/'), dst.trim_matches('/'));
|
||||
if src.is_empty() {
|
||||
crate::bail!("name a file or folder to copy or move");
|
||||
}
|
||||
if dst.is_empty() || (!cat.contains(dst) && !cat.under(dst).is_empty()) {
|
||||
into = true;
|
||||
}
|
||||
let dst = if into {
|
||||
util::join(&[dst, &util::base(src)])
|
||||
} else {
|
||||
dst.to_string()
|
||||
};
|
||||
let raw: Vec<(String, String)> = if cat.contains(src) {
|
||||
vec![(src.to_string(), dst)]
|
||||
} else {
|
||||
let names = cat.under(src);
|
||||
if names.is_empty() {
|
||||
crate::bail!("{src}: not in the vault");
|
||||
}
|
||||
if !recursive {
|
||||
crate::bail!("{src} is a folder; use -r");
|
||||
}
|
||||
if dst == src || dst.starts_with(&format!("{src}/")) {
|
||||
crate::bail!("cannot put {src} inside itself");
|
||||
}
|
||||
names
|
||||
.into_iter()
|
||||
.map(|n| (n.clone(), format!("{dst}{}", &n[src.len()..])))
|
||||
.collect()
|
||||
};
|
||||
let mut pairs = Vec::with_capacity(raw.len());
|
||||
for (from, to) in raw {
|
||||
let Some(name) = VaultName::new(&to) else {
|
||||
crate::bail!("invalid vault path {to:?}")
|
||||
};
|
||||
if cat.contains(&to) {
|
||||
crate::bail!("{to} already exists");
|
||||
}
|
||||
pairs.push((from, name));
|
||||
}
|
||||
Ok(pairs)
|
||||
}
|
||||
|
||||
/// One level of the vault, like ls(1): the files directly in `folder`
|
||||
/// and each sub-folder once, with a trailing slash, the total size and
|
||||
/// the newest date beneath it. Folders first. A file's own name lists it.
|
||||
pub fn ls(cat: &Catalogue, folder: &str) -> Result<Vec<String>> {
|
||||
let mut entries: std::collections::BTreeMap<String, (u64, Option<util::Instant>)> = Default::default();
|
||||
for (name, r) in cat.iter() {
|
||||
let rel = if folder.is_empty() {
|
||||
name.to_string()
|
||||
} else if name == folder {
|
||||
util::base(name)
|
||||
} else if let Some(rest) = name.strip_prefix(folder).and_then(|r| r.strip_prefix('/')) {
|
||||
rest.to_string()
|
||||
} else {
|
||||
continue;
|
||||
};
|
||||
let rel = match rel.find('/') {
|
||||
Some(i) => rel[..i + 1].to_string(),
|
||||
None => rel,
|
||||
};
|
||||
let e = entries.entry(rel).or_default();
|
||||
e.0 += r.size;
|
||||
if e.1.is_none_or(|m| r.modified > m) {
|
||||
e.1 = Some(r.modified);
|
||||
}
|
||||
}
|
||||
if entries.is_empty() && !folder.is_empty() {
|
||||
crate::bail!("{folder}: not in the vault");
|
||||
}
|
||||
let mut names: Vec<_> = entries.into_iter().collect();
|
||||
names.sort_by(|a, b| b.0.ends_with('/').cmp(&a.0.ends_with('/')).then(a.0.cmp(&b.0)));
|
||||
Ok(names
|
||||
.into_iter()
|
||||
.map(|(n, (size, m))| {
|
||||
format!(
|
||||
"{:>10} {} {n}\n",
|
||||
human_size(size),
|
||||
m.unwrap_or(util::Instant { secs: 0, nanos: 0 }).local_minute()
|
||||
)
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn confirm_overwrite(names: &[String]) -> bool {
|
||||
confirm(
|
||||
"these files are already in the vault and would be overwritten:",
|
||||
names,
|
||||
&format!("overwrite {} file(s)? [y/N]", names.len()),
|
||||
)
|
||||
}
|
||||
|
||||
/// Shows `intro` and the first 20 names, then asks on the terminal. No
|
||||
/// terminal means no.
|
||||
fn confirm(intro: &str, names: &[String], question: &str) -> bool {
|
||||
say!("{intro}");
|
||||
for (i, n) in names.iter().enumerate() {
|
||||
if i == 20 {
|
||||
say!(" … and {} more", names.len() - i);
|
||||
break;
|
||||
}
|
||||
say!(" {n}");
|
||||
}
|
||||
match term::confirm(question) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
say!("cannot ask for confirmation: {}", util::io_reason(&e));
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Identifies a file for Go's `os.SameFile`.
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct FileId(u64, u64, PathBuf);
|
||||
|
||||
impl FileId {
|
||||
fn of(p: &Path, follow: bool) -> Option<FileId> {
|
||||
let m = if follow {
|
||||
std::fs::metadata(p)
|
||||
} else {
|
||||
std::fs::symlink_metadata(p)
|
||||
}
|
||||
.ok()?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
let _ = &m;
|
||||
Some(FileId(m.dev(), m.ino(), PathBuf::new()))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = m;
|
||||
Some(FileId(0, 0, std::fs::canonicalize(p).ok()?))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn utf8_rel(p: &Path) -> Result<String> {
|
||||
let mut parts = Vec::new();
|
||||
for c in p.components() {
|
||||
let s = c
|
||||
.as_os_str()
|
||||
.to_str()
|
||||
.ok_or_else(|| Error::new(format!("invalid vault path {:?}", p.to_string_lossy())))?;
|
||||
parts.push(s.to_string());
|
||||
}
|
||||
Ok(parts.join("/"))
|
||||
}
|
||||
|
||||
/// Adds a file, or every regular file under a directory, beneath `dest`,
|
||||
/// keeping the structure. A directory named "." or ".." adds its contents
|
||||
/// directly. With `migrate`, nothing is stored unless `overwrite` approves
|
||||
/// the names it would replace, each original is deleted once stored and
|
||||
/// checked unchanged, then emptied directories go.
|
||||
fn add_path(
|
||||
s: &Store,
|
||||
cat: &mut Catalogue,
|
||||
src: &Path,
|
||||
dest: &str,
|
||||
migrate: bool,
|
||||
skip: &[FileId],
|
||||
overwrite: &mut dyn FnMut(&[String]) -> bool,
|
||||
) -> Result<()> {
|
||||
let info = std::fs::metadata(src).map_err(|e| Error::io("stat", src, &e))?;
|
||||
let skipped = |p: &Path, follow: bool| FileId::of(p, follow).is_some_and(|id| skip.contains(&id));
|
||||
if skipped(src, true) {
|
||||
crate::bail!("{} is the vault or its key", src.display());
|
||||
}
|
||||
let mut items: Vec<(PathBuf, String)> = Vec::new();
|
||||
let mut dirs = Vec::new();
|
||||
if !info.is_dir() {
|
||||
let base = src.file_name().map(|b| utf8_rel(Path::new(b))).transpose()?.unwrap_or_default();
|
||||
items.push((src.to_path_buf(), util::join(&[dest, &base])));
|
||||
} else {
|
||||
let base = match src.components().next_back() {
|
||||
Some(std::path::Component::Normal(b)) => utf8_rel(Path::new(b))?,
|
||||
_ => String::new(),
|
||||
};
|
||||
walk(src, src, &mut |p, rel, ft| {
|
||||
if skipped(p, false) {
|
||||
say!("skipping {}: the vault or its key", p.display());
|
||||
return Ok(false);
|
||||
}
|
||||
if ft.is_dir() {
|
||||
dirs.push(p.to_path_buf());
|
||||
return Ok(true);
|
||||
}
|
||||
if !ft.is_file() {
|
||||
say!("skipping {}: not a regular file", p.display());
|
||||
return Ok(false);
|
||||
}
|
||||
items.push((p.to_path_buf(), util::join(&[dest, &base, &utf8_rel(rel)?])));
|
||||
Ok(false)
|
||||
})?;
|
||||
}
|
||||
if migrate {
|
||||
let existing: Vec<String> = items.iter().filter(|(_, n)| cat.contains(n)).map(|(_, n)| n.clone()).collect();
|
||||
if !existing.is_empty() && !overwrite(&existing) {
|
||||
crate::bail!("migrate cancelled; nothing changed");
|
||||
}
|
||||
}
|
||||
for (p, n) in &items {
|
||||
let name = VaultName::new(n).ok_or_else(|| Error::new(format!("invalid vault path {n:?}")))?;
|
||||
let (added, stamp) = s.add_file(cat, p, &name)?;
|
||||
let mut verb = if added.replaced { "replaced" } else { "added" };
|
||||
if migrate {
|
||||
remove_source(p, stamp)?;
|
||||
verb = "migrated";
|
||||
}
|
||||
say!("{verb} {n}");
|
||||
}
|
||||
if migrate {
|
||||
// Children come after their parents in walk order, so go backwards.
|
||||
// Directories still holding skipped files stay.
|
||||
for d in dirs.iter().rev() {
|
||||
let _ = std::fs::remove_dir(d);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Deletes a migrated original, but only if it is still the very file
|
||||
/// that was stored (same inode, size and modification time).
|
||||
fn remove_source(p: &Path, stamp: SourceStamp) -> Result<()> {
|
||||
match std::fs::metadata(p) {
|
||||
Ok(m) if SourceStamp::of(&m) == stamp => std::fs::remove_file(p).map_err(|e| Error::io("remove", p, &e)),
|
||||
_ => crate::bail!("{} changed after it was stored; not deleting it", p.display()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `filepath.WalkDir`: lexical order, the root first, symlinks not
|
||||
/// followed. `visit` returns whether to descend into a directory.
|
||||
fn walk(root: &Path, p: &Path, visit: &mut dyn FnMut(&Path, &Path, &std::fs::FileType) -> Result<bool>) -> Result<()> {
|
||||
let ft = std::fs::symlink_metadata(p).map_err(|e| Error::io("lstat", p, &e))?.file_type();
|
||||
let rel = p.strip_prefix(root).unwrap_or(Path::new(""));
|
||||
if !visit(p, rel, &ft)? || !ft.is_dir() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut entries: Vec<_> = std::fs::read_dir(p)
|
||||
.map_err(|e| Error::io("open", p, &e))?
|
||||
.map(|e| e.map(|e| e.file_name()))
|
||||
.collect::<std::io::Result<_>>()
|
||||
.map_err(|e| Error::io("readdir", p, &e))?;
|
||||
entries.sort();
|
||||
for e in entries {
|
||||
walk(root, &p.join(e), visit)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn s(v: &[&str]) -> Vec<String> {
|
||||
v.iter().map(|x| x.to_string()).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_flag_syntax() {
|
||||
let mut f = Flags::default();
|
||||
let pos = parse_interleaved(&mut f, &s(&["a", "-o", "out", "--yes", "b", "-for=1h", "-r=false"])).unwrap();
|
||||
assert_eq!(pos, ["a", "b"]);
|
||||
assert_eq!(f.output.as_deref(), Some("out"));
|
||||
assert!(f.yes && !f.r);
|
||||
assert_eq!(f.ttl, Some(3_600_000_000_000));
|
||||
let mut f = Flags::default();
|
||||
assert_eq!(parse_interleaved(&mut f, &s(&["--", "-x", "y"])).unwrap(), ["-x", "y"]);
|
||||
assert_eq!(parse_interleaved(&mut f, &s(&["-", "y"])).unwrap(), ["-", "y"]);
|
||||
for (args, err) in [
|
||||
(vec!["-x"], "flag provided but not defined: -x"),
|
||||
(vec!["-o"], "flag needs an argument: -o"),
|
||||
(vec!["---o"], "bad flag syntax: ---o"),
|
||||
(vec!["-=x"], "bad flag syntax: -=x"),
|
||||
(vec!["-for", "soon"], "invalid value \"soon\" for flag -for: parse error"),
|
||||
(vec!["-yes=maybe"], "invalid boolean value \"maybe\" for -yes: parse error"),
|
||||
(vec!["-h"], "flag: help requested"),
|
||||
] {
|
||||
assert_eq!(parse_interleaved(&mut Flags::default(), &s(&args)).err().as_deref(), Some(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,497 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! FrodoKEM-640-SHAKE exactly as CIRCL v1.6.4 implements it
|
||||
//! (`kem/frodo/frodo640shake`): the round-3 FrodoKEM with a 48-byte key
|
||||
//! seed, 16-byte encapsulation seed, 16-byte shared secret and 9720-byte
|
||||
//! ciphertext. The RustCrypto `frodo-kem` crate implements the later ISO
|
||||
//! variant (salted, 9752-byte ciphertexts, a different seed layout), which
|
||||
//! cannot open existing vaults, so this module is a line-by-line port of
|
||||
//! CIRCL's code, checked against vectors from the Go oracle and against the
|
||||
//! NIST KAT digest CIRCL itself tests.
|
||||
//!
|
||||
//! All arithmetic is mod 2^16 with masking to 15 bits where CIRCL masks;
|
||||
//! loops never branch on secret data, and the decapsulation re-encryption
|
||||
//! check selects the rejection secret with a constant-time mask.
|
||||
|
||||
use sha3::Shake128;
|
||||
use sha3::digest::{ExtendableOutput, Update, XofReader};
|
||||
use zeroize::Zeroize;
|
||||
|
||||
const N: usize = 640;
|
||||
const NBAR: usize = 8;
|
||||
const LOG_Q: u32 = 15;
|
||||
const Q_MASK: u16 = (1 << LOG_Q) - 1;
|
||||
const SEED_A: usize = 16;
|
||||
const PK_HASH: usize = 16;
|
||||
const EXTRACTED_BITS: u32 = 2;
|
||||
const MESSAGE: usize = 16;
|
||||
const BP_PACKED: usize = (LOG_Q as usize * N * NBAR) / 8;
|
||||
|
||||
pub const SHARED_KEY_SIZE: usize = 16;
|
||||
pub const KEY_SEED_SIZE: usize = SHARED_KEY_SIZE + SHARED_KEY_SIZE + 16;
|
||||
pub const ENCAPSULATION_SEED_SIZE: usize = 16;
|
||||
pub const CIPHERTEXT_SIZE: usize = 9720;
|
||||
pub const PUBLIC_KEY_SIZE: usize = 9616;
|
||||
pub const PRIVATE_KEY_SIZE: usize = 19888;
|
||||
|
||||
const CDF: [u16; 13] = [
|
||||
4643, 13363, 20579, 25843, 29227, 31145, 32103, 32525, 32689, 32745, 32762, 32766, 32767,
|
||||
];
|
||||
|
||||
/// A packed public key (`seedA || pack(B)`), as CIRCL's `MarshalBinary`.
|
||||
pub struct PublicKey {
|
||||
seed_a: [u8; SEED_A],
|
||||
b: Vec<u16>, // N x NBAR
|
||||
}
|
||||
|
||||
/// A private key. Its secret parts are cleared on drop.
|
||||
pub struct PrivateKey {
|
||||
s_reject: [u8; SHARED_KEY_SIZE],
|
||||
pk: PublicKey,
|
||||
s: Vec<u16>, // transpose(S), NBAR x N
|
||||
hpk: [u8; PK_HASH],
|
||||
}
|
||||
|
||||
impl Drop for PrivateKey {
|
||||
fn drop(&mut self) {
|
||||
self.s_reject.zeroize();
|
||||
self.s.zeroize();
|
||||
}
|
||||
}
|
||||
|
||||
fn shake(parts: &[&[u8]], out: &mut [u8]) {
|
||||
let mut h = Shake128::default();
|
||||
for p in parts {
|
||||
h.update(p);
|
||||
}
|
||||
h.finalize_xof().read(out);
|
||||
}
|
||||
|
||||
fn le_u16s(bytes: &[u8], out: &mut [u16]) {
|
||||
for (o, c) in out.iter_mut().zip(bytes.as_chunks::<2>().0) {
|
||||
*o = u16::from_le_bytes([c[0], c[1]]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Inverse-transform sampling from the FrodoKEM error distribution.
|
||||
fn sample(v: &mut [u16]) {
|
||||
for x in v.iter_mut() {
|
||||
let sign = *x & 1;
|
||||
let unif = *x >> 1;
|
||||
let mut g: u16 = 0;
|
||||
for c in &CDF[..CDF.len() - 1] {
|
||||
g = g.wrapping_add(c.wrapping_sub(unif) >> 15);
|
||||
}
|
||||
*x = (sign.wrapping_neg() ^ g).wrapping_add(sign);
|
||||
}
|
||||
}
|
||||
|
||||
/// Row i of A is SHAKE128(le16(i) || seedA), as little-endian u16s.
|
||||
fn expand_a(seed_a: &[u8; SEED_A]) -> Vec<u16> {
|
||||
let mut a = vec![0u16; N * N];
|
||||
let mut input = [0u8; 2 + SEED_A];
|
||||
input[2..].copy_from_slice(seed_a);
|
||||
let mut row = [0u8; N * 2];
|
||||
for i in 0..N {
|
||||
input[0] = i as u8;
|
||||
input[1] = (i >> 8) as u8;
|
||||
shake(&[&input], &mut row);
|
||||
le_u16s(&row, &mut a[i * N..(i + 1) * N]);
|
||||
}
|
||||
a
|
||||
}
|
||||
|
||||
/// out (N x NBAR) += A*S + E, with S stored transposed (NBAR x N).
|
||||
fn mul_add_as_plus_e(out: &mut [u16], a: &[u16], s: &[u16], e: &[u16]) {
|
||||
for i in 0..N {
|
||||
let row = &a[i * N..(i + 1) * N];
|
||||
for k in 0..NBAR {
|
||||
let sk = &s[k * N..(k + 1) * N];
|
||||
let mut sum = e[i * NBAR + k];
|
||||
for j in 0..N {
|
||||
sum = sum.wrapping_add(row[j].wrapping_mul(sk[j]));
|
||||
}
|
||||
out[i * NBAR + k] = out[i * NBAR + k].wrapping_add(sum);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// out (NBAR x N) += S*A + E. CIRCL walks A by column; summing row by row
|
||||
/// gives the same result mod 2^16 with far better locality.
|
||||
fn mul_add_sa_plus_e(out: &mut [u16], s: &[u16], a: &[u16], e: &[u16]) {
|
||||
let mut acc = e[..NBAR * N].to_vec();
|
||||
for j in 0..N {
|
||||
let row = &a[j * N..(j + 1) * N];
|
||||
for k in 0..NBAR {
|
||||
let skj = s[k * N + j];
|
||||
let dst = &mut acc[k * N..(k + 1) * N];
|
||||
for i in 0..N {
|
||||
dst[i] = dst[i].wrapping_add(row[i].wrapping_mul(skj));
|
||||
}
|
||||
}
|
||||
}
|
||||
for (o, v) in out.iter_mut().zip(&acc) {
|
||||
*o = o.wrapping_add(*v);
|
||||
}
|
||||
acc.zeroize();
|
||||
}
|
||||
|
||||
/// out (NBAR x NBAR) = S*B + E, masked.
|
||||
fn mul_add_sb_plus_e(out: &mut [u16], s: &[u16], b: &[u16], e: &[u16]) {
|
||||
for k in 0..NBAR {
|
||||
for i in 0..NBAR {
|
||||
let mut v = e[k * NBAR + i];
|
||||
for j in 0..N {
|
||||
v = v.wrapping_add(s[k * N + j].wrapping_mul(b[j * NBAR + i]));
|
||||
}
|
||||
out[k * NBAR + i] = v & Q_MASK;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// out (NBAR x NBAR) = B'*S, with S transposed, masked.
|
||||
fn mul_bs(out: &mut [u16], b: &[u16], s: &[u16]) {
|
||||
for i in 0..NBAR {
|
||||
for j in 0..NBAR {
|
||||
let mut v: u16 = 0;
|
||||
for k in 0..N {
|
||||
v = v.wrapping_add(b[i * N + k].wrapping_mul(s[j * N + k]));
|
||||
}
|
||||
out[i * NBAR + j] = v & Q_MASK;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn pack(out: &mut [u8], v: &[u16]) {
|
||||
for (o, x) in out.as_chunks_mut::<15>().0.iter_mut().zip(v.as_chunks::<8>().0) {
|
||||
let m: [u16; 8] = core::array::from_fn(|i| x[i] & Q_MASK);
|
||||
o[0] = (m[0] >> 7) as u8;
|
||||
o[1] = (((m[0] & 0x7f) as u8) << 1) | (m[1] >> 14) as u8;
|
||||
o[2] = (m[1] >> 6) as u8;
|
||||
o[3] = (((m[1] & 0x3f) as u8) << 2) | (m[2] >> 13) as u8;
|
||||
o[4] = (m[2] >> 5) as u8;
|
||||
o[5] = (((m[2] & 0x1f) as u8) << 3) | (m[3] >> 12) as u8;
|
||||
o[6] = (m[3] >> 4) as u8;
|
||||
o[7] = (((m[3] & 0x0f) as u8) << 4) | (m[4] >> 11) as u8;
|
||||
o[8] = (m[4] >> 3) as u8;
|
||||
o[9] = (((m[4] & 0x07) as u8) << 5) | (m[5] >> 10) as u8;
|
||||
o[10] = (m[5] >> 2) as u8;
|
||||
o[11] = (((m[5] & 0x03) as u8) << 6) | (m[6] >> 9) as u8;
|
||||
o[12] = (m[6] >> 1) as u8;
|
||||
o[13] = (((m[6] & 0x01) as u8) << 7) | (m[7] >> 8) as u8;
|
||||
o[14] = m[7] as u8;
|
||||
}
|
||||
}
|
||||
|
||||
fn unpack(out: &mut [u16], b: &[u8]) {
|
||||
for (o, i) in out.as_chunks_mut::<8>().0.iter_mut().zip(b.as_chunks::<15>().0) {
|
||||
let w = |x: u8| x as u16;
|
||||
o[0] = (w(i[0]) << 7) | (w(i[1] & 0xfe) >> 1);
|
||||
o[1] = (w(i[1] & 0x01) << 14) | (w(i[2]) << 6) | (w(i[3] & 0xfc) >> 2);
|
||||
o[2] = (w(i[3] & 0x03) << 13) | (w(i[4]) << 5) | (w(i[5] & 0xf8) >> 3);
|
||||
o[3] = (w(i[5] & 0x07) << 12) | (w(i[6]) << 4) | (w(i[7] & 0xf0) >> 4);
|
||||
o[4] = (w(i[7] & 0x0f) << 11) | (w(i[8]) << 3) | (w(i[9] & 0xe0) >> 5);
|
||||
o[5] = (w(i[9] & 0x1f) << 10) | (w(i[10]) << 2) | (w(i[11] & 0xc0) >> 6);
|
||||
o[6] = (w(i[11] & 0x3f) << 9) | (w(i[12]) << 1) | (w(i[13] & 0x80) >> 7);
|
||||
o[7] = (w(i[13] & 0x7f) << 8) | w(i[14]);
|
||||
}
|
||||
}
|
||||
|
||||
fn encode_message(out: &mut [u16], msg: &[u8; MESSAGE]) {
|
||||
let mask = (1u16 << EXTRACTED_BITS) - 1;
|
||||
let mut pos = 0;
|
||||
for c in msg.as_chunks::<2>().0 {
|
||||
let mut v = u16::from_le_bytes([c[0], c[1]]);
|
||||
for _ in 0..(16 / EXTRACTED_BITS) {
|
||||
out[pos] = (v & mask) << (LOG_Q - EXTRACTED_BITS);
|
||||
pos += 1;
|
||||
v >>= EXTRACTED_BITS;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_message(out: &mut [u8; MESSAGE], m: &[u16]) {
|
||||
let mask = (1u16 << EXTRACTED_BITS) - 1;
|
||||
let mut pos = 0;
|
||||
for o in out.iter_mut() {
|
||||
for j in 0..(8 / EXTRACTED_BITS) {
|
||||
let mut t = (m[pos] & Q_MASK) + (1 << (LOG_Q - EXTRACTED_BITS - 1));
|
||||
t >>= LOG_Q - EXTRACTED_BITS;
|
||||
t &= mask;
|
||||
*o |= (t as u8) << (j * EXTRACTED_BITS);
|
||||
pos += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// 0 if equal, 1 otherwise, without branching on the contents.
|
||||
fn ct_differs(a: &[u16], b: &[u16]) -> u8 {
|
||||
let mut v: u16 = 0;
|
||||
for (x, y) in a.iter().zip(b) {
|
||||
v |= x ^ y;
|
||||
}
|
||||
((v | v.wrapping_neg()) >> 15) as u8
|
||||
}
|
||||
|
||||
impl PublicKey {
|
||||
pub fn to_bytes(&self) -> Vec<u8> {
|
||||
let mut out = vec![0u8; PUBLIC_KEY_SIZE];
|
||||
out[..SEED_A].copy_from_slice(&self.seed_a);
|
||||
pack(&mut out[SEED_A..], &self.b);
|
||||
out
|
||||
}
|
||||
|
||||
/// Any 9616 bytes form a key, as in CIRCL; only the length is checked.
|
||||
pub fn from_bytes(buf: &[u8]) -> Option<Self> {
|
||||
if buf.len() != PUBLIC_KEY_SIZE {
|
||||
return None;
|
||||
}
|
||||
let mut seed_a = [0u8; SEED_A];
|
||||
seed_a.copy_from_slice(&buf[..SEED_A]);
|
||||
let mut b = vec![0u16; N * NBAR];
|
||||
unpack(&mut b, &buf[SEED_A..]);
|
||||
Some(PublicKey { seed_a, b })
|
||||
}
|
||||
|
||||
fn hash(&self) -> [u8; PK_HASH] {
|
||||
let mut h = [0u8; PK_HASH];
|
||||
shake(&[&self.to_bytes()], &mut h);
|
||||
h
|
||||
}
|
||||
|
||||
/// Encapsulates with the 16-byte message `mu` (CIRCL's
|
||||
/// `EncapsulateTo` seed), returning the ciphertext and shared secret.
|
||||
/// Callers must pass fresh randomness; reusing `mu` repeats the secret.
|
||||
pub fn encapsulate(&self, mu: &[u8; ENCAPSULATION_SEED_SIZE]) -> (Vec<u8>, [u8; SHARED_KEY_SIZE]) {
|
||||
let hpk = self.hash();
|
||||
let mut g2 = [0u8; 2 * SHARED_KEY_SIZE];
|
||||
shake(&[&hpk, mu], &mut g2);
|
||||
let (sp, ep, epp, mut noise) = noise(&g2[..SHARED_KEY_SIZE]);
|
||||
let a = expand_a(&self.seed_a);
|
||||
let mut bp = vec![0u16; NBAR * N];
|
||||
mul_add_sa_plus_e(&mut bp, &noise[sp.clone()], &a, &noise[ep]);
|
||||
let mut v = [0u16; NBAR * NBAR];
|
||||
mul_add_sb_plus_e(&mut v, &noise[sp], &self.b, &noise[epp]);
|
||||
let mut c = [0u16; NBAR * NBAR];
|
||||
encode_message(&mut c, mu);
|
||||
for (c, v) in c.iter_mut().zip(&v) {
|
||||
*c = c.wrapping_add(*v) & Q_MASK;
|
||||
}
|
||||
let mut ct = vec![0u8; CIPHERTEXT_SIZE];
|
||||
pack(&mut ct[..BP_PACKED], &bp);
|
||||
pack(&mut ct[BP_PACKED..], &c);
|
||||
let mut ss = [0u8; SHARED_KEY_SIZE];
|
||||
shake(&[&ct, &g2[SHARED_KEY_SIZE..]], &mut ss);
|
||||
noise.zeroize();
|
||||
g2.zeroize();
|
||||
v.zeroize();
|
||||
(ct, ss)
|
||||
}
|
||||
}
|
||||
|
||||
type Span = core::ops::Range<usize>;
|
||||
|
||||
/// SHAKE128(0x96 || seedSE) expanded and sampled into S', E', E''.
|
||||
fn noise(seed_se: &[u8]) -> (Span, Span, Span, Vec<u16>) {
|
||||
let total = 2 * N * NBAR + NBAR * NBAR;
|
||||
let mut bytes = vec![0u8; 2 * total];
|
||||
shake(&[&[0x96], seed_se], &mut bytes);
|
||||
let mut v = vec![0u16; total];
|
||||
le_u16s(&bytes, &mut v);
|
||||
bytes.zeroize();
|
||||
sample(&mut v);
|
||||
(0..N * NBAR, N * NBAR..2 * N * NBAR, 2 * N * NBAR..total, v)
|
||||
}
|
||||
|
||||
impl PrivateKey {
|
||||
/// CIRCL's `newKeyFromSeed`: seed is s || seedSE || z.
|
||||
pub fn derive(seed: &[u8; KEY_SEED_SIZE]) -> Self {
|
||||
let mut seed_a = [0u8; SEED_A];
|
||||
shake(&[&seed[2 * SHARED_KEY_SIZE..]], &mut seed_a);
|
||||
let mut bytes = vec![0u8; 2 * 2 * N * NBAR];
|
||||
shake(&[&[0x5f], &seed[SHARED_KEY_SIZE..2 * SHARED_KEY_SIZE]], &mut bytes);
|
||||
let mut s = vec![0u16; N * NBAR];
|
||||
let mut e = vec![0u16; N * NBAR];
|
||||
le_u16s(&bytes[..2 * N * NBAR], &mut s);
|
||||
le_u16s(&bytes[2 * N * NBAR..], &mut e);
|
||||
bytes.zeroize();
|
||||
sample(&mut s);
|
||||
sample(&mut e);
|
||||
let a = expand_a(&seed_a);
|
||||
let mut b = vec![0u16; N * NBAR];
|
||||
mul_add_as_plus_e(&mut b, &a, &s, &e);
|
||||
e.zeroize();
|
||||
let pk = PublicKey { seed_a, b };
|
||||
let hpk = pk.hash();
|
||||
let mut s_reject = [0u8; SHARED_KEY_SIZE];
|
||||
s_reject.copy_from_slice(&seed[..SHARED_KEY_SIZE]);
|
||||
PrivateKey { s_reject, pk, s, hpk }
|
||||
}
|
||||
|
||||
pub fn public(&self) -> &PublicKey {
|
||||
&self.pk
|
||||
}
|
||||
|
||||
/// CIRCL's packed form: s || pk || S (little-endian) || H(pk).
|
||||
pub fn to_bytes(&self) -> zeroize::Zeroizing<Vec<u8>> {
|
||||
let mut out = zeroize::Zeroizing::new(Vec::with_capacity(PRIVATE_KEY_SIZE));
|
||||
out.extend_from_slice(&self.s_reject);
|
||||
out.extend_from_slice(&self.pk.to_bytes());
|
||||
for x in &self.s {
|
||||
out.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
out.extend_from_slice(&self.hpk);
|
||||
out
|
||||
}
|
||||
|
||||
/// Decapsulates with implicit rejection: a ciphertext that does not
|
||||
/// re-encrypt identically yields SHAKE(ct || s) instead of an error.
|
||||
pub fn decapsulate(&self, ct: &[u8]) -> Option<[u8; SHARED_KEY_SIZE]> {
|
||||
if ct.len() != CIPHERTEXT_SIZE {
|
||||
return None;
|
||||
}
|
||||
let mut bp = vec![0u16; NBAR * N];
|
||||
let mut c = [0u16; NBAR * NBAR];
|
||||
unpack(&mut bp, &ct[..BP_PACKED]);
|
||||
unpack(&mut c, &ct[BP_PACKED..]);
|
||||
let mut w = [0u16; NBAR * NBAR];
|
||||
mul_bs(&mut w, &bp, &self.s);
|
||||
for (w, c) in w.iter_mut().zip(&c) {
|
||||
*w = c.wrapping_sub(*w) & Q_MASK;
|
||||
}
|
||||
let mut mu = [0u8; MESSAGE];
|
||||
decode_message(&mut mu, &w);
|
||||
|
||||
let mut g2 = [0u8; 2 * SHARED_KEY_SIZE];
|
||||
shake(&[&self.hpk, &mu], &mut g2);
|
||||
let (sp, ep, epp, mut noise) = noise(&g2[..SHARED_KEY_SIZE]);
|
||||
let a = expand_a(&self.pk.seed_a);
|
||||
let mut bbp = vec![0u16; NBAR * N];
|
||||
mul_add_sa_plus_e(&mut bbp, &noise[sp.clone()], &a, &noise[ep]);
|
||||
for x in bbp.iter_mut() {
|
||||
*x &= Q_MASK;
|
||||
}
|
||||
mul_add_sb_plus_e(&mut w, &noise[sp], &self.pk.b, &noise[epp]);
|
||||
let mut cc = [0u16; NBAR * NBAR];
|
||||
encode_message(&mut cc, &mu);
|
||||
for (cc, w) in cc.iter_mut().zip(&w) {
|
||||
*cc = cc.wrapping_add(*w) & Q_MASK;
|
||||
}
|
||||
let differs = ct_differs(&bp, &bbp) | ct_differs(&c, &cc);
|
||||
let mask = differs.wrapping_neg(); // 0xff to reject, 0 to accept
|
||||
let mut k = [0u8; SHARED_KEY_SIZE];
|
||||
for (i, k) in k.iter_mut().enumerate() {
|
||||
*k = (g2[SHARED_KEY_SIZE + i] & !mask) | (self.s_reject[i] & mask);
|
||||
}
|
||||
let mut ss = [0u8; SHARED_KEY_SIZE];
|
||||
shake(&[ct, &k], &mut ss);
|
||||
for z in [&mut mu[..], &mut g2[..], &mut k[..]] {
|
||||
z.zeroize();
|
||||
}
|
||||
noise.zeroize();
|
||||
w.zeroize();
|
||||
cc.zeroize();
|
||||
bbp.zeroize();
|
||||
Some(ss)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use aes::cipher::{BlockCipherEncrypt, KeyInit};
|
||||
use sha2::Digest;
|
||||
|
||||
/// NIST's AES-256 CTR_DRBG from the KAT generator's randombytes.c.
|
||||
struct Drbg {
|
||||
key: [u8; 32],
|
||||
v: [u8; 16],
|
||||
}
|
||||
|
||||
impl Drbg {
|
||||
fn new(entropy: &[u8; 48]) -> Self {
|
||||
let mut d = Drbg { key: [0; 32], v: [0; 16] };
|
||||
d.update(Some(entropy));
|
||||
d
|
||||
}
|
||||
|
||||
fn block(&mut self) -> [u8; 16] {
|
||||
for b in self.v.iter_mut().rev() {
|
||||
*b = b.wrapping_add(1);
|
||||
if *b != 0 {
|
||||
break;
|
||||
}
|
||||
}
|
||||
let c = aes::Aes256::new(&self.key.into());
|
||||
let mut blk = self.v.into();
|
||||
c.encrypt_block(&mut blk);
|
||||
blk.into()
|
||||
}
|
||||
|
||||
fn update(&mut self, data: Option<&[u8; 48]>) {
|
||||
let mut t = [0u8; 48];
|
||||
for i in 0..3 {
|
||||
t[16 * i..16 * i + 16].copy_from_slice(&self.block());
|
||||
}
|
||||
if let Some(d) = data {
|
||||
for (t, d) in t.iter_mut().zip(d) {
|
||||
*t ^= d;
|
||||
}
|
||||
}
|
||||
self.key.copy_from_slice(&t[..32]);
|
||||
self.v.copy_from_slice(&t[32..]);
|
||||
}
|
||||
|
||||
fn fill(&mut self, out: &mut [u8]) {
|
||||
for c in out.chunks_mut(16) {
|
||||
let b = self.block();
|
||||
c.copy_from_slice(&b[..c.len()]);
|
||||
}
|
||||
self.update(None);
|
||||
}
|
||||
}
|
||||
|
||||
fn upper(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02X}")).collect()
|
||||
}
|
||||
|
||||
/// CIRCL's TestPQCgenKATKem: the SHA-256 of the KAT response file it
|
||||
/// regenerates, which matches the FrodoKEM reference implementation's
|
||||
/// PQCkemKAT_19888_shake.rsp.
|
||||
#[test]
|
||||
fn nist_kat_digest() {
|
||||
let mut seed = [0u8; 48];
|
||||
for (i, s) in seed.iter_mut().enumerate() {
|
||||
*s = i as u8;
|
||||
}
|
||||
let mut g = Drbg::new(&seed);
|
||||
let mut f = sha2::Sha256::new();
|
||||
Digest::update(&mut f, b"# FrodoKEM-640-SHAKE\n\n");
|
||||
for i in 0..100 {
|
||||
g.fill(&mut seed);
|
||||
Digest::update(&mut f, format!("count = {i}\nseed = {}\n", upper(&seed)).as_bytes());
|
||||
let mut g2 = Drbg::new(&seed);
|
||||
let mut kseed = [0u8; KEY_SEED_SIZE];
|
||||
g2.fill(&mut kseed);
|
||||
let sk = PrivateKey::derive(&kseed);
|
||||
let pk = sk.public().to_bytes();
|
||||
let mut eseed = [0u8; ENCAPSULATION_SEED_SIZE];
|
||||
g2.fill(&mut eseed);
|
||||
let (ct, ss) = PublicKey::from_bytes(&pk).unwrap().encapsulate(&eseed);
|
||||
assert_eq!(sk.decapsulate(&ct).unwrap(), ss);
|
||||
Digest::update(
|
||||
&mut f,
|
||||
format!(
|
||||
"pk = {}\nsk = {}\nct = {}\nss = {}\n\n",
|
||||
upper(&pk),
|
||||
upper(&sk.to_bytes()),
|
||||
upper(&ct),
|
||||
upper(&ss)
|
||||
)
|
||||
.as_bytes(),
|
||||
);
|
||||
}
|
||||
let digest: String = f.finalize().iter().map(|x| format!("{x:02x}")).collect();
|
||||
assert_eq!(digest, "604a10cfc871dfaed9cb5b057c644ab03b16852cea7f39bc7f9831513b5b1cfa");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Whole-file Fury encryption: a header wrapping a fresh 32-byte file key,
|
||||
//! a 16-byte nonce, then the STREAM payload.
|
||||
//!
|
||||
//! - header MAC: HMAC-SHA-256 keyed by HKDF-SHA-256(file key, salt none,
|
||||
//! info "header"), over the header up to and including "---";
|
||||
//! - payload key: HKDF-SHA-256(file key, salt = nonce, info "payload").
|
||||
|
||||
use super::header::{self, Header, ParseError};
|
||||
use super::kem::{FILE_KEY_SIZE, Identity, Recipient, UnwrapError};
|
||||
use super::stream::{self, StreamReader, StreamWriter};
|
||||
use crate::rng::Rng;
|
||||
use crate::secret::{SecretArray, SecretVec};
|
||||
use hkdf::Hkdf;
|
||||
use hmac::{KeyInit, Mac};
|
||||
use sha2::Sha256;
|
||||
use std::io::{self, Read, Write};
|
||||
|
||||
pub const NONCE_SIZE: usize = 16;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum Error {
|
||||
/// No stanza opens with this identity: the file is someone else's.
|
||||
NoIdentityMatch,
|
||||
Header(ParseError),
|
||||
/// A stanza of our suite was malformed.
|
||||
Stanza(String),
|
||||
BadMac,
|
||||
Io(io::Error),
|
||||
TooLarge,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Error::NoIdentityMatch => f.write_str("no identity matched any of the recipients"),
|
||||
Error::Header(e) => write!(f, "failed to read header: {e}"),
|
||||
Error::Stanza(m) => f.write_str(m),
|
||||
Error::BadMac => f.write_str("bad header MAC"),
|
||||
Error::Io(e) => write!(f, "{e}"),
|
||||
Error::TooLarge => f.write_str("encrypted payload is larger than expected"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl From<io::Error> for Error {
|
||||
fn from(e: io::Error) -> Self {
|
||||
Error::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
fn header_mac(file_key: &[u8; FILE_KEY_SIZE], h: &Header) -> hmac::Hmac<Sha256> {
|
||||
let mut k = SecretArray::<32>::zero();
|
||||
Hkdf::<Sha256>::new(None, file_key)
|
||||
.expand(b"header", k.expose_mut())
|
||||
.expect("32 bytes");
|
||||
let mut m = <hmac::Hmac<Sha256> as KeyInit>::new_from_slice(k.expose()).expect("any key length");
|
||||
m.update(&h.marshal_without_mac());
|
||||
m
|
||||
}
|
||||
|
||||
pub fn payload_key(file_key: &[u8; FILE_KEY_SIZE], nonce: &[u8; NONCE_SIZE]) -> SecretArray<32> {
|
||||
let mut k = SecretArray::<32>::zero();
|
||||
Hkdf::<Sha256>::new(Some(nonce), file_key)
|
||||
.expand(b"payload", k.expose_mut())
|
||||
.expect("32 bytes");
|
||||
k
|
||||
}
|
||||
|
||||
/// Builds a header for `recipient` and returns it with the payload key and
|
||||
/// nonce to use. All randomness is fresh from `rng`.
|
||||
fn begin(recipient: &Recipient, rng: &mut Rng) -> (Header, [u8; NONCE_SIZE], SecretArray<32>) {
|
||||
let mut file_key = SecretArray::<FILE_KEY_SIZE>::zero();
|
||||
rng.fill(file_key.expose_mut());
|
||||
let mut h = Header {
|
||||
recipients: vec![recipient.wrap(file_key.expose(), rng)],
|
||||
mac: [0; 32],
|
||||
};
|
||||
h.mac = header_mac(file_key.expose(), &h).finalize().into_bytes().into();
|
||||
let nonce: [u8; NONCE_SIZE] = rng.array();
|
||||
let key = payload_key(file_key.expose(), &nonce);
|
||||
(h, nonce, key)
|
||||
}
|
||||
|
||||
/// Encrypts a plaintext held in memory, appending the file to `out`.
|
||||
pub fn encrypt_to(out: &mut Vec<u8>, recipient: &Recipient, plaintext: &[u8], rng: &mut Rng) {
|
||||
let (h, nonce, key) = begin(recipient, rng);
|
||||
out.extend_from_slice(&h.marshal());
|
||||
out.extend_from_slice(&nonce);
|
||||
stream::seal_all(key.expose(), plaintext, out);
|
||||
}
|
||||
|
||||
/// Streaming encryption to `dst`; call `finish` on the result.
|
||||
pub fn encrypt_writer<W: Write>(mut dst: W, recipient: &Recipient, rng: &mut Rng) -> io::Result<StreamWriter<W>> {
|
||||
let (h, nonce, key) = begin(recipient, rng);
|
||||
h.write_to(&mut dst)?;
|
||||
dst.write_all(&nonce)?;
|
||||
Ok(StreamWriter::new(key.expose(), dst))
|
||||
}
|
||||
|
||||
/// Go's `decryptHdr`: unwrap with `id`, then check the header MAC in
|
||||
/// constant time.
|
||||
fn file_key(h: &Header, id: &Identity) -> Result<SecretArray<FILE_KEY_SIZE>, Error> {
|
||||
let fk = match id.unwrap(&h.recipients) {
|
||||
Ok(k) => k,
|
||||
Err(UnwrapError::NotMine) => return Err(Error::NoIdentityMatch),
|
||||
Err(UnwrapError::Malformed(m)) => return Err(Error::Stanza(m)),
|
||||
};
|
||||
header_mac(fk.expose(), h).verify_slice(&h.mac).map_err(|_| Error::BadMac)?;
|
||||
Ok(fk)
|
||||
}
|
||||
|
||||
/// Streaming decryption. Nothing read from the result is authentic until
|
||||
/// it has returned end of input without error.
|
||||
pub fn decrypt_reader<R: Read>(src: R, id: &Identity) -> Result<StreamReader<impl Read>, Error> {
|
||||
let (h, mut payload) = header::parse(src).map_err(Error::Header)?;
|
||||
let fk = file_key(&h, id)?;
|
||||
let mut nonce = [0u8; NONCE_SIZE];
|
||||
payload
|
||||
.read_exact(&mut nonce)
|
||||
.map_err(|e| Error::Io(io::Error::new(e.kind(), format!("failed to read nonce: {e}"))))?;
|
||||
let key = payload_key(fk.expose(), &nonce);
|
||||
Ok(StreamReader::new(key.expose(), payload))
|
||||
}
|
||||
|
||||
/// Decrypts a whole file whose plaintext is at most `max_plaintext` bytes,
|
||||
/// reading no more than that much ciphertext. The result is fully
|
||||
/// authenticated, final chunk and end of input included.
|
||||
pub fn decrypt_bounded<R: Read>(src: R, id: &Identity, max_plaintext: usize) -> Result<SecretVec, Error> {
|
||||
let mut buf = SecretVec::with_capacity(max_ciphertext(max_plaintext) + 1);
|
||||
decrypt_bounded_into(src, id, max_plaintext, &mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// The largest STREAM payload holding at most `max_plaintext` bytes.
|
||||
pub fn max_ciphertext(max_plaintext: usize) -> usize {
|
||||
max_plaintext + max_plaintext.div_ceil(stream::CHUNK_SIZE).max(1) * stream::TAG_SIZE
|
||||
}
|
||||
|
||||
/// As [`decrypt_bounded`], into a caller-supplied buffer whose capacity
|
||||
/// must be at least `max_ciphertext(max_plaintext) + 1`. On error the
|
||||
/// buffer is left empty.
|
||||
pub fn decrypt_bounded_into<R: Read>(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> {
|
||||
buf.clear();
|
||||
let r = decrypt_into(src, id, max_plaintext, buf);
|
||||
if r.is_err() {
|
||||
buf.clear();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn decrypt_into<R: Read>(src: R, id: &Identity, max_plaintext: usize, buf: &mut SecretVec) -> Result<(), Error> {
|
||||
let (h, mut payload) = header::parse(src).map_err(Error::Header)?;
|
||||
let fk = file_key(&h, id)?;
|
||||
let mut nonce = [0u8; NONCE_SIZE];
|
||||
payload
|
||||
.read_exact(&mut nonce)
|
||||
.map_err(|e| Error::Io(io::Error::new(e.kind(), format!("failed to read nonce: {e}"))))?;
|
||||
let key = payload_key(fk.expose(), &nonce);
|
||||
let max_ct = max_ciphertext(max_plaintext);
|
||||
buf.resize_within_capacity(max_ct + 1);
|
||||
let n = stream::read_full(&mut payload, buf.expose_mut())?;
|
||||
if n > max_ct {
|
||||
return Err(Error::TooLarge);
|
||||
}
|
||||
let plain = stream::open_all(key.expose(), &mut buf.expose_mut()[..n]).map_err(|e| Error::Io(e.into()))?;
|
||||
buf.resize_within_capacity(plain);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::crypto::kem::{
|
||||
SUITE_ID,
|
||||
tests::{unhex, vectors},
|
||||
};
|
||||
use crate::crypto::stream::tests::hex;
|
||||
|
||||
fn pattern(n: usize, salt: u8) -> Vec<u8> {
|
||||
(0..n).map(|i| (i.wrapping_mul(7) as u8) ^ salt).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_vectors_match_go() {
|
||||
for v in vectors().wrap {
|
||||
let raw = unhex(&v.header_hex);
|
||||
let (h, _) = header::parse(&raw[..]).unwrap();
|
||||
assert_eq!(h.marshal(), raw, "canonical serialisation");
|
||||
let fk: [u8; 32] = unhex(&v.file_key).try_into().unwrap();
|
||||
assert_eq!(hex(&header_mac(&fk, &h).finalize().into_bytes()), v.mac);
|
||||
let id = Identity::from_suite_seed(SUITE_ID, &unhex(&v.master_seed)).unwrap();
|
||||
assert_eq!(file_key(&h, &id).unwrap().expose(), &fk);
|
||||
let nonce: [u8; 16] = unhex(&v.nonce).try_into().unwrap();
|
||||
assert_eq!(hex(payload_key(&fk, &nonce).expose()), v.payload_key);
|
||||
// A one-bit change anywhere the MAC covers is caught.
|
||||
let mut tampered = h.clone();
|
||||
tampered.recipients[0].body[0] ^= 1;
|
||||
assert!(file_key(&tampered, &id).is_err());
|
||||
let mut wrong_mac = h.clone();
|
||||
wrong_mac.mac[31] ^= 1;
|
||||
assert!(matches!(file_key(&wrong_mac, &id), Err(Error::BadMac)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_parser_agrees_with_go() {
|
||||
for (i, c) in vectors().header_cases.into_iter().enumerate() {
|
||||
let input = unhex(&c.input_hex);
|
||||
let got = header::parse(&input[..]);
|
||||
assert_eq!(
|
||||
got.is_ok(),
|
||||
c.accept,
|
||||
"case {i}: {:?}",
|
||||
String::from_utf8_lossy(&input[..input.len().min(80)])
|
||||
);
|
||||
if let Ok((_, mut rest)) = got {
|
||||
let mut r = Vec::new();
|
||||
rest.read_to_end(&mut r).unwrap();
|
||||
assert_eq!(hex(&r), c.rest_hex.unwrap_or_default(), "case {i} payload");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decrypts_go_files() {
|
||||
let id = Identity::from_suite_seed(SUITE_ID, &pattern(32, 1)).unwrap();
|
||||
for size in [0usize, 1, 65535, 65536, 65537, 131072, 200000] {
|
||||
let path = format!("{}/tests/vectors/fury/pattern-{size}.fury", env!("CARGO_MANIFEST_DIR"));
|
||||
let data = std::fs::read(&path).unwrap();
|
||||
let mut out = Vec::new();
|
||||
decrypt_reader(&data[..], &id).unwrap().read_to_end(&mut out).unwrap();
|
||||
assert_eq!(out, pattern(size, 0x5a), "{path}");
|
||||
let b = decrypt_bounded(&data[..], &id, size).unwrap();
|
||||
assert_eq!(b.expose(), &pattern(size, 0x5a)[..]);
|
||||
if size > 0 {
|
||||
assert!(matches!(
|
||||
decrypt_bounded(&data[..], &id, size - 1),
|
||||
Err(Error::TooLarge) | Err(Error::Io(_))
|
||||
));
|
||||
}
|
||||
let other = Identity::generate(&mut crate::rng::os());
|
||||
assert!(matches!(decrypt_reader(&data[..], &other), Err(Error::NoIdentityMatch)));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_and_tamper() {
|
||||
let id = Identity::generate(&mut crate::rng::os());
|
||||
for size in [0usize, 1, 65536, 70000] {
|
||||
let p = pattern(size, 3);
|
||||
let mut f = Vec::new();
|
||||
encrypt_to(&mut f, &id.recipient(), &p, &mut crate::rng::os());
|
||||
let mut w = encrypt_writer(Vec::new(), &id.recipient(), &mut crate::rng::os()).unwrap();
|
||||
w.write_all(&p).unwrap();
|
||||
let f2 = w.finish().unwrap();
|
||||
for f in [&f, &f2] {
|
||||
assert_eq!(decrypt_bounded(&f[..], &id, size).unwrap().expose(), &p[..]);
|
||||
let mut g = f.clone();
|
||||
let last = g.len() - 1;
|
||||
g[last] ^= 1;
|
||||
assert!(decrypt_bounded(&g[..], &id, size).is_err());
|
||||
let mut out = Vec::new();
|
||||
assert!(decrypt_reader(&g[..], &id).unwrap().read_to_end(&mut out).is_err());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// With identical synthetic randomness the in-memory and streaming
|
||||
/// encryptors produce identical files.
|
||||
#[test]
|
||||
fn deterministic_under_test_entropy() {
|
||||
let id = Identity::generate(&mut crate::rng::os());
|
||||
let e = crate::rng::Entropy::Seeded([9; 32]);
|
||||
let p = pattern(200_000, 1);
|
||||
let mut a = Vec::new();
|
||||
encrypt_to(&mut a, &id.recipient(), &p, &mut e.rng(b"x"));
|
||||
let mut w = encrypt_writer(Vec::new(), &id.recipient(), &mut e.rng(b"x")).unwrap();
|
||||
w.write_all(&p).unwrap();
|
||||
assert_eq!(a, w.finish().unwrap());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,396 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The Fury header: serialisation and a strict parser matching the Go
|
||||
//! `internal/format` package byte for byte.
|
||||
//!
|
||||
//! ```text
|
||||
//! fury-encryption.org/v1\n
|
||||
//! -> TYPE ARG...\n one stanza per recipient
|
||||
//! BASE64 body, 64 columns per line, ending with a line shorter than 64
|
||||
//! --- BASE64(MAC)\n
|
||||
//! ```
|
||||
//!
|
||||
//! Base64 is the standard alphabet without padding, with non-zero trailing
|
||||
//! bits rejected. A stanza's lines total at most 1 MiB and the header at
|
||||
//! most 16 MiB.
|
||||
|
||||
use base64::Engine;
|
||||
use base64::engine::{DecodePaddingMode, GeneralPurpose, GeneralPurposeConfig};
|
||||
use std::io::{self, Read, Write};
|
||||
|
||||
pub const MAX_HEADER_SIZE: u64 = 16 << 20;
|
||||
pub const MAX_STANZA_SIZE: u64 = 1 << 20;
|
||||
pub const INTRO: &[u8] = b"fury-encryption.org/v1\n";
|
||||
const STANZA_PREFIX: &[u8] = b"->";
|
||||
const FOOTER_PREFIX: &[u8] = b"---";
|
||||
const COLUMNS: usize = 64;
|
||||
const BYTES_PER_LINE: usize = COLUMNS / 4 * 3;
|
||||
|
||||
pub const B64: GeneralPurpose = GeneralPurpose::new(
|
||||
&base64::alphabet::STANDARD,
|
||||
GeneralPurposeConfig::new()
|
||||
.with_encode_padding(false)
|
||||
.with_decode_padding_mode(DecodePaddingMode::RequireNone)
|
||||
.with_decode_allow_trailing_bits(false),
|
||||
);
|
||||
|
||||
pub fn encode(b: &[u8]) -> String {
|
||||
B64.encode(b)
|
||||
}
|
||||
|
||||
/// Strict unpadded base64; CR and LF are rejected outright, as in Go.
|
||||
pub fn decode(s: &[u8]) -> Result<Vec<u8>, String> {
|
||||
if s.iter().any(|&c| c == b'\n' || c == b'\r') {
|
||||
return Err("unexpected newline character".into());
|
||||
}
|
||||
B64.decode(s).map_err(|e| e.to_string().to_lowercase())
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct Stanza {
|
||||
pub typ: String,
|
||||
pub args: Vec<String>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Clone, PartialEq, Eq)]
|
||||
pub struct Header {
|
||||
pub recipients: Vec<Stanza>,
|
||||
pub mac: [u8; 32],
|
||||
}
|
||||
|
||||
impl Stanza {
|
||||
pub fn marshal(&self, w: &mut Vec<u8>) {
|
||||
w.extend_from_slice(STANZA_PREFIX);
|
||||
for a in std::iter::once(&self.typ).chain(&self.args) {
|
||||
w.push(b' ');
|
||||
w.extend_from_slice(a.as_bytes());
|
||||
}
|
||||
w.push(b'\n');
|
||||
let enc = encode(&self.body);
|
||||
let mut chunks = enc.as_bytes().chunks(COLUMNS).peekable();
|
||||
let mut last_full = true;
|
||||
while let Some(c) = chunks.next() {
|
||||
w.extend_from_slice(c);
|
||||
if c.len() == COLUMNS {
|
||||
w.push(b'\n');
|
||||
} else {
|
||||
last_full = false;
|
||||
}
|
||||
if chunks.peek().is_none() && c.len() < COLUMNS {
|
||||
w.push(b'\n');
|
||||
}
|
||||
}
|
||||
// A body whose encoding fills its last line (or is empty) ends with
|
||||
// an empty line, so every stanza ends with a short line.
|
||||
if last_full {
|
||||
w.push(b'\n');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Header {
|
||||
/// Everything the MAC covers: intro, stanzas and the footer prefix.
|
||||
pub fn marshal_without_mac(&self) -> Vec<u8> {
|
||||
let mut w = INTRO.to_vec();
|
||||
for s in &self.recipients {
|
||||
s.marshal(&mut w);
|
||||
}
|
||||
w.extend_from_slice(FOOTER_PREFIX);
|
||||
w
|
||||
}
|
||||
|
||||
pub fn marshal(&self) -> Vec<u8> {
|
||||
let mut w = self.marshal_without_mac();
|
||||
w.push(b' ');
|
||||
w.extend_from_slice(encode(&self.mac).as_bytes());
|
||||
w.push(b'\n');
|
||||
w
|
||||
}
|
||||
|
||||
pub fn write_to(&self, w: &mut impl Write) -> io::Result<()> {
|
||||
w.write_all(&self.marshal())
|
||||
}
|
||||
}
|
||||
|
||||
/// A reader with as much look-ahead as the parser needs. After parsing,
|
||||
/// [`Peek::into_payload`] hands back the unread bytes followed by the rest
|
||||
/// of the source.
|
||||
/// The payload after a parsed header: read-ahead bytes, then the source.
|
||||
pub type Payload<R> = io::Chain<io::Cursor<Vec<u8>>, R>;
|
||||
|
||||
pub struct Peek<R> {
|
||||
inner: R,
|
||||
buf: Vec<u8>,
|
||||
pos: usize,
|
||||
eof: bool,
|
||||
}
|
||||
|
||||
impl<R: Read> Peek<R> {
|
||||
pub fn new(inner: R) -> Self {
|
||||
Peek {
|
||||
inner,
|
||||
buf: Vec::with_capacity(8192),
|
||||
pos: 0,
|
||||
eof: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn fill(&mut self) -> io::Result<bool> {
|
||||
if self.eof {
|
||||
return Ok(false);
|
||||
}
|
||||
if self.pos > 0 && self.pos == self.buf.len() {
|
||||
self.buf.clear();
|
||||
self.pos = 0;
|
||||
}
|
||||
let start = self.buf.len();
|
||||
self.buf.resize(start + 8192, 0);
|
||||
let n = loop {
|
||||
match self.inner.read(&mut self.buf[start..]) {
|
||||
Ok(n) => break n,
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => {
|
||||
self.buf.truncate(start);
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
};
|
||||
self.buf.truncate(start + n);
|
||||
if n == 0 {
|
||||
self.eof = true;
|
||||
}
|
||||
Ok(n > 0)
|
||||
}
|
||||
|
||||
/// Up to `n` bytes ahead, fewer only at end of input.
|
||||
fn peek(&mut self, n: usize) -> io::Result<&[u8]> {
|
||||
while self.buf.len() - self.pos < n && self.fill()? {}
|
||||
let end = (self.pos + n).min(self.buf.len());
|
||||
Ok(&self.buf[self.pos..end])
|
||||
}
|
||||
|
||||
/// Reads one line including its newline. Fails if the line, added to
|
||||
/// `*count`, would exceed `limit`, or if input ends before a newline.
|
||||
fn line(&mut self, count: &mut u64, limit: u64) -> Result<Vec<u8>, LineError> {
|
||||
let mut line = Vec::new();
|
||||
loop {
|
||||
if self.pos == self.buf.len() && !self.fill().map_err(LineError::Io)? {
|
||||
return Err(LineError::Eof);
|
||||
}
|
||||
let avail = &self.buf[self.pos..];
|
||||
let (take, done) = match avail.iter().position(|&c| c == b'\n') {
|
||||
Some(i) => (i + 1, true),
|
||||
None => (avail.len(), false),
|
||||
};
|
||||
if *count + take as u64 > limit {
|
||||
return Err(LineError::TooLong);
|
||||
}
|
||||
*count += take as u64;
|
||||
line.extend_from_slice(&avail[..take]);
|
||||
self.pos += take;
|
||||
if done {
|
||||
return Ok(line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn into_payload(self) -> Payload<R> {
|
||||
let mut rest = self.buf;
|
||||
rest.drain(..self.pos);
|
||||
io::Cursor::new(rest).chain(self.inner)
|
||||
}
|
||||
}
|
||||
|
||||
enum LineError {
|
||||
Io(io::Error),
|
||||
Eof,
|
||||
TooLong,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum ParseError {
|
||||
Io(io::Error),
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ParseError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
ParseError::Io(e) => write!(f, "failed to read header: {e}"),
|
||||
ParseError::Malformed(m) => write!(f, "parsing fury header: {m}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn bad<T>(m: impl Into<String>) -> Result<T, ParseError> {
|
||||
Err(ParseError::Malformed(m.into()))
|
||||
}
|
||||
|
||||
fn split_args(line: &[u8]) -> (&[u8], Vec<&[u8]>) {
|
||||
let l = line.strip_suffix(b"\n").unwrap_or(line);
|
||||
let mut parts = l.split(|&c| c == b' ');
|
||||
let first = parts.next().unwrap_or(b"");
|
||||
(first, parts.collect())
|
||||
}
|
||||
|
||||
fn valid_string(s: &[u8]) -> bool {
|
||||
!s.is_empty() && s.iter().all(|&c| (33..=126).contains(&c))
|
||||
}
|
||||
|
||||
fn read_stanza<R: Read>(p: &mut Peek<R>) -> Result<(Stanza, u64), ParseError> {
|
||||
let mut n = 0u64;
|
||||
let line = match p.line(&mut n, MAX_STANZA_SIZE) {
|
||||
Ok(l) => l,
|
||||
Err(LineError::TooLong) => return bad(format!("stanza exceeds {MAX_STANZA_SIZE} byte limit")),
|
||||
Err(LineError::Eof) => return bad("failed to read line: unexpected eof"),
|
||||
Err(LineError::Io(e)) => return Err(ParseError::Io(e)),
|
||||
};
|
||||
if !line.starts_with(STANZA_PREFIX) {
|
||||
return bad("malformed stanza opening line");
|
||||
}
|
||||
let (prefix, args) = split_args(&line);
|
||||
if prefix != STANZA_PREFIX || args.is_empty() || !args.iter().all(|a| valid_string(a)) {
|
||||
return bad("malformed stanza");
|
||||
}
|
||||
// valid_string admits only printable ASCII, so these are UTF-8.
|
||||
let text = |a: &[u8]| String::from_utf8(a.to_vec()).expect("ascii");
|
||||
let mut s = Stanza {
|
||||
typ: text(args[0]),
|
||||
args: args[1..].iter().map(|a| text(a)).collect(),
|
||||
body: Vec::new(),
|
||||
};
|
||||
loop {
|
||||
let line = match p.line(&mut n, MAX_STANZA_SIZE) {
|
||||
Ok(l) => l,
|
||||
Err(LineError::TooLong) => return bad(format!("stanza exceeds {MAX_STANZA_SIZE} byte limit")),
|
||||
Err(LineError::Eof) => return bad("failed to read line: unexpected eof"),
|
||||
Err(LineError::Io(e)) => return Err(ParseError::Io(e)),
|
||||
};
|
||||
let b = match decode(line.strip_suffix(b"\n").unwrap_or(&line)) {
|
||||
Ok(b) => b,
|
||||
Err(e) => {
|
||||
if line.starts_with(FOOTER_PREFIX) || line.starts_with(STANZA_PREFIX) {
|
||||
return bad("malformed body line: stanza ended without a short line");
|
||||
}
|
||||
return bad(format!("malformed body line: {e}"));
|
||||
}
|
||||
};
|
||||
if b.len() > BYTES_PER_LINE {
|
||||
return bad("malformed body line: too long");
|
||||
}
|
||||
let short = b.len() < BYTES_PER_LINE;
|
||||
s.body.extend_from_slice(&b);
|
||||
if short {
|
||||
return Ok((s, n));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses a header, returning it and the reader positioned at the payload.
|
||||
pub fn parse<R: Read>(input: R) -> Result<(Header, Payload<R>), ParseError> {
|
||||
let mut p = Peek::new(input);
|
||||
let mut total = 0u64;
|
||||
match p.line(&mut total, MAX_HEADER_SIZE) {
|
||||
Ok(l) if l == INTRO => {}
|
||||
Ok(_) => return bad("unexpected intro"),
|
||||
Err(LineError::TooLong) => return bad(format!("header exceeds {MAX_HEADER_SIZE} byte limit")),
|
||||
Err(LineError::Eof) if total == 0 => return bad("file is empty"),
|
||||
Err(LineError::Eof) => return bad("failed to read intro: unexpected eof"),
|
||||
Err(LineError::Io(e)) => return Err(ParseError::Io(e)),
|
||||
}
|
||||
let mut recipients = Vec::new();
|
||||
loop {
|
||||
let peek = p.peek(FOOTER_PREFIX.len()).map_err(ParseError::Io)?;
|
||||
if peek.len() < FOOTER_PREFIX.len() {
|
||||
return bad("failed to read header: unexpected eof");
|
||||
}
|
||||
if peek == FOOTER_PREFIX {
|
||||
let line = match p.line(&mut total, MAX_HEADER_SIZE) {
|
||||
Ok(l) => l,
|
||||
Err(LineError::TooLong) => return bad(format!("header exceeds {MAX_HEADER_SIZE} byte limit")),
|
||||
Err(LineError::Eof) => return bad("failed to read header: unexpected eof"),
|
||||
Err(LineError::Io(e)) => return Err(ParseError::Io(e)),
|
||||
};
|
||||
let (prefix, args) = split_args(&line);
|
||||
if prefix != FOOTER_PREFIX || args.len() != 1 {
|
||||
return bad("malformed closing line");
|
||||
}
|
||||
let mac = match decode(args[0]) {
|
||||
Ok(m) if m.len() == 32 => m,
|
||||
_ => return bad("malformed closing line"),
|
||||
};
|
||||
let mut m = [0u8; 32];
|
||||
m.copy_from_slice(&mac);
|
||||
return Ok((Header { recipients, mac: m }, p.into_payload()));
|
||||
}
|
||||
let (s, n) = read_stanza(&mut p)?;
|
||||
total += n;
|
||||
if total > MAX_HEADER_SIZE {
|
||||
return bad(format!("header exceeds {MAX_HEADER_SIZE} byte limit"));
|
||||
}
|
||||
recipients.push(s);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn stanza(body: usize) -> Stanza {
|
||||
Stanza {
|
||||
typ: "t".into(),
|
||||
args: vec!["a".into()],
|
||||
body: vec![7u8; body],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn body_line_wrapping() {
|
||||
let mut w = Vec::new();
|
||||
stanza(0).marshal(&mut w);
|
||||
assert_eq!(w, b"-> t a\n\n");
|
||||
w.clear();
|
||||
stanza(48).marshal(&mut w);
|
||||
let line = encode(&[7u8; 48]);
|
||||
assert_eq!(line.len(), 64);
|
||||
assert_eq!(w, format!("-> t a\n{line}\n\n").as_bytes());
|
||||
w.clear();
|
||||
stanza(50).marshal(&mut w);
|
||||
let enc = encode(&[7u8; 50]);
|
||||
assert_eq!(w, format!("-> t a\n{}\n{}\n", &enc[..64], &enc[64..]).as_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn round_trip_and_payload() {
|
||||
let h = Header {
|
||||
recipients: vec![stanza(48), stanza(3)],
|
||||
mac: [9; 32],
|
||||
};
|
||||
let mut b = h.marshal();
|
||||
b.extend_from_slice(b"payload");
|
||||
let (got, mut rest) = parse(&b[..]).unwrap();
|
||||
assert!(got == h);
|
||||
let mut p = Vec::new();
|
||||
rest.read_to_end(&mut p).unwrap();
|
||||
assert_eq!(p, b"payload");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn oversized_stanza_and_header() {
|
||||
let big = "A".repeat(MAX_STANZA_SIZE as usize);
|
||||
let h = format!("fury-encryption.org/v1\n-> x {big}\n\n--- {}\n", encode(&[0; 32]));
|
||||
assert!(parse(h.as_bytes()).is_err());
|
||||
// Just under the stanza limit is fine...
|
||||
let ok = "A".repeat(MAX_STANZA_SIZE as usize - 10);
|
||||
let h = format!("fury-encryption.org/v1\n-> x {ok}\n\n--- {}\n", encode(&[0; 32]));
|
||||
assert!(parse(h.as_bytes()).is_ok());
|
||||
// ...but seventeen such stanzas pass 16 MiB.
|
||||
let mut h = String::from("fury-encryption.org/v1\n");
|
||||
for _ in 0..17 {
|
||||
h.push_str(&format!("-> x {ok}\n\n"));
|
||||
}
|
||||
h.push_str(&format!("--- {}\n", encode(&[0; 32])));
|
||||
assert!(matches!(parse(h.as_bytes()), Err(ParseError::Malformed(m)) if m.contains("header exceeds")));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,425 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The hybrid FrodoKEM-640-SHAKE + ML-KEM-768 suite (ID 0x0001, stanza tag
|
||||
//! `frodo640shake+mlkem768`), as the Go `pqcrypt` package defines it.
|
||||
//!
|
||||
//! - Keys: a 32-byte master seed is expanded with
|
||||
//! HKDF-SHA-256(seed, salt = none, info = "fury/v1 seed-expand " + tag)
|
||||
//! into a 48-byte FrodoKEM seed followed by a 64-byte ML-KEM seed (d||z).
|
||||
//! - Wrap: both KEMs encapsulate with fresh randomness; the KEK is
|
||||
//! HKDF-SHA-256(ss1 || ss2, salt = none, info = "fury/v1 kek " + tag ||
|
||||
//! ct1 || ct2), and the 32-byte file key is sealed under it with
|
||||
//! ChaCha20-Poly1305 and an all-zero nonce. The zero nonce is safe only
|
||||
//! because every KEK is fresh and used once; nothing else may use
|
||||
//! [`seal_once`].
|
||||
|
||||
use super::frodo;
|
||||
use super::header::{self, Stanza};
|
||||
use crate::rng::Rng;
|
||||
use crate::secret::SecretArray;
|
||||
use chacha20poly1305::aead::{AeadInOut, KeyInit};
|
||||
use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, Tag};
|
||||
use hkdf::Hkdf;
|
||||
use ml_kem::kem::{Decapsulate, KeyExport};
|
||||
use ml_kem::{DecapsulationKey, EncapsulationKey, MlKem768};
|
||||
use sha2::Sha256;
|
||||
use zeroize::{Zeroize, Zeroizing};
|
||||
|
||||
pub const SUITE_ID: u16 = 0x0001;
|
||||
pub const TAG: &str = "frodo640shake+mlkem768";
|
||||
pub const SEED_SIZE: usize = 32;
|
||||
pub const FILE_KEY_SIZE: usize = 32;
|
||||
const MLKEM_PK: usize = 1184;
|
||||
const MLKEM_CT: usize = 1088;
|
||||
const MLKEM_SEED: usize = 64;
|
||||
pub const PUBLIC_KEY_SIZE: usize = frodo::PUBLIC_KEY_SIZE + MLKEM_PK;
|
||||
|
||||
/// Why a stanza could not be unwrapped.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum UnwrapError {
|
||||
/// The stanza is not for this identity (wrong suite, or the file key
|
||||
/// does not open): try the next stanza or identity.
|
||||
NotMine,
|
||||
/// The stanza claims this suite but is malformed; Go treats this as
|
||||
/// fatal for the whole file.
|
||||
Malformed(String),
|
||||
}
|
||||
|
||||
fn hkdf_expand(ikm: &[u8], salt: Option<&[u8]>, info: &[&[u8]], out: &mut [u8]) {
|
||||
let h = Hkdf::<Sha256>::new(salt, ikm);
|
||||
h.expand_multi_info(info, out).expect("hkdf output length within limits");
|
||||
}
|
||||
|
||||
/// The recipient: FrodoKEM public key || ML-KEM encapsulation key.
|
||||
pub struct Recipient {
|
||||
bytes: Vec<u8>,
|
||||
frodo: frodo::PublicKey,
|
||||
mlkem: EncapsulationKey<MlKem768>,
|
||||
}
|
||||
|
||||
impl Recipient {
|
||||
pub fn from_bytes(pub_key: &[u8]) -> Result<Self, String> {
|
||||
if pub_key.len() != PUBLIC_KEY_SIZE {
|
||||
return Err(format!("invalid {TAG} public key size"));
|
||||
}
|
||||
let frodo = frodo::PublicKey::from_bytes(&pub_key[..frodo::PUBLIC_KEY_SIZE]).ok_or("invalid FrodoKEM public key")?;
|
||||
let ek = ml_kem::kem::Key::<EncapsulationKey<MlKem768>>::try_from(&pub_key[frodo::PUBLIC_KEY_SIZE..])
|
||||
.map_err(|_| "invalid ML-KEM public key")?;
|
||||
let mlkem = EncapsulationKey::<MlKem768>::new(&ek).map_err(|_| "invalid ML-KEM public key")?;
|
||||
Ok(Recipient {
|
||||
bytes: pub_key.to_vec(),
|
||||
frodo,
|
||||
mlkem,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn bytes(&self) -> &[u8] {
|
||||
&self.bytes
|
||||
}
|
||||
|
||||
/// Wraps `file_key` with fresh encapsulations drawn from `rng`.
|
||||
pub fn wrap(&self, file_key: &[u8; FILE_KEY_SIZE], rng: &mut Rng) -> Stanza {
|
||||
let mut mu = Zeroizing::new([0u8; frodo::ENCAPSULATION_SEED_SIZE]);
|
||||
rng.fill(&mut mu[..]);
|
||||
let mut m = Zeroizing::new([0u8; 32]);
|
||||
rng.fill(&mut m[..]);
|
||||
self.wrap_with(file_key, &mu, &m)
|
||||
}
|
||||
|
||||
fn wrap_with(&self, file_key: &[u8; FILE_KEY_SIZE], mu: &[u8; 16], m: &[u8; 32]) -> Stanza {
|
||||
let (ct1, mut ss1) = self.frodo.encapsulate(mu);
|
||||
let (ct2, ss2) = self.mlkem.encapsulate_deterministic(&(*m).into());
|
||||
let mut ss2 = Zeroizing::new(ss2);
|
||||
let kek = derive_kek(&ss1, &ss2[..], &ct1, &ct2[..]);
|
||||
ss1.zeroize();
|
||||
ss2.zeroize();
|
||||
let body = seal_once(&kek, file_key);
|
||||
Stanza {
|
||||
typ: TAG.into(),
|
||||
args: vec![header::encode(&ct1), header::encode(&ct2[..])],
|
||||
body,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// HKDF over both shared secrets, binding both ciphertexts.
|
||||
fn derive_kek(ss1: &[u8], ss2: &[u8], ct1: &[u8], ct2: &[u8]) -> SecretArray<32> {
|
||||
let mut secret = Zeroizing::new([0u8; frodo::SHARED_KEY_SIZE + 32]);
|
||||
secret[..frodo::SHARED_KEY_SIZE].copy_from_slice(ss1);
|
||||
secret[frodo::SHARED_KEY_SIZE..].copy_from_slice(ss2);
|
||||
let label = format!("fury/v1 kek {TAG}");
|
||||
let mut kek = SecretArray::<32>::zero();
|
||||
hkdf_expand(&secret[..], None, &[label.as_bytes(), ct1, ct2], kek.expose_mut());
|
||||
kek
|
||||
}
|
||||
|
||||
/// ChaCha20-Poly1305 under an all-zero nonce, for a key used exactly once.
|
||||
fn seal_once(kek: &SecretArray<32>, file_key: &[u8; FILE_KEY_SIZE]) -> Vec<u8> {
|
||||
let mut k = Key::from(*kek.expose());
|
||||
let aead = ChaCha20Poly1305::new(&k);
|
||||
k.zeroize();
|
||||
let mut body = file_key.to_vec();
|
||||
let tag = aead
|
||||
.encrypt_inout_detached(&Nonce::default(), &[], body.as_mut_slice().into())
|
||||
.expect("32 bytes fit");
|
||||
body.extend_from_slice(&tag);
|
||||
body
|
||||
}
|
||||
|
||||
fn open_once(kek: &SecretArray<32>, body: &[u8]) -> Option<SecretArray<FILE_KEY_SIZE>> {
|
||||
if body.len() != FILE_KEY_SIZE + 16 {
|
||||
return None;
|
||||
}
|
||||
let mut k = Key::from(*kek.expose());
|
||||
let aead = ChaCha20Poly1305::new(&k);
|
||||
k.zeroize();
|
||||
let mut out = SecretArray::<FILE_KEY_SIZE>::zero();
|
||||
out.expose_mut().copy_from_slice(&body[..FILE_KEY_SIZE]);
|
||||
let tag = Tag::try_from(&body[FILE_KEY_SIZE..]).ok()?;
|
||||
aead.decrypt_inout_detached(&Nonce::default(), &[], (&mut out.expose_mut()[..]).into(), &tag)
|
||||
.ok()?;
|
||||
Some(out)
|
||||
}
|
||||
|
||||
/// A private identity derived from a master seed. It owns the seed and
|
||||
/// both expanded private keys; dropping it clears what this crate owns
|
||||
/// (see `crate::secret` for the limits of that). There is no way to use an
|
||||
/// identity after [`Identity::close`] or drop:
|
||||
///
|
||||
/// ```compile_fail
|
||||
/// let id = vault::crypto::kem::Identity::generate(&mut vault::rng::os());
|
||||
/// id.close();
|
||||
/// let _ = id.seed(); // use of a moved value
|
||||
/// ```
|
||||
pub struct Identity {
|
||||
seed: SecretArray<SEED_SIZE>,
|
||||
frodo: frodo::PrivateKey,
|
||||
mlkem: Box<DecapsulationKey<MlKem768>>,
|
||||
recipient_bytes: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Identity {
|
||||
pub fn generate(rng: &mut Rng) -> Self {
|
||||
let mut seed = SecretArray::<SEED_SIZE>::zero();
|
||||
rng.fill(seed.expose_mut());
|
||||
Self::from_seed(seed)
|
||||
}
|
||||
|
||||
/// Derives the identity of `suite` from its master seed.
|
||||
pub fn from_suite_seed(suite: u16, seed: &[u8]) -> Result<Self, String> {
|
||||
if suite != SUITE_ID {
|
||||
return Err(format!("unknown suite id {suite:#06x}"));
|
||||
}
|
||||
let s = SecretArray::<SEED_SIZE>::from_slice(seed).ok_or_else(|| format!("invalid seed size {} for suite {TAG}", seed.len()))?;
|
||||
Ok(Self::from_seed(s))
|
||||
}
|
||||
|
||||
pub fn from_seed(seed: SecretArray<SEED_SIZE>) -> Self {
|
||||
let mut expanded = Zeroizing::new([0u8; frodo::KEY_SEED_SIZE + MLKEM_SEED]);
|
||||
let info = format!("fury/v1 seed-expand {TAG}");
|
||||
hkdf_expand(seed.expose(), None, &[info.as_bytes()], &mut expanded[..]);
|
||||
let mut fseed = Zeroizing::new([0u8; frodo::KEY_SEED_SIZE]);
|
||||
fseed.copy_from_slice(&expanded[..frodo::KEY_SEED_SIZE]);
|
||||
let frodo = frodo::PrivateKey::derive(&fseed);
|
||||
// ml-kem takes the seed by value (a stack copy we cannot clear) and
|
||||
// keeps it, zeroizing it on drop.
|
||||
let mut mseed = ml_kem::Seed::default();
|
||||
mseed.copy_from_slice(&expanded[frodo::KEY_SEED_SIZE..]);
|
||||
let mlkem = Box::new(DecapsulationKey::<MlKem768>::from_seed(mseed));
|
||||
mseed.zeroize();
|
||||
let mut recipient_bytes = frodo.public().to_bytes();
|
||||
recipient_bytes.extend_from_slice(&mlkem.encapsulation_key().to_bytes());
|
||||
Identity {
|
||||
seed,
|
||||
frodo,
|
||||
mlkem,
|
||||
recipient_bytes,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn suite_id(&self) -> u16 {
|
||||
SUITE_ID
|
||||
}
|
||||
|
||||
/// The master seed: secret-equivalent to the whole identity.
|
||||
pub fn seed(&self) -> &[u8; SEED_SIZE] {
|
||||
self.seed.expose()
|
||||
}
|
||||
|
||||
pub fn recipient(&self) -> Recipient {
|
||||
Recipient::from_bytes(&self.recipient_bytes).expect("derived key is valid")
|
||||
}
|
||||
|
||||
pub fn recipient_bytes(&self) -> &[u8] {
|
||||
&self.recipient_bytes
|
||||
}
|
||||
|
||||
/// The private key bytes as Go serialises them (FrodoKEM || ML-KEM
|
||||
/// expanded), for compatibility checks only.
|
||||
#[allow(deprecated)]
|
||||
pub fn private_key_bytes(&self) -> Zeroizing<Vec<u8>> {
|
||||
use ml_kem::ExpandedKeyEncoding;
|
||||
let mut v = self.frodo.to_bytes();
|
||||
let mut m = self.mlkem.to_expanded_bytes();
|
||||
v.extend_from_slice(&m);
|
||||
m.zeroize();
|
||||
v
|
||||
}
|
||||
|
||||
pub fn close(self) {}
|
||||
|
||||
/// Go's `unwrap` for one stanza.
|
||||
pub fn unwrap_stanza(&self, s: &Stanza) -> Result<SecretArray<FILE_KEY_SIZE>, UnwrapError> {
|
||||
if s.typ != TAG {
|
||||
return Err(UnwrapError::NotMine);
|
||||
}
|
||||
if s.args.len() != 2 {
|
||||
return Err(UnwrapError::Malformed(format!("invalid {TAG} stanza: expected 2 arguments")));
|
||||
}
|
||||
let ct1 = header::decode(s.args[0].as_bytes()).map_err(|e| UnwrapError::Malformed(format!("invalid FrodoKEM ciphertext: {e}")))?;
|
||||
let ct2 = header::decode(s.args[1].as_bytes()).map_err(|e| UnwrapError::Malformed(format!("invalid ML-KEM ciphertext: {e}")))?;
|
||||
if ct1.len() != frodo::CIPHERTEXT_SIZE || ct2.len() != MLKEM_CT {
|
||||
return Err(UnwrapError::Malformed("encrypted value has unexpected length".into()));
|
||||
}
|
||||
let mut ss1 = self.frodo.decapsulate(&ct1).expect("length checked");
|
||||
let ct2a = ml_kem::Ciphertext::<MlKem768>::try_from(&ct2[..]).expect("length checked");
|
||||
let mut ss2 = self.mlkem.decapsulate(&ct2a);
|
||||
// Both KEMs reject implicitly: a key mismatch shows up as a wrong
|
||||
// KEK, which the AEAD below detects.
|
||||
let kek = derive_kek(&ss1, &ss2[..], &ct1, &ct2);
|
||||
ss1.zeroize();
|
||||
ss2.zeroize();
|
||||
open_once(&kek, &s.body).ok_or(UnwrapError::NotMine)
|
||||
}
|
||||
|
||||
/// Go's `multiUnwrap`: the first stanza that opens wins; a malformed
|
||||
/// stanza of this suite stops the search.
|
||||
pub fn unwrap(&self, stanzas: &[Stanza]) -> Result<SecretArray<FILE_KEY_SIZE>, UnwrapError> {
|
||||
for s in stanzas {
|
||||
match self.unwrap_stanza(s) {
|
||||
Err(UnwrapError::NotMine) => continue,
|
||||
r => return r,
|
||||
}
|
||||
}
|
||||
Err(UnwrapError::NotMine)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use crate::crypto::stream::tests::hex;
|
||||
use serde::Deserialize;
|
||||
use sha2::Digest;
|
||||
|
||||
pub fn unhex(s: &str) -> Vec<u8> {
|
||||
(0..s.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&s[i..i + 2], 16).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sha(b: &[u8]) -> String {
|
||||
hex(&sha2::Sha256::digest(b))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct Crypto {
|
||||
pub frodo640shake: Vec<KemV>,
|
||||
pub mlkem768: Vec<KemV>,
|
||||
pub hybrid: Vec<HybridV>,
|
||||
pub wrap: Vec<WrapV>,
|
||||
pub header_cases: Vec<HeaderCase>,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct KemV {
|
||||
seed: String,
|
||||
pub_sha256: String,
|
||||
priv_sha256: String,
|
||||
encap_seed: String,
|
||||
ct_sha256: String,
|
||||
ss: String,
|
||||
reject_flip_index: usize,
|
||||
reject_ss: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct HybridV {
|
||||
master_seed: String,
|
||||
pub_sha256: String,
|
||||
priv_sha256: String,
|
||||
pub_prefix: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct WrapV {
|
||||
pub master_seed: String,
|
||||
frodo_encap_seed: String,
|
||||
mlkem_encap_seed: String,
|
||||
pub file_key: String,
|
||||
kek: String,
|
||||
pub header_hex: String,
|
||||
pub mac: String,
|
||||
pub nonce: String,
|
||||
pub payload_key: String,
|
||||
stanza_arg0_sha256: String,
|
||||
stanza_body: String,
|
||||
}
|
||||
#[derive(Deserialize)]
|
||||
pub struct HeaderCase {
|
||||
pub input_hex: String,
|
||||
pub accept: bool,
|
||||
pub rest_hex: Option<String>,
|
||||
}
|
||||
|
||||
pub fn vectors() -> Crypto {
|
||||
serde_json::from_str(include_str!("../../tests/vectors/crypto.json")).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn frodo_matches_circl() {
|
||||
for v in vectors().frodo640shake {
|
||||
let seed: [u8; frodo::KEY_SEED_SIZE] = unhex(&v.seed).try_into().unwrap();
|
||||
let sk = frodo::PrivateKey::derive(&seed);
|
||||
let pk = sk.public().to_bytes();
|
||||
assert_eq!(sha(&pk), v.pub_sha256);
|
||||
assert_eq!(sha(&sk.to_bytes()), v.priv_sha256);
|
||||
let pk2 = frodo::PublicKey::from_bytes(&pk).unwrap();
|
||||
let mu: [u8; 16] = unhex(&v.encap_seed).try_into().unwrap();
|
||||
let (ct, ss) = pk2.encapsulate(&mu);
|
||||
assert_eq!(sha(&ct), v.ct_sha256);
|
||||
assert_eq!(hex(&ss), v.ss);
|
||||
assert_eq!(hex(&sk.decapsulate(&ct).unwrap()), v.ss);
|
||||
let mut bad = ct.clone();
|
||||
bad[v.reject_flip_index] ^= 1;
|
||||
assert_eq!(hex(&sk.decapsulate(&bad).unwrap()), v.reject_ss, "implicit rejection");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[allow(deprecated)]
|
||||
fn mlkem_matches_circl() {
|
||||
use ml_kem::ExpandedKeyEncoding;
|
||||
for v in vectors().mlkem768 {
|
||||
let mut seed = ml_kem::Seed::default();
|
||||
seed.copy_from_slice(&unhex(&v.seed));
|
||||
let dk = DecapsulationKey::<MlKem768>::from_seed(seed);
|
||||
let ek = dk.encapsulation_key();
|
||||
assert_eq!(sha(&ek.to_bytes()), v.pub_sha256);
|
||||
assert_eq!(sha(&dk.to_expanded_bytes()), v.priv_sha256);
|
||||
let m: [u8; 32] = unhex(&v.encap_seed).try_into().unwrap();
|
||||
let (ct, ss) = ek.encapsulate_deterministic(&m.into());
|
||||
assert_eq!(sha(&ct), v.ct_sha256);
|
||||
assert_eq!(hex(&ss), v.ss);
|
||||
let mut bad = ct;
|
||||
bad[v.reject_flip_index] ^= 1;
|
||||
assert_eq!(hex(&dk.decapsulate(&bad)), v.reject_ss, "implicit rejection");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hybrid_derivation_matches_go() {
|
||||
for v in vectors().hybrid {
|
||||
let id = Identity::from_suite_seed(SUITE_ID, &unhex(&v.master_seed)).unwrap();
|
||||
assert_eq!(sha(id.recipient_bytes()), v.pub_sha256);
|
||||
assert_eq!(hex(&id.recipient_bytes()[..64]), v.pub_prefix);
|
||||
assert_eq!(sha(&id.private_key_bytes()), v.priv_sha256);
|
||||
}
|
||||
assert!(Identity::from_suite_seed(2, &[0; 32]).is_err());
|
||||
assert!(Identity::from_suite_seed(SUITE_ID, &[0; 31]).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn wrap_matches_go_and_unwraps() {
|
||||
for v in vectors().wrap {
|
||||
let id = Identity::from_suite_seed(SUITE_ID, &unhex(&v.master_seed)).unwrap();
|
||||
let fk: [u8; 32] = unhex(&v.file_key).try_into().unwrap();
|
||||
let mu: [u8; 16] = unhex(&v.frodo_encap_seed).try_into().unwrap();
|
||||
let m: [u8; 32] = unhex(&v.mlkem_encap_seed).try_into().unwrap();
|
||||
let st = id.recipient().wrap_with(&fk, &mu, &m);
|
||||
assert_eq!(sha(st.args[0].as_bytes()), v.stanza_arg0_sha256);
|
||||
assert_eq!(hex(&st.body), v.stanza_body);
|
||||
let ct1 = header::decode(st.args[0].as_bytes()).unwrap();
|
||||
let ct2 = header::decode(st.args[1].as_bytes()).unwrap();
|
||||
let ss1 = id.frodo.decapsulate(&ct1).unwrap();
|
||||
let ss2 = id.mlkem.decapsulate(&ml_kem::Ciphertext::<MlKem768>::try_from(&ct2[..]).unwrap());
|
||||
assert_eq!(hex(derive_kek(&ss1, &ss2[..], &ct1, &ct2).expose()), v.kek);
|
||||
assert_eq!(id.unwrap(&[st]).unwrap().expose(), &fk);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_wraps_differ_and_foreign_keys_do_not_match() {
|
||||
let a = Identity::generate(&mut crate::rng::os());
|
||||
let b = Identity::generate(&mut crate::rng::os());
|
||||
let fk = [7u8; 32];
|
||||
let s1 = a.recipient().wrap(&fk, &mut crate::rng::os());
|
||||
let s2 = a.recipient().wrap(&fk, &mut crate::rng::os());
|
||||
assert!(s1.args != s2.args && s1.body != s2.body, "encapsulations must be fresh");
|
||||
assert_eq!(b.unwrap(std::slice::from_ref(&s1)).err(), Some(UnwrapError::NotMine));
|
||||
let mut bad = s1.clone();
|
||||
bad.args.pop();
|
||||
assert!(matches!(a.unwrap(&[bad]), Err(UnwrapError::Malformed(_))));
|
||||
let mut other = s1.clone();
|
||||
other.typ = "x25519".into();
|
||||
assert_eq!(a.unwrap(&[other, s1]).unwrap().expose(), &fk);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Fury file encryption, compatible with the Go `internal/pqcrypt`,
|
||||
//! `internal/format` and `internal/stream` packages.
|
||||
|
||||
pub mod frodo;
|
||||
pub mod fury;
|
||||
pub mod header;
|
||||
pub mod kem;
|
||||
pub mod stream;
|
||||
@@ -0,0 +1,816 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The STREAM payload encryption of the Go `internal/stream` package.
|
||||
//!
|
||||
//! The plaintext is cut into 64 KiB chunks, each sealed with
|
||||
//! ChaCha20-Poly1305 under a 12-byte nonce whose bytes 3..11 hold the
|
||||
//! big-endian chunk index and whose last byte is 1 for the final chunk
|
||||
//! only. Only the final chunk may be short; it may be empty only when it is
|
||||
//! the sole chunk (an empty plaintext). A reader rejects truncation (no
|
||||
//! final-flagged chunk), an empty non-first final chunk, and any byte after
|
||||
//! the final chunk.
|
||||
//!
|
||||
//! Chunk nonces depend only on the index, so chunks may be sealed or opened
|
||||
//! in any order and produce identical bytes; the readers here classify each
|
||||
//! chunk as final or not from the shape of the stream (one chunk of
|
||||
//! read-ahead) and open it exactly once, which is the same verdict Go's
|
||||
//! sequential try-then-retry reader reaches.
|
||||
|
||||
use chacha20poly1305::aead::{AeadInOut, KeyInit};
|
||||
use chacha20poly1305::{ChaCha20Poly1305, Key, Nonce, Tag};
|
||||
use std::io::{self, Read, Write};
|
||||
use zeroize::Zeroize;
|
||||
|
||||
pub const CHUNK_SIZE: usize = 64 * 1024;
|
||||
pub const TAG_SIZE: usize = 16;
|
||||
pub const ENC_CHUNK_SIZE: usize = CHUNK_SIZE + TAG_SIZE;
|
||||
const LAST_CHUNK_FLAG: u8 = 0x01;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum StreamError {
|
||||
/// The stream ended before a final chunk.
|
||||
Truncated,
|
||||
/// A chunk failed to authenticate.
|
||||
Auth,
|
||||
EmptyLastChunk,
|
||||
TrailingData,
|
||||
CounterOverflow,
|
||||
/// The ciphertext length cannot be a STREAM encoding.
|
||||
InvalidSize(u64),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for StreamError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(match self {
|
||||
StreamError::Truncated => "unexpected eof: encrypted payload is truncated",
|
||||
StreamError::Auth => "failed to decrypt and authenticate payload chunk, file may be corrupted or tampered with",
|
||||
StreamError::EmptyLastChunk => "last chunk is empty; encrypted payload is malformed",
|
||||
StreamError::TrailingData => "trailing data after end of encrypted file",
|
||||
StreamError::CounterOverflow => "stream: chunk counter would wrap around",
|
||||
StreamError::InvalidSize(n) => return write!(f, "invalid encrypted payload size: {n}"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StreamError {}
|
||||
|
||||
impl From<StreamError> for io::Error {
|
||||
fn from(e: StreamError) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::InvalidData, e)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn nonce(index: u64, last: bool) -> Nonce {
|
||||
let mut n = [0u8; 12];
|
||||
n[3..11].copy_from_slice(&index.to_be_bytes());
|
||||
if last {
|
||||
n[11] = LAST_CHUNK_FLAG;
|
||||
}
|
||||
Nonce::from(n)
|
||||
}
|
||||
|
||||
/// A STREAM key. The cipher state holds a copy of the key that
|
||||
/// `chacha20poly1305` does not expose for clearing; see SECURITY.md.
|
||||
pub struct Cipher(ChaCha20Poly1305);
|
||||
|
||||
impl Cipher {
|
||||
pub fn new(key: &[u8; 32]) -> Self {
|
||||
let mut k = Key::from(*key);
|
||||
let c = ChaCha20Poly1305::new(&k);
|
||||
k.zeroize();
|
||||
Cipher(c)
|
||||
}
|
||||
|
||||
/// Seals `buf` (at most CHUNK_SIZE bytes of plaintext followed by
|
||||
/// TAG_SIZE bytes of room) in place.
|
||||
pub fn seal(&self, index: u64, last: bool, buf: &mut [u8]) {
|
||||
let n = buf.len() - TAG_SIZE;
|
||||
debug_assert!(n <= CHUNK_SIZE);
|
||||
let (msg, tag) = buf.split_at_mut(n);
|
||||
let t = self
|
||||
.0
|
||||
.encrypt_inout_detached(&nonce(index, last), &[], msg.into())
|
||||
.expect("chunk within aead limits");
|
||||
tag.copy_from_slice(&t);
|
||||
}
|
||||
|
||||
/// Opens a sealed chunk in place; on success the first `len - 16` bytes
|
||||
/// are plaintext. On failure the buffer holds no plaintext.
|
||||
pub fn open(&self, index: u64, last: bool, buf: &mut [u8]) -> Result<usize, StreamError> {
|
||||
if buf.len() < TAG_SIZE {
|
||||
return Err(StreamError::Auth);
|
||||
}
|
||||
let n = buf.len() - TAG_SIZE;
|
||||
let (msg, tag) = buf.split_at_mut(n);
|
||||
let tag = Tag::try_from(&*tag).expect("tag size");
|
||||
self.0
|
||||
.decrypt_inout_detached(&nonce(index, last), &[], msg.into(), &tag)
|
||||
.map_err(|_| StreamError::Auth)?;
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
fn next_index(i: u64) -> Result<u64, StreamError> {
|
||||
i.checked_add(1).ok_or(StreamError::CounterOverflow)
|
||||
}
|
||||
|
||||
/// Encrypting writer. Call [`StreamWriter::finish`] to seal the final
|
||||
/// chunk; dropping without finishing leaves a truncated stream that every
|
||||
/// reader rejects.
|
||||
pub struct StreamWriter<W: Write> {
|
||||
cipher: Cipher,
|
||||
dst: Option<W>,
|
||||
buf: Vec<u8>,
|
||||
index: u64,
|
||||
/// Set by any failure: like Go's sticky error, every later call fails.
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
impl<W: Write> StreamWriter<W> {
|
||||
pub fn new(key: &[u8; 32], dst: W) -> Self {
|
||||
StreamWriter {
|
||||
cipher: Cipher::new(key),
|
||||
dst: Some(dst),
|
||||
buf: Vec::with_capacity(ENC_CHUNK_SIZE),
|
||||
index: 0,
|
||||
failed: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn starting_at(key: &[u8; 32], dst: W, index: u64) -> Self {
|
||||
let mut w = Self::new(key, dst);
|
||||
w.index = index;
|
||||
w
|
||||
}
|
||||
|
||||
fn check(&self) -> io::Result<()> {
|
||||
if self.failed {
|
||||
Err(io::Error::other("encrypted stream writer already failed"))
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn flush_chunk(&mut self, last: bool) -> io::Result<()> {
|
||||
let r = self.seal_and_write(last);
|
||||
if r.is_err() {
|
||||
self.failed = true;
|
||||
self.buf.zeroize();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn seal_and_write(&mut self, last: bool) -> io::Result<()> {
|
||||
let n = self.buf.len();
|
||||
self.buf.resize(n + TAG_SIZE, 0);
|
||||
self.cipher.seal(self.index, last, &mut self.buf);
|
||||
self.dst.as_mut().expect("writer in use").write_all(&self.buf)?;
|
||||
self.buf.clear();
|
||||
self.index = next_index(self.index)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn finish(mut self) -> io::Result<W> {
|
||||
self.check()?;
|
||||
self.flush_chunk(true)?;
|
||||
Ok(self.dst.take().expect("writer in use"))
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Write for StreamWriter<W> {
|
||||
fn write(&mut self, mut p: &[u8]) -> io::Result<usize> {
|
||||
self.check()?;
|
||||
let total = p.len();
|
||||
while !p.is_empty() {
|
||||
let take = p.len().min(CHUNK_SIZE - self.buf.len());
|
||||
self.buf.extend_from_slice(&p[..take]);
|
||||
p = &p[take..];
|
||||
// Flush a full chunk only once more data proves it is not the
|
||||
// last one.
|
||||
if self.buf.len() == CHUNK_SIZE && !p.is_empty() {
|
||||
self.flush_chunk(false)?;
|
||||
}
|
||||
}
|
||||
Ok(total)
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.check()?;
|
||||
self.dst.as_mut().expect("writer in use").flush()
|
||||
}
|
||||
}
|
||||
|
||||
impl<W: Write> Drop for StreamWriter<W> {
|
||||
fn drop(&mut self) {
|
||||
self.buf.zeroize();
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads until `buf` is full or the source ends; returns the count.
|
||||
pub fn read_full<R: Read + ?Sized>(src: &mut R, buf: &mut [u8]) -> io::Result<usize> {
|
||||
let mut n = 0;
|
||||
while n < buf.len() {
|
||||
match src.read(&mut buf[n..]) {
|
||||
Ok(0) => break,
|
||||
Ok(k) => n += k,
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// After a final chunk: the source must be at end of input.
|
||||
fn expect_eof(src: &mut impl Read) -> io::Result<()> {
|
||||
let mut b = [0u8; 1];
|
||||
loop {
|
||||
match src.read(&mut b) {
|
||||
Ok(0) => return Ok(()),
|
||||
Ok(_) => return Err(StreamError::TrailingData.into()),
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decrypting reader with one chunk of read-ahead: a full-length chunk is
|
||||
/// held until the next read shows whether anything follows it.
|
||||
pub struct StreamReader<R: Read> {
|
||||
cipher: Cipher,
|
||||
src: R,
|
||||
index: u64,
|
||||
started: bool,
|
||||
done: bool,
|
||||
failed: bool,
|
||||
pending: Option<Vec<u8>>,
|
||||
/// Plaintext of a short final chunk, opened together with the chunk
|
||||
/// before it and delivered after it.
|
||||
queued: Option<Vec<u8>>,
|
||||
out: Vec<u8>,
|
||||
pos: usize,
|
||||
spare: Vec<Vec<u8>>,
|
||||
}
|
||||
|
||||
impl<R: Read> StreamReader<R> {
|
||||
pub fn new(key: &[u8; 32], src: R) -> Self {
|
||||
StreamReader {
|
||||
cipher: Cipher::new(key),
|
||||
src,
|
||||
index: 0,
|
||||
started: false,
|
||||
done: false,
|
||||
failed: false,
|
||||
pending: None,
|
||||
queued: None,
|
||||
out: Vec::new(),
|
||||
pos: 0,
|
||||
spare: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads one raw chunk into a recycled buffer, truncated to its length.
|
||||
fn read_raw(&mut self) -> io::Result<Vec<u8>> {
|
||||
let mut b = self.spare.pop().unwrap_or_default();
|
||||
b.resize(ENC_CHUNK_SIZE, 0);
|
||||
let n = read_full(&mut self.src, &mut b)?;
|
||||
b.truncate(n);
|
||||
Ok(b)
|
||||
}
|
||||
|
||||
/// Opens `raw` in place as chunk `self.index`, leaving its plaintext.
|
||||
fn open(&mut self, mut raw: Vec<u8>, last: bool) -> io::Result<Vec<u8>> {
|
||||
match self.cipher.open(self.index, last, &mut raw) {
|
||||
Ok(n) => {
|
||||
raw.truncate(n);
|
||||
self.index = next_index(self.index)?;
|
||||
Ok(raw)
|
||||
}
|
||||
Err(e) => {
|
||||
raw.zeroize();
|
||||
Err(e.into())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn deliver(&mut self, plain: Vec<u8>) {
|
||||
let mut old = std::mem::replace(&mut self.out, plain);
|
||||
old.zeroize();
|
||||
old.clear();
|
||||
self.spare.push(old);
|
||||
self.pos = 0;
|
||||
}
|
||||
|
||||
fn advance(&mut self) -> io::Result<()> {
|
||||
if !self.started {
|
||||
self.started = true;
|
||||
let raw = self.read_raw()?;
|
||||
if raw.is_empty() {
|
||||
return Err(StreamError::Truncated.into());
|
||||
}
|
||||
if raw.len() < ENC_CHUNK_SIZE {
|
||||
// A short first chunk is final; empty is the empty file.
|
||||
let mut p = self.open(raw, true)?;
|
||||
if let Err(e) = expect_eof(&mut self.src) {
|
||||
p.zeroize();
|
||||
return Err(e);
|
||||
}
|
||||
self.deliver(p);
|
||||
self.done = true;
|
||||
return Ok(());
|
||||
}
|
||||
self.pending = Some(raw);
|
||||
}
|
||||
let pending = self.pending.take().expect("pending chunk");
|
||||
let next = self.read_raw()?;
|
||||
if next.is_empty() {
|
||||
// Nothing follows: a full-length final chunk, and end of input
|
||||
// has just been seen.
|
||||
let p = self.open(pending, true)?;
|
||||
self.deliver(p);
|
||||
self.done = true;
|
||||
} else if next.len() < ENC_CHUNK_SIZE {
|
||||
if next.len() == TAG_SIZE {
|
||||
return Err(StreamError::EmptyLastChunk.into());
|
||||
}
|
||||
let mut p = self.open(pending, false)?;
|
||||
let mut q = match self.open(next, true) {
|
||||
Ok(q) => q,
|
||||
Err(e) => {
|
||||
p.zeroize();
|
||||
return Err(e);
|
||||
}
|
||||
};
|
||||
if let Err(e) = expect_eof(&mut self.src) {
|
||||
p.zeroize();
|
||||
q.zeroize();
|
||||
return Err(e);
|
||||
}
|
||||
self.deliver(p);
|
||||
self.queued = Some(q);
|
||||
self.done = true;
|
||||
} else {
|
||||
let p = self.open(pending, false)?;
|
||||
self.deliver(p);
|
||||
self.pending = Some(next);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for StreamReader<R> {
|
||||
fn read(&mut self, p: &mut [u8]) -> io::Result<usize> {
|
||||
if p.is_empty() {
|
||||
return Ok(0);
|
||||
}
|
||||
loop {
|
||||
if self.pos < self.out.len() {
|
||||
let n = p.len().min(self.out.len() - self.pos);
|
||||
p[..n].copy_from_slice(&self.out[self.pos..self.pos + n]);
|
||||
self.pos += n;
|
||||
return Ok(n);
|
||||
}
|
||||
if let Some(q) = self.queued.take() {
|
||||
self.deliver(q);
|
||||
continue;
|
||||
}
|
||||
if self.failed {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "encrypted stream already failed"));
|
||||
}
|
||||
if self.done {
|
||||
return Ok(0);
|
||||
}
|
||||
if let Err(e) = self.advance() {
|
||||
self.failed = true;
|
||||
self.out.zeroize();
|
||||
self.out.clear();
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Drop for StreamReader<R> {
|
||||
fn drop(&mut self) {
|
||||
self.out.zeroize();
|
||||
for b in self.pending.iter_mut().chain(self.queued.iter_mut()).chain(self.spare.iter_mut()) {
|
||||
b.zeroize();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Seals a whole plaintext held in memory, chunk by chunk, appending to
|
||||
/// `out`. Byte-identical to [`StreamWriter`].
|
||||
pub fn seal_all(key: &[u8; 32], plaintext: &[u8], out: &mut Vec<u8>) {
|
||||
let c = Cipher::new(key);
|
||||
let chunks = plaintext.len().div_ceil(CHUNK_SIZE).max(1);
|
||||
out.reserve(plaintext.len() + chunks * TAG_SIZE);
|
||||
for i in 0..chunks {
|
||||
let start = i * CHUNK_SIZE;
|
||||
let end = (start + CHUNK_SIZE).min(plaintext.len());
|
||||
let at = out.len();
|
||||
out.extend_from_slice(&plaintext[start..end]);
|
||||
out.resize(out.len() + TAG_SIZE, 0);
|
||||
c.seal(i as u64, i + 1 == chunks, &mut out[at..]);
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens a whole sealed payload in place. On success the plaintext is
|
||||
/// `buf[..returned]`; on any failure the buffer is cleared.
|
||||
pub fn open_all(key: &[u8; 32], buf: &mut [u8]) -> Result<usize, StreamError> {
|
||||
let r = open_all_inner(key, buf);
|
||||
if r.is_err() {
|
||||
buf.zeroize();
|
||||
}
|
||||
r
|
||||
}
|
||||
|
||||
fn open_all_inner(key: &[u8; 32], buf: &mut [u8]) -> Result<usize, StreamError> {
|
||||
let size = buf.len() as u64;
|
||||
let chunks = encrypted_chunk_count(size)?;
|
||||
let c = Cipher::new(key);
|
||||
let mut plain = 0usize;
|
||||
for i in 0..chunks {
|
||||
let start = i as usize * ENC_CHUNK_SIZE;
|
||||
let end = (start + ENC_CHUNK_SIZE).min(buf.len());
|
||||
let n = c.open(i, i + 1 == chunks, &mut buf[start..end])?;
|
||||
// Compact: plaintext of chunk i lands at i*CHUNK_SIZE <= start.
|
||||
buf.copy_within(start..start + n, plain);
|
||||
plain += n;
|
||||
}
|
||||
Ok(plain)
|
||||
}
|
||||
|
||||
/// The number of chunks in a payload of `size` bytes, or an error if no
|
||||
/// STREAM encoding has that size (Go's `EncryptedChunkCount`).
|
||||
pub fn encrypted_chunk_count(size: u64) -> Result<u64, StreamError> {
|
||||
let enc = ENC_CHUNK_SIZE as u64;
|
||||
let chunks = size.div_ceil(enc);
|
||||
let plain = size.checked_sub(chunks * TAG_SIZE as u64).ok_or(StreamError::InvalidSize(size))?;
|
||||
let expected = if plain == 0 { 1 } else { plain.div_ceil(CHUNK_SIZE as u64) };
|
||||
if expected != chunks {
|
||||
return Err(StreamError::InvalidSize(size));
|
||||
}
|
||||
Ok(chunks)
|
||||
}
|
||||
|
||||
pub fn plaintext_size(encrypted: u64) -> Result<u64, StreamError> {
|
||||
Ok(encrypted - encrypted_chunk_count(encrypted)? * TAG_SIZE as u64)
|
||||
}
|
||||
|
||||
/// Positional reads, as `std::os::unix::fs::FileExt::read_at`.
|
||||
pub trait ReadAt {
|
||||
fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize>;
|
||||
}
|
||||
|
||||
impl ReadAt for [u8] {
|
||||
fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
|
||||
let off = usize::try_from(offset).unwrap_or(usize::MAX).min(self.len());
|
||||
let n = buf.len().min(self.len() - off);
|
||||
buf[..n].copy_from_slice(&self[off..off + n]);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
impl ReadAt for std::fs::File {
|
||||
fn read_at(&self, buf: &mut [u8], offset: u64) -> io::Result<usize> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
std::os::unix::fs::FileExt::read_at(self, buf, offset)
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
std::os::windows::fs::FileExt::seek_read(self, buf, offset)
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
let _ = (buf, offset);
|
||||
Err(io::Error::new(io::ErrorKind::Unsupported, "positional reads unsupported"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_exact_at<S: ReadAt + ?Sized>(src: &S, buf: &mut [u8], mut off: u64) -> io::Result<()> {
|
||||
let mut done = 0;
|
||||
while done < buf.len() {
|
||||
match src.read_at(&mut buf[done..], off) {
|
||||
Ok(0) => return Err(io::ErrorKind::UnexpectedEof.into()),
|
||||
Ok(n) => {
|
||||
done += n;
|
||||
off += n as u64;
|
||||
}
|
||||
Err(e) if e.kind() == io::ErrorKind::Interrupted => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Random-access decryption (Go's `DecryptReaderAt`). Construction checks
|
||||
/// the size is a valid encoding and authenticates the final chunk, so a
|
||||
/// truncated or extended payload is rejected before any read.
|
||||
pub struct StreamReaderAt<'a, S: ReadAt + ?Sized> {
|
||||
cipher: Cipher,
|
||||
src: &'a S,
|
||||
size: u64,
|
||||
chunks: u64,
|
||||
cache: std::sync::Mutex<Option<(u64, Vec<u8>)>>,
|
||||
}
|
||||
|
||||
impl<'a, S: ReadAt + ?Sized> StreamReaderAt<'a, S> {
|
||||
pub fn new(key: &[u8; 32], src: &'a S, encrypted_size: u64) -> io::Result<Self> {
|
||||
let cipher = Cipher::new(key);
|
||||
let chunks = encrypted_chunk_count(encrypted_size)?;
|
||||
let last = chunks - 1;
|
||||
let off = last * ENC_CHUNK_SIZE as u64;
|
||||
let mut buf = vec![0u8; (encrypted_size - off) as usize];
|
||||
read_exact_at(src, &mut buf, off)?;
|
||||
let n = cipher.open(last, true, &mut buf).inspect_err(|_| buf.zeroize())?;
|
||||
buf.truncate(n);
|
||||
Ok(StreamReaderAt {
|
||||
cipher,
|
||||
src,
|
||||
size: encrypted_size - chunks * TAG_SIZE as u64,
|
||||
chunks,
|
||||
cache: std::sync::Mutex::new(Some((last, buf))),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn size(&self) -> u64 {
|
||||
self.size
|
||||
}
|
||||
|
||||
/// Fills as much of `p` as lies before the end of the plaintext and
|
||||
/// returns the count; reading at or past the end returns 0.
|
||||
pub fn read_at(&self, p: &mut [u8], mut off: u64) -> io::Result<usize> {
|
||||
if off > self.size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!("offset out of range [0:{}]: {off}", self.size),
|
||||
));
|
||||
}
|
||||
let mut n = 0;
|
||||
// Cleared on every exit, early returns included.
|
||||
let mut chunk = zeroize::Zeroizing::new(vec![0u8; ENC_CHUNK_SIZE]);
|
||||
let mut cache = self.cache.lock().unwrap_or_else(|e| e.into_inner());
|
||||
while n < p.len() && off < self.size {
|
||||
let idx = off / CHUNK_SIZE as u64;
|
||||
let plain: &[u8] = match &*cache {
|
||||
Some((i, data)) if *i == idx => data,
|
||||
_ => {
|
||||
let coff = idx * ENC_CHUNK_SIZE as u64;
|
||||
let enc = (self.size + self.chunks * TAG_SIZE as u64 - coff).min(ENC_CHUNK_SIZE as u64) as usize;
|
||||
read_exact_at(self.src, &mut chunk[..enc], coff)?;
|
||||
let k = self.cipher.open(idx, idx == self.chunks - 1, &mut chunk[..enc])?;
|
||||
if let Some((_, old)) = cache.as_mut() {
|
||||
old.zeroize();
|
||||
}
|
||||
*cache = Some((idx, chunk[..k].to_vec()));
|
||||
&cache.as_ref().expect("just set").1
|
||||
}
|
||||
};
|
||||
let within = (off - idx * CHUNK_SIZE as u64) as usize;
|
||||
let take = (plain.len() - within).min(p.len() - n);
|
||||
p[n..n + take].copy_from_slice(&plain[within..within + take]);
|
||||
n += take;
|
||||
off += take as u64;
|
||||
}
|
||||
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: ReadAt + ?Sized> Drop for StreamReaderAt<'_, S> {
|
||||
fn drop(&mut self) {
|
||||
if let Some((_, d)) = self.cache.get_mut().unwrap_or_else(|e| e.into_inner()).as_mut() {
|
||||
d.zeroize();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
|
||||
fn pt(n: usize) -> Vec<u8> {
|
||||
(0..n).map(|i| (i * 13 + 7) as u8).collect()
|
||||
}
|
||||
|
||||
fn seal(key: &[u8; 32], p: &[u8]) -> Vec<u8> {
|
||||
let mut w = StreamWriter::new(key, Vec::new());
|
||||
w.write_all(p).unwrap();
|
||||
w.finish().unwrap()
|
||||
}
|
||||
|
||||
fn open_stream(key: &[u8; 32], c: &[u8]) -> io::Result<Vec<u8>> {
|
||||
let mut out = Vec::new();
|
||||
StreamReader::new(key, c).read_to_end(&mut out)?;
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Vectors {
|
||||
key: String,
|
||||
vectors: Vec<V>,
|
||||
}
|
||||
#[derive(serde::Deserialize)]
|
||||
struct V {
|
||||
size: usize,
|
||||
ct_len: usize,
|
||||
ct_sha256: String,
|
||||
ct_hex: Option<String>,
|
||||
}
|
||||
|
||||
fn sha256(b: &[u8]) -> String {
|
||||
use sha2::Digest;
|
||||
hex(&sha2::Sha256::digest(b))
|
||||
}
|
||||
|
||||
pub(crate) fn hex(b: &[u8]) -> String {
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn go_oracle_vectors() {
|
||||
let v: Vectors = serde_json::from_str(include_str!("../../tests/vectors/stream.json")).unwrap();
|
||||
let key: [u8; 32] = (1..=32).collect::<Vec<u8>>().try_into().unwrap();
|
||||
assert_eq!(hex(&key), v.key);
|
||||
for t in v.vectors {
|
||||
let p = pt(t.size);
|
||||
let c = seal(&key, &p);
|
||||
assert_eq!(c.len(), t.ct_len, "size {}", t.size);
|
||||
assert_eq!(sha256(&c), t.ct_sha256, "size {}", t.size);
|
||||
if let Some(h) = t.ct_hex {
|
||||
assert_eq!(hex(&c), h);
|
||||
}
|
||||
let mut all = Vec::new();
|
||||
seal_all(&key, &p, &mut all);
|
||||
assert_eq!(all, c, "seal_all size {}", t.size);
|
||||
assert_eq!(open_stream(&key, &c).unwrap(), p);
|
||||
let n = open_all(&key, &mut all).unwrap();
|
||||
assert_eq!(&all[..n], &p[..]);
|
||||
assert_eq!(plaintext_size(c.len() as u64).unwrap(), t.size as u64);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn writes_in_any_steps_match() {
|
||||
let key = [3u8; 32];
|
||||
let p = pt(3 * CHUNK_SIZE + 5);
|
||||
let whole = seal(&key, &p);
|
||||
for step in [1, 7, 4096, CHUNK_SIZE - 1, CHUNK_SIZE, CHUNK_SIZE + 1] {
|
||||
let mut w = StreamWriter::new(&key, Vec::new());
|
||||
for c in p.chunks(step) {
|
||||
w.write_all(c).unwrap();
|
||||
}
|
||||
assert_eq!(w.finish().unwrap(), whole, "step {step}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_in_any_steps_match() {
|
||||
let key = [4u8; 32];
|
||||
for size in [0, 1, CHUNK_SIZE, CHUNK_SIZE + 1, 2 * CHUNK_SIZE, 2 * CHUNK_SIZE + 17] {
|
||||
let p = pt(size);
|
||||
let c = seal(&key, &p);
|
||||
for step in [1, 1000, CHUNK_SIZE, CHUNK_SIZE + 3] {
|
||||
let mut r = StreamReader::new(&key, &c[..]);
|
||||
let mut got = Vec::new();
|
||||
let mut b = vec![0u8; step];
|
||||
loop {
|
||||
let n = r.read(&mut b).unwrap();
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
got.extend_from_slice(&b[..n]);
|
||||
}
|
||||
assert_eq!(got, p, "size {size} step {step}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Every malformation is rejected by the streaming reader, the in-memory
|
||||
/// opener and (where the size is valid) the random-access reader alike.
|
||||
#[test]
|
||||
fn malformed_streams_rejected_consistently() {
|
||||
let key = [5u8; 32];
|
||||
let mut cases: Vec<(String, Vec<u8>)> = Vec::new();
|
||||
for size in [
|
||||
0,
|
||||
1,
|
||||
CHUNK_SIZE - 1,
|
||||
CHUNK_SIZE,
|
||||
CHUNK_SIZE + 1,
|
||||
2 * CHUNK_SIZE,
|
||||
2 * CHUNK_SIZE + 100,
|
||||
] {
|
||||
let c = seal(&key, &pt(size));
|
||||
cases.push((format!("{size} truncated by one"), c[..c.len() - 1].to_vec()));
|
||||
cases.push((format!("{size} trailing byte"), [&c[..], &[0]].concat()));
|
||||
cases.push((format!("{size} trailing chunk"), [&c[..], &c[..]].concat()));
|
||||
if c.len() > ENC_CHUNK_SIZE {
|
||||
cases.push((format!("{size} final chunk dropped"), c[..ENC_CHUNK_SIZE].to_vec()));
|
||||
let mut swapped = c.clone();
|
||||
swapped[..ENC_CHUNK_SIZE].copy_from_slice(&c[c.len() - ENC_CHUNK_SIZE..]);
|
||||
if swapped != c {
|
||||
cases.push((format!("{size} chunk swapped"), swapped));
|
||||
}
|
||||
}
|
||||
for at in [0, c.len() / 2, c.len() - 1] {
|
||||
let mut f = c.clone();
|
||||
f[at] ^= 0x80;
|
||||
cases.push((format!("{size} flipped at {at}"), f));
|
||||
}
|
||||
}
|
||||
cases.push(("empty".into(), Vec::new()));
|
||||
cases.push(("tag only, second chunk".into(), {
|
||||
// A full non-final chunk followed by an empty final chunk is
|
||||
// never produced and must be refused.
|
||||
let p = pt(CHUNK_SIZE);
|
||||
let c = Cipher::new(&key);
|
||||
let mut a = p.clone();
|
||||
a.resize(ENC_CHUNK_SIZE, 0);
|
||||
c.seal(0, false, &mut a);
|
||||
let mut b = vec![0u8; TAG_SIZE];
|
||||
c.seal(1, true, &mut b);
|
||||
[a, b].concat()
|
||||
}));
|
||||
for (name, c) in cases {
|
||||
assert!(open_stream(&key, &c).is_err(), "stream accepted: {name}");
|
||||
let mut b = c.clone();
|
||||
assert!(open_all(&key, &mut b).is_err(), "open_all accepted: {name}");
|
||||
assert!(b.iter().all(|&x| x == 0), "open_all left data behind: {name}");
|
||||
if let Ok(r) = StreamReaderAt::new(&key, &c[..], c.len() as u64) {
|
||||
let mut out = vec![0u8; r.size() as usize];
|
||||
assert!(r.read_at(&mut out, 0).is_err(), "reader-at accepted: {name}");
|
||||
}
|
||||
}
|
||||
let wrong = seal(&[6u8; 32], b"x");
|
||||
assert!(open_stream(&key, &wrong).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn random_access() {
|
||||
let key = [8u8; 32];
|
||||
for size in [0, 1, CHUNK_SIZE, CHUNK_SIZE + 1, 3 * CHUNK_SIZE - 5] {
|
||||
let p = pt(size);
|
||||
let c = seal(&key, &p);
|
||||
let r = StreamReaderAt::new(&key, &c[..], c.len() as u64).unwrap();
|
||||
assert_eq!(r.size(), size as u64);
|
||||
for (off, len) in [(0, size), (size / 2, size / 3 + 1), (size.saturating_sub(1), 10), (size, 4)] {
|
||||
let mut b = vec![0u8; len];
|
||||
let n = r.read_at(&mut b, off as u64).unwrap();
|
||||
let want = &p[off.min(size)..(off + len).min(size)];
|
||||
assert_eq!(&b[..n], want, "size {size} off {off} len {len}");
|
||||
}
|
||||
assert!(r.read_at(&mut [0u8; 1], size as u64 + 1).is_err());
|
||||
}
|
||||
// Concurrent readers share the cache safely.
|
||||
let p = pt(4 * CHUNK_SIZE);
|
||||
let c = seal(&key, &p);
|
||||
let r = StreamReaderAt::new(&key, &c[..], c.len() as u64).unwrap();
|
||||
std::thread::scope(|s| {
|
||||
for t in 0..8u64 {
|
||||
let (r, p) = (&r, &p);
|
||||
s.spawn(move || {
|
||||
for i in 0..200u64 {
|
||||
let off = ((i * 7919 + t * 104729) % p.len() as u64) as usize;
|
||||
let mut b = [0u8; 100];
|
||||
let n = r.read_at(&mut b, off as u64).unwrap();
|
||||
assert_eq!(&b[..n], &p[off..(off + 100).min(p.len())]);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
assert!(StreamReaderAt::new(&key, &c[..], c.len() as u64 - 1).is_err());
|
||||
assert!(encrypted_chunk_count(15).is_err());
|
||||
assert!(encrypted_chunk_count(ENC_CHUNK_SIZE as u64 + 16).is_err());
|
||||
}
|
||||
|
||||
/// After a failed write the writer refuses everything, as Go's does,
|
||||
/// instead of growing past a chunk.
|
||||
#[test]
|
||||
fn writer_failure_is_sticky() {
|
||||
struct Full;
|
||||
impl Write for Full {
|
||||
fn write(&mut self, _: &[u8]) -> io::Result<usize> {
|
||||
Err(io::Error::other("disk full"))
|
||||
}
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
let mut w = StreamWriter::new(&[1u8; 32], Full);
|
||||
assert!(w.write_all(&pt(CHUNK_SIZE + 1)).is_err());
|
||||
assert!(w.write_all(&pt(10)).is_err());
|
||||
assert!(w.flush().is_err());
|
||||
assert!(w.finish().is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counter_overflow_is_refused() {
|
||||
let mut w = StreamWriter::starting_at(&[1u8; 32], Vec::new(), u64::MAX);
|
||||
w.write_all(&pt(CHUNK_SIZE + 1)).unwrap_err();
|
||||
assert_eq!(next_index(u64::MAX), Err(StreamError::CounterOverflow));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The command-level error: a lower-case message without a trailing full
|
||||
//! stop, printed as `vault: MESSAGE`.
|
||||
|
||||
use std::fmt;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Error {
|
||||
pub msg: String,
|
||||
pub kind: Kind,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Kind {
|
||||
Other,
|
||||
/// A source ended before its declared size.
|
||||
SourceShort,
|
||||
/// A file was written under its final name, but a later durability step
|
||||
/// failed: what it depends on must not be deleted.
|
||||
AfterPublish,
|
||||
/// The thing to remove or read was not there.
|
||||
NotFound,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn new(msg: impl Into<String>) -> Error {
|
||||
Error {
|
||||
msg: msg.into(),
|
||||
kind: Kind::Other,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_kind(mut self, kind: Kind) -> Error {
|
||||
self.kind = kind;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn io(op: &str, path: &std::path::Path, e: &std::io::Error) -> Error {
|
||||
Error::new(crate::util::io_msg(op, path, e))
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.write_str(&self.msg)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for Error {}
|
||||
|
||||
impl From<String> for Error {
|
||||
fn from(s: String) -> Self {
|
||||
Error::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<&str> for Error {
|
||||
fn from(s: &str) -> Self {
|
||||
Error::new(s)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<std::io::Error> for Error {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
Error::new(crate::util::io_reason(&e))
|
||||
}
|
||||
}
|
||||
|
||||
pub type Result<T> = std::result::Result<T, Error>;
|
||||
|
||||
#[macro_export]
|
||||
macro_rules! bail {
|
||||
($($t:tt)*) => { return Err($crate::error::Error::new(format!($($t)*))) };
|
||||
}
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The passphrase-protected key file, byte-compatible with the Go version.
|
||||
//!
|
||||
//! A PEM block of type `VAULT KEY` holds 198 bytes: an 18-byte header
|
||||
//! (`VAULTKEY1`, Argon2id time 3 and memory 65536 KiB as big-endian u32s,
|
||||
//! and 4 lanes as one byte) and two 90-byte slots. A slot is a 16-byte
|
||||
//! salt, a 24-byte XChaCha20-Poly1305 nonce and the sealed big-endian suite
|
||||
//! ID and 32-byte master seed, with the header as associated data and the
|
||||
//! key Argon2id(passphrase, salt). The second slot is a decoy identity or
|
||||
//! random bytes, and which slot is real is random, so the file says nothing
|
||||
//! about whether a decoy exists. Unlocking always runs Argon2id for both
|
||||
//! slots, so timing does not reveal which one matched.
|
||||
|
||||
use crate::crypto::kem::{self, Identity};
|
||||
use crate::rng::Rng;
|
||||
use crate::secret::{SecretArray, SecretVec};
|
||||
use argon2::{Algorithm, Argon2, Block, Params, Version};
|
||||
use base64::Engine;
|
||||
use chacha20poly1305::aead::{AeadInOut, KeyInit};
|
||||
use chacha20poly1305::{Tag, XChaCha20Poly1305, XNonce};
|
||||
use std::io::Write;
|
||||
use std::path::Path;
|
||||
use zeroize::Zeroize;
|
||||
|
||||
pub const PEM_TYPE: &str = "VAULT KEY";
|
||||
const MAGIC: &[u8] = b"VAULTKEY1";
|
||||
const ARGON_TIME: u32 = 3;
|
||||
const ARGON_MEMORY: u32 = 64 * 1024; // KiB
|
||||
const ARGON_THREADS: u8 = 4;
|
||||
const SALT: usize = 16;
|
||||
const XNONCE: usize = 24;
|
||||
const SLOT_PLAIN: usize = 2 + kem::SEED_SIZE;
|
||||
pub const SLOT_SIZE: usize = SALT + XNONCE + SLOT_PLAIN + 16;
|
||||
pub const HEADER_SIZE: usize = 18;
|
||||
pub const BODY_SIZE: usize = HEADER_SIZE + 2 * SLOT_SIZE;
|
||||
/// Key files are a few hundred bytes; anything this large is not one.
|
||||
const MAX_FILE: u64 = 64 << 10;
|
||||
|
||||
pub fn header() -> [u8; HEADER_SIZE] {
|
||||
let mut h = [0u8; HEADER_SIZE];
|
||||
h[..9].copy_from_slice(MAGIC);
|
||||
h[9..13].copy_from_slice(&ARGON_TIME.to_be_bytes());
|
||||
h[13..17].copy_from_slice(&ARGON_MEMORY.to_be_bytes());
|
||||
h[17] = ARGON_THREADS;
|
||||
h
|
||||
}
|
||||
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Error {
|
||||
NotAKeyFile,
|
||||
IncorrectPassphrase,
|
||||
Io(String),
|
||||
Invalid(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Error::NotAKeyFile => f.write_str("not a vault key file"),
|
||||
Error::IncorrectPassphrase => f.write_str("incorrect passphrase"),
|
||||
Error::Io(m) | Error::Invalid(m) => f.write_str(m),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Argon2id(passphrase, salt) with the file's fixed parameters. The
|
||||
/// passphrase bytes are used exactly as given: no normalisation.
|
||||
pub fn slot_key(passphrase: &[u8], salt: &[u8]) -> SecretArray<32> {
|
||||
let params = Params::new(ARGON_MEMORY, ARGON_TIME, ARGON_THREADS as u32, Some(32)).expect("valid parameters");
|
||||
let a = Argon2::new(Algorithm::Argon2id, Version::V0x13, params);
|
||||
let mut blocks = vec![Block::default(); a.params().block_count()];
|
||||
let mut key = SecretArray::<32>::zero();
|
||||
let r = a.hash_password_into_with_memory(passphrase, salt, key.expose_mut(), &mut blocks[..]);
|
||||
// The working memory is derived from the passphrase; clear all 64 MiB.
|
||||
for b in blocks.iter_mut() {
|
||||
b.zeroize();
|
||||
}
|
||||
r.expect("argon2 inputs within limits");
|
||||
key
|
||||
}
|
||||
|
||||
fn seal_slot(id: &Identity, passphrase: &[u8], rng: &mut Rng) -> Vec<u8> {
|
||||
let mut slot = vec![0u8; SALT + XNONCE];
|
||||
rng.fill(&mut slot);
|
||||
let key = slot_key(passphrase, &slot[..SALT]);
|
||||
let aead = XChaCha20Poly1305::new(key.expose().into());
|
||||
let mut plain = SecretVec::with_capacity(SLOT_PLAIN);
|
||||
plain.extend_within_capacity(&id.suite_id().to_be_bytes());
|
||||
plain.extend_within_capacity(id.seed());
|
||||
let nonce = XNonce::try_from(&slot[SALT..]).expect("nonce size");
|
||||
let tag = aead
|
||||
.encrypt_inout_detached(&nonce, &header(), plain.expose_mut().into())
|
||||
.expect("small message");
|
||||
slot.extend_from_slice(plain.expose());
|
||||
slot.extend_from_slice(&tag);
|
||||
slot
|
||||
}
|
||||
|
||||
/// Opens one slot, or None if the passphrase does not open it.
|
||||
fn open_slot(slot: &[u8], passphrase: &[u8]) -> Option<SecretVec> {
|
||||
let key = slot_key(passphrase, &slot[..SALT]);
|
||||
let aead = XChaCha20Poly1305::new(key.expose().into());
|
||||
let nonce = XNonce::try_from(&slot[SALT..SALT + XNONCE]).expect("nonce size");
|
||||
let ct = &slot[SALT + XNONCE..];
|
||||
let mut plain = SecretVec::zeroed(SLOT_PLAIN);
|
||||
plain.expose_mut().copy_from_slice(&ct[..SLOT_PLAIN]);
|
||||
let tag = Tag::try_from(&ct[SLOT_PLAIN..]).expect("tag size");
|
||||
aead.decrypt_inout_detached(&nonce, &header(), plain.expose_mut().into(), &tag)
|
||||
.ok()?;
|
||||
Some(plain)
|
||||
}
|
||||
|
||||
/// The PEM text for a key file body, exactly as Go's `pem.Encode` writes it.
|
||||
pub fn encode_pem(body: &[u8]) -> String {
|
||||
let b64 = base64::engine::general_purpose::STANDARD.encode(body);
|
||||
let mut s = format!("-----BEGIN {PEM_TYPE}-----\n");
|
||||
for line in b64.as_bytes().chunks(64) {
|
||||
s.push_str(std::str::from_utf8(line).expect("ascii"));
|
||||
s.push('\n');
|
||||
}
|
||||
s.push_str(&format!("-----END {PEM_TYPE}-----\n"));
|
||||
s
|
||||
}
|
||||
|
||||
/// The block body if `data` is one `VAULT KEY` PEM block, optionally with
|
||||
/// text before it and only whitespace after: a port of Go's `pem.Decode`
|
||||
/// followed by vault's checks on its result.
|
||||
pub fn decode_pem(data: &[u8]) -> Option<Vec<u8>> {
|
||||
let (typ, body, rest) = go_pem_decode(data)?;
|
||||
if typ != PEM_TYPE.as_bytes() || !go_trim_space(rest).is_empty() {
|
||||
return None;
|
||||
}
|
||||
Some(body)
|
||||
}
|
||||
|
||||
/// Go's `getLine`: up to the next LF (dropping one CR before it), with
|
||||
/// trailing spaces and tabs removed, and the rest after the LF.
|
||||
fn get_line(data: &[u8]) -> (&[u8], &[u8]) {
|
||||
let (mut i, j) = match data.iter().position(|&c| c == b'\n') {
|
||||
Some(i) => (i, i + 1),
|
||||
None => (data.len(), data.len()),
|
||||
};
|
||||
if j > i && i > 0 && data[i - 1] == b'\r' {
|
||||
i -= 1;
|
||||
}
|
||||
let mut line = &data[..i];
|
||||
while let [rest @ .., b' ' | b'\t'] = line {
|
||||
line = rest;
|
||||
}
|
||||
(line, &data[j..])
|
||||
}
|
||||
|
||||
/// Go's `bytes.TrimSpace` (Unicode white space in UTF-8 text).
|
||||
fn go_trim_space(b: &[u8]) -> &[u8] {
|
||||
match std::str::from_utf8(b) {
|
||||
Ok(s) => s.trim_matches(char::is_whitespace).as_bytes(),
|
||||
Err(_) => b.trim_ascii(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `encoding/pem.Decode`: the first well-formed block's type, bytes
|
||||
/// and the data after it. Malformed candidates are skipped, as in Go.
|
||||
fn go_pem_decode(data: &[u8]) -> Option<(Vec<u8>, Vec<u8>, &[u8])> {
|
||||
const START: &[u8] = b"\n-----BEGIN ";
|
||||
const END: &[u8] = b"\n-----END ";
|
||||
const EOL: &[u8] = b"-----";
|
||||
let mut rest = data;
|
||||
loop {
|
||||
if rest.starts_with(&START[1..]) {
|
||||
rest = &rest[START.len() - 1..];
|
||||
} else {
|
||||
let i = find(rest, START)?;
|
||||
rest = &rest[i + START.len()..];
|
||||
}
|
||||
let (type_line, r) = get_line(rest);
|
||||
rest = r;
|
||||
let Some(typ) = type_line.strip_suffix(EOL) else { continue };
|
||||
let mut headers = 0;
|
||||
loop {
|
||||
if rest.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (line, next) = get_line(rest);
|
||||
if !line.contains(&b':') {
|
||||
break;
|
||||
}
|
||||
headers += 1;
|
||||
rest = next;
|
||||
}
|
||||
let (end_index, trailer_index) = if headers == 0 && rest.starts_with(&END[1..]) {
|
||||
(0, END.len() - 1)
|
||||
} else {
|
||||
match find(rest, END) {
|
||||
Some(i) => (i, i + END.len()),
|
||||
None => continue,
|
||||
}
|
||||
};
|
||||
let trailer = &rest[trailer_index..];
|
||||
let tlen = typ.len() + EOL.len();
|
||||
if trailer.len() < tlen {
|
||||
continue;
|
||||
}
|
||||
let (t, rest_of_end) = trailer.split_at(tlen);
|
||||
if !t.starts_with(typ) || !t.ends_with(EOL) || !get_line(rest_of_end).0.is_empty() {
|
||||
continue;
|
||||
}
|
||||
// Go's StdEncoding skips CR and LF; removeSpacesAndTabs drops the rest.
|
||||
let b64: Vec<u8> = rest[..end_index]
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|c| !matches!(c, b' ' | b'\t' | b'\r' | b'\n'))
|
||||
.collect();
|
||||
let lenient = base64::engine::GeneralPurpose::new(
|
||||
&base64::alphabet::STANDARD,
|
||||
base64::engine::GeneralPurposeConfig::new().with_decode_allow_trailing_bits(true),
|
||||
);
|
||||
let Ok(bytes) = lenient.decode(&b64) else { continue };
|
||||
let after = get_line(&rest[end_index + END.len() - 1..]).1;
|
||||
return Some((typ.to_vec(), bytes, after));
|
||||
}
|
||||
}
|
||||
|
||||
fn find(h: &[u8], n: &[u8]) -> Option<usize> {
|
||||
h.windows(n.len()).position(|w| w == n)
|
||||
}
|
||||
|
||||
/// Builds a key file body: one slot for `passphrase`, the other for
|
||||
/// `decoy` or random bytes if it is empty, in random order.
|
||||
pub fn new_body(passphrase: &[u8], decoy: &[u8], rng: &mut Rng) -> Vec<u8> {
|
||||
let real_slot = (rng.array::<1>()[0] & 1) as usize;
|
||||
let mut slots: [Vec<u8>; 2] = [Vec::new(), Vec::new()];
|
||||
for (i, pass) in [passphrase, decoy].into_iter().enumerate() {
|
||||
let idx = real_slot ^ i;
|
||||
if pass.is_empty() {
|
||||
let mut r = vec![0u8; SLOT_SIZE];
|
||||
rng.fill(&mut r);
|
||||
slots[idx] = r;
|
||||
continue;
|
||||
}
|
||||
let id = Identity::generate(rng);
|
||||
slots[idx] = seal_slot(&id, pass, rng);
|
||||
}
|
||||
let mut body = header().to_vec();
|
||||
body.extend_from_slice(&slots[0]);
|
||||
body.extend_from_slice(&slots[1]);
|
||||
body
|
||||
}
|
||||
|
||||
/// Creates a new key file at `path`, refusing to replace an existing one.
|
||||
pub fn write(path: &Path, passphrase: &[u8], decoy: &[u8]) -> Result<(), Error> {
|
||||
let body = new_body(passphrase, decoy, &mut crate::rng::os());
|
||||
let pem = encode_pem(&body);
|
||||
let ioerr = |op: &str, e: std::io::Error| Error::Io(crate::util::io_msg(op, path, &e));
|
||||
if let Some(parent) = path.parent().filter(|p| !p.as_os_str().is_empty()) {
|
||||
crate::platform::fs::create_private_dir_all(parent).map_err(|e| Error::Io(crate::util::io_msg("mkdir", parent, &e)))?;
|
||||
}
|
||||
let mut f = crate::platform::fs::create_new_private(path).map_err(|e| ioerr("open", e))?;
|
||||
if let Err(e) = f.write_all(pem.as_bytes()).and_then(|_| f.sync_all()) {
|
||||
drop(f);
|
||||
let _ = std::fs::remove_file(path);
|
||||
return Err(ioerr("write", e));
|
||||
}
|
||||
drop(f);
|
||||
if let Some(parent) = path.parent() {
|
||||
let _ = crate::platform::fs::sync_dir(if parent.as_os_str().is_empty() { Path::new(".") } else { parent });
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Checks a key file body's framing, returning its two slots.
|
||||
pub fn parse_body(body: &[u8]) -> Result<[&[u8]; 2], Error> {
|
||||
if body.len() != BODY_SIZE || body[..HEADER_SIZE] != header() {
|
||||
return Err(Error::NotAKeyFile);
|
||||
}
|
||||
let s = &body[HEADER_SIZE..];
|
||||
Ok([&s[..SLOT_SIZE], &s[SLOT_SIZE..]])
|
||||
}
|
||||
|
||||
/// The identity whose slot `passphrase` opens. Both slots are always tried.
|
||||
pub fn unlock_body(body: &[u8], passphrase: &[u8]) -> Result<Identity, Error> {
|
||||
let slots = parse_body(body)?;
|
||||
// Both Argon2id runs happen whatever the outcome, concurrently, so the
|
||||
// time taken does not depend on which slot (if any) matched.
|
||||
let opened: Vec<Option<SecretVec>> = std::thread::scope(|s| {
|
||||
let h: Vec<_> = slots.iter().map(|slot| s.spawn(move || open_slot(slot, passphrase))).collect();
|
||||
h.into_iter().map(|h| h.join().expect("argon2 thread")).collect()
|
||||
});
|
||||
if let Some(plain) = opened.into_iter().flatten().next() {
|
||||
let p = plain.expose();
|
||||
let suite = u16::from_be_bytes([p[0], p[1]]);
|
||||
return Identity::from_suite_seed(suite, &p[2..]).map_err(Error::Invalid);
|
||||
}
|
||||
Err(Error::IncorrectPassphrase)
|
||||
}
|
||||
|
||||
pub fn read_body(path: &Path) -> Result<Vec<u8>, Error> {
|
||||
use std::io::Read;
|
||||
let f = std::fs::File::open(path).map_err(|e| Error::Io(crate::util::io_msg("open", path, &e)))?;
|
||||
let mut data = Vec::new();
|
||||
f.take(MAX_FILE + 1)
|
||||
.read_to_end(&mut data)
|
||||
.map_err(|e| Error::Io(crate::util::io_msg("read", path, &e)))?;
|
||||
if data.len() as u64 > MAX_FILE {
|
||||
return Err(Error::NotAKeyFile);
|
||||
}
|
||||
let body = decode_pem(&data).ok_or(Error::NotAKeyFile)?;
|
||||
parse_body(&body)?;
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn unlock(path: &Path, passphrase: &[u8]) -> Result<Identity, Error> {
|
||||
unlock_body(&read_body(path)?, passphrase)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::crypto::kem::tests::unhex;
|
||||
use crate::crypto::stream::tests::hex;
|
||||
use serde_json::Value;
|
||||
|
||||
fn store_vectors() -> Value {
|
||||
serde_json::from_str(include_str!("../tests/vectors/store.json")).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn argon2id_matches_go() {
|
||||
let v = store_vectors();
|
||||
for c in v["argon2id"].as_array().unwrap() {
|
||||
let k = slot_key(&unhex(c["passphrase_hex"].as_str().unwrap()), &unhex(c["salt"].as_str().unwrap()));
|
||||
assert_eq!(hex(k.expose()), c["key"].as_str().unwrap());
|
||||
}
|
||||
assert_eq!(hex(&header()), v["key_header_hex"].as_str().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opens_go_key_files() {
|
||||
let v = store_vectors();
|
||||
for (name, slots) in v["keys"].as_object().unwrap() {
|
||||
let path = format!("{}/tests/vectors/{name}", env!("CARGO_MANIFEST_DIR"));
|
||||
let body = read_body(Path::new(&path)).unwrap();
|
||||
assert_eq!(body.len(), 198);
|
||||
for (_, s) in slots.as_object().unwrap() {
|
||||
let id = unlock_body(&body, s["passphrase"].as_str().unwrap().as_bytes()).unwrap();
|
||||
assert_eq!(hex(id.seed()), s["seed"].as_str().unwrap());
|
||||
assert_eq!(id.suite_id() as u64, s["suite"].as_u64().unwrap());
|
||||
}
|
||||
assert_eq!(unlock_body(&body, b"wrong").err(), Some(Error::IncorrectPassphrase));
|
||||
// Re-encoding reproduces Go's PEM text exactly.
|
||||
assert_eq!(encode_pem(&body).as_bytes(), &std::fs::read(&path).unwrap()[..]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn new_key_files_round_trip() {
|
||||
let body = new_body(b"real", b"decoy", &mut crate::rng::os());
|
||||
let a = unlock_body(&body, b"real").unwrap();
|
||||
let b = unlock_body(&body, b"decoy").unwrap();
|
||||
assert_ne!(a.seed(), b.seed());
|
||||
assert!(unlock_body(&body, b"other").is_err());
|
||||
let solo = new_body(b"real", b"", &mut crate::rng::os());
|
||||
assert_eq!(solo.len(), body.len());
|
||||
assert!(unlock_body(&solo, b"").is_err());
|
||||
// Passphrases are bytes: no normalisation of composed/decomposed é.
|
||||
let nfc = "caf\u{e9}".as_bytes();
|
||||
let nfd = "cafe\u{301}".as_bytes();
|
||||
let k = new_body(nfc, b"", &mut crate::rng::os());
|
||||
assert!(unlock_body(&k, nfc).is_ok());
|
||||
assert!(unlock_body(&k, nfd).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_key_files() {
|
||||
let body = new_body(b"p", b"", &mut crate::rng::os());
|
||||
let pem = encode_pem(&body);
|
||||
assert_eq!(decode_pem(pem.as_bytes()).unwrap(), body);
|
||||
assert_eq!(decode_pem(format!("junk\n{pem}\n \n").as_bytes()).unwrap(), body);
|
||||
assert!(decode_pem(format!("{pem}x").as_bytes()).is_none());
|
||||
assert!(decode_pem(pem.replace("VAULT KEY", "VAULT KEX").as_bytes()).is_none());
|
||||
assert!(decode_pem(pem.replace("-----END", "-----FIN").as_bytes()).is_none());
|
||||
// Cases where an earlier, simpler parser differed from Go's pem.Decode.
|
||||
// (In the second body line: a colon in the first makes it a header.)
|
||||
let first = pem.find('\n').unwrap();
|
||||
let mid = first + 1 + pem[first + 1..].find('\n').unwrap() + 30;
|
||||
assert!(
|
||||
decode_pem(format!("{}:{}", &pem[..mid], &pem[mid..]).as_bytes()).is_none(),
|
||||
"colon inside the body"
|
||||
);
|
||||
assert!(
|
||||
decode_pem(format!("{pem}\u{b}\u{a0}").as_bytes()).is_some(),
|
||||
"Unicode space after the block"
|
||||
);
|
||||
assert!(decode_pem(pem.replacen("\n", "\r\n", 3).as_bytes()).is_some(), "CRLF lines");
|
||||
assert!(
|
||||
decode_pem(format!("-----BEGIN X\n{pem}").as_bytes()).is_some(),
|
||||
"a bad BEGIN line before the block"
|
||||
);
|
||||
assert!(
|
||||
decode_pem(format!("-----BEGIN VAULT KEY-----\nProc-Type: 4,x\n\n{}", &pem[26..]).as_bytes()).is_some(),
|
||||
"headers"
|
||||
);
|
||||
let mut short = body.clone();
|
||||
short.pop();
|
||||
assert_eq!(parse_body(&short).err(), Some(Error::NotAKeyFile));
|
||||
let mut wrong = body.clone();
|
||||
wrong[17] = 1; // a different Argon2 lane count
|
||||
assert_eq!(parse_body(&wrong).err(), Some(Error::NotAKeyFile));
|
||||
let mut tampered = body.clone();
|
||||
tampered[HEADER_SIZE + SALT + XNONCE] ^= 1;
|
||||
tampered[HEADER_SIZE + SLOT_SIZE + SALT + XNONCE] ^= 1;
|
||||
assert_eq!(unlock_body(&tampered, b"p").err(), Some(Error::IncorrectPassphrase));
|
||||
}
|
||||
}
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! vault: a post-quantum encrypted file vault. The binary is a thin wrapper
|
||||
//! around [`cli::main`]; the library exists so integration tests and
|
||||
//! benchmarks can reach the same code.
|
||||
|
||||
#![deny(unsafe_op_in_unsafe_fn)]
|
||||
|
||||
pub mod absorb;
|
||||
pub mod agent;
|
||||
pub mod cli;
|
||||
pub mod crypto;
|
||||
pub mod error;
|
||||
pub mod keyfile;
|
||||
pub mod net;
|
||||
pub mod pipeline;
|
||||
pub mod platform;
|
||||
pub mod rng;
|
||||
pub mod secret;
|
||||
pub mod store;
|
||||
#[cfg(test)]
|
||||
mod testutil;
|
||||
pub mod util;
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! vault: a post-quantum encrypted file vault.
|
||||
|
||||
fn main() {
|
||||
std::process::exit(vault::cli::main());
|
||||
}
|
||||
+503
@@ -0,0 +1,503 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Link rewriting for mirrored HTML and CSS.
|
||||
//!
|
||||
//! HTML is scanned with a small tokenizer following the HTML5 tokenizer's
|
||||
//! rules for what matters here: tag and attribute syntax (quoted, unquoted
|
||||
//! and valueless attributes), comments, doctype and other `<!`/`<?`
|
||||
//! markup, raw-text elements (script, style, xmp, iframe, noembed,
|
||||
//! noframes), RCDATA (textarea, title) and plaintext. It records byte
|
||||
//! spans, so rewriting splices new values in and leaves every other byte —
|
||||
//! quote style included — untouched.
|
||||
//!
|
||||
//! Rewritten: the values of `href`, `src`, `action` and `poster` on start
|
||||
//! tags, and CSS `url(...)` inside `<style>` elements and `style`
|
||||
//! attributes. CSS files get the `url(...)` rewrite only, with the same
|
||||
//! matching as the Go version's regular expression
|
||||
//! `(?i)(url\(\s*["']?)([^"')\s]+)`. Character references in attribute
|
||||
//! values are decoded before a URL is parsed: numeric ones and the named
|
||||
//! `amp`, `lt`, `gt`, `quot`, `apos` and `nbsp`; other named references
|
||||
//! are left as written.
|
||||
|
||||
/// Something the tokenizer found.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Item {
|
||||
/// An attribute value on a start tag: lower-case tag and attribute
|
||||
/// names, the value's byte span, and whether it was quoted.
|
||||
Attr {
|
||||
tag: String,
|
||||
name: String,
|
||||
start: usize,
|
||||
end: usize,
|
||||
quoted: bool,
|
||||
},
|
||||
/// The raw text inside a `<style>` element.
|
||||
Style { start: usize, end: usize },
|
||||
}
|
||||
|
||||
fn is_ws(c: u8) -> bool {
|
||||
matches!(c, b'\t' | b'\n' | b'\x0c' | b'\r' | b' ')
|
||||
}
|
||||
|
||||
fn find(h: &[u8], from: usize, n: &[u8]) -> Option<usize> {
|
||||
if from > h.len() {
|
||||
return None;
|
||||
}
|
||||
h[from..].windows(n.len()).position(|w| w == n).map(|i| i + from)
|
||||
}
|
||||
|
||||
/// Scans `b` and returns every attribute value and style element, in
|
||||
/// document order.
|
||||
pub fn tokenize(b: &[u8]) -> Vec<Item> {
|
||||
let mut items = Vec::new();
|
||||
let mut pos = 0;
|
||||
while let Some(off) = b[pos..].iter().position(|&c| c == b'<') {
|
||||
let i = pos + off;
|
||||
let next = b.get(i + 1).copied();
|
||||
pos = match next {
|
||||
Some(b'!') => {
|
||||
if b[i..].starts_with(b"<!--") {
|
||||
let body = i + 4;
|
||||
if b[body..].starts_with(b">") {
|
||||
body + 1
|
||||
} else if b[body..].starts_with(b"->") {
|
||||
body + 2
|
||||
} else {
|
||||
find(b, body, b"-->").map_or(b.len(), |e| e + 3)
|
||||
}
|
||||
} else {
|
||||
to_gt(b, i + 2)
|
||||
}
|
||||
}
|
||||
Some(b'?') => to_gt(b, i + 2),
|
||||
Some(b'/') => match b.get(i + 2) {
|
||||
Some(c) if c.is_ascii_alphabetic() => {
|
||||
let (_, after) = tag(b, i + 2, &mut Vec::new(), "");
|
||||
after
|
||||
}
|
||||
Some(b'>') => i + 3,
|
||||
Some(_) => to_gt(b, i + 2),
|
||||
None => b.len(),
|
||||
},
|
||||
Some(c) if c.is_ascii_alphabetic() => {
|
||||
let (name, after) = tag(b, i + 1, &mut items, "start");
|
||||
match name.as_str() {
|
||||
"script" | "style" | "xmp" | "iframe" | "noembed" | "noframes" | "textarea" | "title" => {
|
||||
let end = raw_text_end(b, after, &name);
|
||||
if name == "style" {
|
||||
items.push(Item::Style { start: after, end });
|
||||
}
|
||||
end
|
||||
}
|
||||
"plaintext" => b.len(),
|
||||
_ => after,
|
||||
}
|
||||
}
|
||||
_ => i + 1,
|
||||
};
|
||||
if pos >= b.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
items
|
||||
}
|
||||
|
||||
fn to_gt(b: &[u8], from: usize) -> usize {
|
||||
b[from.min(b.len())..]
|
||||
.iter()
|
||||
.position(|&c| c == b'>')
|
||||
.map_or(b.len(), |e| from + e + 1)
|
||||
}
|
||||
|
||||
/// Where raw text started at `from` ends: at `</name` followed by
|
||||
/// whitespace, `/` or `>` (ASCII case-insensitive), or at end of input.
|
||||
fn raw_text_end(b: &[u8], from: usize, name: &str) -> usize {
|
||||
let mut p = from;
|
||||
while let Some(i) = find(b, p, b"</") {
|
||||
let n = i + 2;
|
||||
let candidate = b.get(n..n + name.len());
|
||||
if candidate.is_some_and(|c| c.eq_ignore_ascii_case(name.as_bytes())) {
|
||||
match b.get(n + name.len()) {
|
||||
Some(&c) if is_ws(c) || c == b'/' || c == b'>' => return i,
|
||||
None => return i,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
p = i + 2;
|
||||
}
|
||||
b.len()
|
||||
}
|
||||
|
||||
/// Parses a tag whose name starts at `from`; records start-tag attributes
|
||||
/// (when `kind` is "start") and returns the lower-case name and the index
|
||||
/// just past the closing `>` (or the end of input).
|
||||
fn tag(b: &[u8], from: usize, items: &mut Vec<Item>, kind: &str) -> (String, usize) {
|
||||
let mut attrs = Vec::new();
|
||||
let (name, end, closed) = tag_inner(b, from, &mut attrs);
|
||||
// A tag cut off by the end of input is not a tag at all.
|
||||
if closed && kind == "start" {
|
||||
items.extend(attrs);
|
||||
}
|
||||
(name, end)
|
||||
}
|
||||
|
||||
fn tag_inner(b: &[u8], from: usize, items: &mut Vec<Item>) -> (String, usize, bool) {
|
||||
let mut p = from;
|
||||
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' {
|
||||
p += 1;
|
||||
}
|
||||
let name = String::from_utf8_lossy(&b[from..p]).to_ascii_lowercase();
|
||||
loop {
|
||||
while p < b.len() && (is_ws(b[p]) || b[p] == b'/') {
|
||||
p += 1;
|
||||
}
|
||||
if p >= b.len() {
|
||||
return (name, b.len(), false);
|
||||
}
|
||||
if b[p] == b'>' {
|
||||
return (name, p + 1, true);
|
||||
}
|
||||
// Attribute name: the first character may be anything (even '='),
|
||||
// then up to whitespace, '/', '>' or '='.
|
||||
let ns = p;
|
||||
p += 1;
|
||||
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' && b[p] != b'=' {
|
||||
p += 1;
|
||||
}
|
||||
let attr = String::from_utf8_lossy(&b[ns..p]).to_ascii_lowercase();
|
||||
let mut q = p;
|
||||
while q < b.len() && is_ws(b[q]) {
|
||||
q += 1;
|
||||
}
|
||||
if q >= b.len() || b[q] != b'=' {
|
||||
p = q;
|
||||
continue;
|
||||
}
|
||||
q += 1;
|
||||
while q < b.len() && is_ws(b[q]) {
|
||||
q += 1;
|
||||
}
|
||||
if q >= b.len() {
|
||||
return (name, b.len(), false);
|
||||
}
|
||||
let (start, end, quoted, next) = match b[q] {
|
||||
quote @ (b'"' | b'\'') => {
|
||||
let s = q + 1;
|
||||
match b[s..].iter().position(|&c| c == quote) {
|
||||
Some(e) => (s, s + e, true, s + e + 1),
|
||||
// An unterminated value runs to the end; the tag is
|
||||
// incomplete, so nothing is recorded.
|
||||
None => return (name, b.len(), false),
|
||||
}
|
||||
}
|
||||
b'>' => (q, q, false, q),
|
||||
_ => {
|
||||
let mut e = q;
|
||||
while e < b.len() && !is_ws(b[e]) && b[e] != b'>' {
|
||||
e += 1;
|
||||
}
|
||||
(q, e, false, e)
|
||||
}
|
||||
};
|
||||
items.push(Item::Attr {
|
||||
tag: name.clone(),
|
||||
name: attr,
|
||||
start,
|
||||
end,
|
||||
quoted,
|
||||
});
|
||||
p = next;
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes character references (see the module documentation).
|
||||
pub fn unescape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut rest = s;
|
||||
while let Some(i) = rest.find('&') {
|
||||
out.push_str(&rest[..i]);
|
||||
rest = &rest[i..];
|
||||
let (text, used) = reference(rest);
|
||||
out.push_str(&text);
|
||||
rest = &rest[used..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
/// One reference at the start of `s` (which begins with '&'): its text and
|
||||
/// how many bytes it used; an unknown one is passed through as "&".
|
||||
fn reference(s: &str) -> (String, usize) {
|
||||
let b = s.as_bytes();
|
||||
if b.get(1) == Some(&b'#') {
|
||||
let (hex, start) = if matches!(b.get(2), Some(b'x' | b'X')) {
|
||||
(true, 3)
|
||||
} else {
|
||||
(false, 2)
|
||||
};
|
||||
let mut e = start;
|
||||
while e < b.len() && (if hex { b[e].is_ascii_hexdigit() } else { b[e].is_ascii_digit() }) {
|
||||
e += 1;
|
||||
}
|
||||
if e == start {
|
||||
return ("&".into(), 1);
|
||||
}
|
||||
let v = u32::from_str_radix(&s[start..e], if hex { 16 } else { 10 }).unwrap_or(0xfffd);
|
||||
let c = char::from_u32(v).filter(|&c| c != '\0').unwrap_or('\u{fffd}');
|
||||
let used = if b.get(e) == Some(&b';') { e + 1 } else { e };
|
||||
return (c.to_string(), used);
|
||||
}
|
||||
for (name, text, legacy) in [
|
||||
("amp", "&", true),
|
||||
("lt", "<", true),
|
||||
("gt", ">", true),
|
||||
("quot", "\"", true),
|
||||
("apos", "'", false),
|
||||
("nbsp", "\u{a0}", true),
|
||||
] {
|
||||
if let Some(r) = s[1..].strip_prefix(name) {
|
||||
if r.starts_with(';') {
|
||||
return (text.into(), name.len() + 2);
|
||||
}
|
||||
if legacy {
|
||||
return (text.into(), name.len() + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
("&".into(), 1)
|
||||
}
|
||||
|
||||
/// Escapes a value for an attribute, as Go's `html.EscapeString`.
|
||||
pub fn escape_attr(s: &str) -> String {
|
||||
let mut o = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => o.push_str("&"),
|
||||
'"' => o.push_str("""),
|
||||
'\'' => o.push_str("'"),
|
||||
'<' => o.push_str("<"),
|
||||
'>' => o.push_str(">"),
|
||||
c => o.push(c),
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// Rewrites `url(...)` references in CSS. `map` gets each URL (HTML-
|
||||
/// unescaped, as the Go version did) and returns its replacement, or None
|
||||
/// to leave it; replacements are attribute-escaped when `in_attr`. Bytes
|
||||
/// that are not UTF-8 pass through unchanged.
|
||||
pub fn rewrite_css(b: &[u8], in_attr: bool, map: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(b.len());
|
||||
let mut last = 0;
|
||||
let mut i = 0;
|
||||
while i + 4 <= b.len() {
|
||||
if !b[i..i + 4].eq_ignore_ascii_case(b"url(") {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let mut p = i + 4;
|
||||
while p < b.len() && is_ws(b[p]) {
|
||||
p += 1;
|
||||
}
|
||||
if p < b.len() && (b[p] == b'"' || b[p] == b'\'') {
|
||||
p += 1;
|
||||
} else if in_attr {
|
||||
p += entity_quote(&b[p..]);
|
||||
}
|
||||
let start = p;
|
||||
while p < b.len() && !is_ws(b[p]) && !matches!(b[p], b'"' | b'\'' | b')') && !(in_attr && entity_quote(&b[p..]) > 0) {
|
||||
p += 1;
|
||||
}
|
||||
if p == start {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(r) = map(&unescape(&String::from_utf8_lossy(&b[start..p]))) {
|
||||
out.extend_from_slice(&b[last..start]);
|
||||
out.extend_from_slice(if in_attr { escape_attr(&r) } else { r }.as_bytes());
|
||||
last = p;
|
||||
}
|
||||
i = p;
|
||||
}
|
||||
out.extend_from_slice(&b[last..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The length of a quote written as a character reference at the start of
|
||||
/// `b` (inside an attribute value, CSS quotes arrive that way), or 0.
|
||||
fn entity_quote(b: &[u8]) -> usize {
|
||||
[""", """, "'", "'", """, "'"]
|
||||
.iter()
|
||||
.find(|q| b.starts_with(q.as_bytes()))
|
||||
.map_or(0, |q| q.len())
|
||||
}
|
||||
|
||||
/// Rewrites an HTML page. `base` is called with the first `<base href>`
|
||||
/// value (decoded) before any link after it, and returns what that value
|
||||
/// becomes; `link` maps a decoded link to its replacement, or None to
|
||||
/// leave it. Every byte outside a replaced value is kept.
|
||||
pub fn rewrite_html(page: &[u8], base: &mut dyn FnMut(&str) -> Option<String>, link: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(page.len());
|
||||
let mut last = 0;
|
||||
let mut seen_base = false;
|
||||
for item in tokenize(page) {
|
||||
let (start, end, replacement) = match item {
|
||||
Item::Attr {
|
||||
tag,
|
||||
name,
|
||||
start,
|
||||
end,
|
||||
quoted,
|
||||
} => {
|
||||
let raw = &page[start..end];
|
||||
let value = || unescape(&String::from_utf8_lossy(raw));
|
||||
let r = match name.as_str() {
|
||||
"href" if tag == "base" && !seen_base => {
|
||||
seen_base = true;
|
||||
base(&value())
|
||||
}
|
||||
"href" | "src" | "action" | "poster" => link(&value()),
|
||||
"style" => {
|
||||
let r = rewrite_css(raw, true, link);
|
||||
if r != raw {
|
||||
out.extend_from_slice(&page[last..start]);
|
||||
out.extend_from_slice(&r);
|
||||
last = end;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let Some(r) = r else { continue };
|
||||
let mut v = escape_attr(&r);
|
||||
if !quoted && (v.is_empty() || v.bytes().any(|c| is_ws(c) || b"\"'=<>`".contains(&c))) {
|
||||
v = format!("\"{v}\"");
|
||||
}
|
||||
(start, end, v.into_bytes())
|
||||
}
|
||||
Item::Style { start, end } => (start, end, rewrite_css(&page[start..end], false, link)),
|
||||
};
|
||||
out.extend_from_slice(&page[last..start]);
|
||||
out.extend_from_slice(&replacement);
|
||||
last = end;
|
||||
}
|
||||
out.extend_from_slice(&page[last..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn upper(s: &str) -> Option<String> {
|
||||
Some(s.to_uppercase())
|
||||
}
|
||||
|
||||
fn rw(page: &str) -> String {
|
||||
String::from_utf8(rewrite_html(page.as_bytes(), &mut |_| None, &mut upper)).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attributes_and_quotes_preserved() {
|
||||
assert_eq!(
|
||||
rw(r#"<a href="x">x</a> <link href='y'> <img src=z alt=q>"#),
|
||||
r#"<a href="X">x</a> <link href='Y'> <img src=Z alt=q>"#
|
||||
);
|
||||
assert_eq!(rw(r#"<A HREF = "x" >"#), r#"<A HREF = "X" >"#);
|
||||
assert_eq!(rw(r#"<form action="/p?a=1&b=2">"#), r#"<form action="/P?A=1&B=2">"#);
|
||||
assert_eq!(
|
||||
rw(r#"<video poster="p.jpg" data-src="d">"#),
|
||||
r#"<video poster="P.JPG" data-src="d">"#
|
||||
);
|
||||
assert_eq!(rw("<p>href=\"x\"</p>"), "<p>href=\"x\"</p>", "text is not markup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_scripts_rcdata_untouched() {
|
||||
let s = r#"<!-- <a href="c"> --><script>var a = '<a href="s">';</script><textarea><a href="t"></textarea><title><img src=u></title><a href="ok">"#;
|
||||
assert_eq!(rw(s), s.replace("\"ok\"", "\"OK\""));
|
||||
assert_eq!(rw("<!--><a href=a>"), "<!--><a href=A>");
|
||||
assert_eq!(rw("<!DOCTYPE html><a href=a>"), "<!DOCTYPE html><a href=A>");
|
||||
assert_eq!(rw("<script>x</SCRIPT ><a href=a>"), "<script>x</SCRIPT ><a href=A>");
|
||||
assert_eq!(rw("<script>x</scripty><a href=a>"), "<script>x</scripty><a href=a>");
|
||||
assert_eq!(rw("<plaintext><a href=a>"), "<plaintext><a href=a>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn css_in_html_and_files() {
|
||||
assert_eq!(
|
||||
rw("<style>body{background:url( 'a.png' )}</style>"),
|
||||
"<style>body{background:url( 'A.PNG' )}</style>"
|
||||
);
|
||||
assert_eq!(
|
||||
rw(r#"<div style="background: URL("b.png")">"#),
|
||||
r#"<div style="background: URL("B.PNG")">"#
|
||||
);
|
||||
let mut m = |s: &str| Some(format!("<{s}>"));
|
||||
assert_eq!(
|
||||
rewrite_css(b"a{b:url(x)} c{d:url()} url(\"y\")", false, &mut m),
|
||||
b"a{b:url(<x>)} c{d:url()} url(\"<y>\")"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_href_and_escaping() {
|
||||
let mut seen = Vec::new();
|
||||
let out = rewrite_html(
|
||||
br#"<base href="http://h/d/"><base href="x"><a href=a>"#,
|
||||
&mut |b| {
|
||||
seen.push(b.to_string());
|
||||
Some("p.html".into())
|
||||
},
|
||||
&mut upper,
|
||||
);
|
||||
assert_eq!(seen, ["http://h/d/"]);
|
||||
assert_eq!(out, br#"<base href="p.html"><base href="X"><a href=A>"#);
|
||||
let out = rewrite_html(b"<a href=a>", &mut |_| None, &mut |_| Some("a b\"&".into()));
|
||||
assert_eq!(out, br#"<a href="a b"&">"#);
|
||||
// Non-UTF-8 bytes around and inside values survive.
|
||||
let latin1 = b"<p>caf\xe9</p><a href=\"x\xe9\">";
|
||||
assert_eq!(rewrite_html(latin1, &mut |_| None, &mut |_| None), latin1);
|
||||
assert_eq!(unescape("a&b//<&unknown;'&apos"), "a&b//<&unknown;'&apos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unterminated_input() {
|
||||
for s in ["<a href=\"x", "<a href=", "<a", "<", "<!--", "<script>", "</a", "<a href='x' "] {
|
||||
assert_eq!(rw(s), s, "{s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Random and mutated pages never panic, and an identity rewrite
|
||||
/// returns the input byte for byte.
|
||||
#[test]
|
||||
fn fuzz_identity() {
|
||||
let seeds = [
|
||||
r#"<html><head><base href="/x/"><style>a{b:url(c)}</style></head><body><a href='d'>e</a><img src=f style="g:url(h)"><!-- i --><script>j</script></body></html>"#,
|
||||
"<a href=\"x\"><b <c/> </d e='f'> <!DOCTYPE> <? pi ?> <textarea></textarea>",
|
||||
];
|
||||
let mut rng = crate::rng::os();
|
||||
let alphabet = b"<>/!-='\" \tabcdefhrsxyzlptu()&;#";
|
||||
for i in 0..20000 {
|
||||
let mut page: Vec<u8> = seeds[i % 2].as_bytes().to_vec();
|
||||
for _ in 0..rng.below(8) {
|
||||
let at = rng.below(page.len() as u64 + 1) as usize;
|
||||
match rng.below(3) {
|
||||
0 => page.insert(at, alphabet[rng.below(alphabet.len() as u64) as usize]),
|
||||
1 if at < page.len() => {
|
||||
page.remove(at);
|
||||
}
|
||||
_ => page.truncate(at),
|
||||
}
|
||||
}
|
||||
if i % 3 == 0 {
|
||||
page.push(0xff);
|
||||
}
|
||||
assert_eq!(rewrite_html(&page, &mut |_| None, &mut |_| None), page);
|
||||
// Rewriting everything must not panic either.
|
||||
let _ = rewrite_html(&page, &mut |b| Some(b.into()), &mut |v| Some(format!("{v}&x")));
|
||||
let _ = tokenize(&page);
|
||||
}
|
||||
}
|
||||
}
|
||||
+750
@@ -0,0 +1,750 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Network import: `add URL` and `mirror`.
|
||||
//!
|
||||
//! Every request goes through one explicit client: TLS certificates are
|
||||
//! always verified against the platform's roots, redirects are followed by
|
||||
//! hand (http and https only, at most [`Limits::max_redirects`], and for
|
||||
//! `mirror` only within the starting host), and every body is counted
|
||||
//! after decompression against a byte budget — `Content-Length` is never
|
||||
//! trusted. Proxies follow Go's `ProxyFromEnvironment`: `HTTPS_PROXY` for
|
||||
//! https, `HTTP_PROXY` for http (ignored under CGI, when `REQUEST_METHOD`
|
||||
//! is set), `NO_PROXY` exclusions, and never for localhost or loopback.
|
||||
//!
|
||||
//! Defaults (all overridable from the command line, the byte budget never
|
||||
//! above the 1 TiB file limit):
|
||||
//!
|
||||
//! | limit | `add URL` | `mirror` |
|
||||
//! |------------------------|-----------|-----------------|
|
||||
//! | connect | 30 s | 30 s |
|
||||
//! | response headers | 60 s | 60 s |
|
||||
//! | whole request | 2 h | 5 min each |
|
||||
//! | bytes | 16 GiB | 4 GiB in total |
|
||||
//! | redirects per request | 10 | 10 |
|
||||
//! | pages | — | 10 000 |
|
||||
//! | HTML/CSS held to rewrite | — | 16 MiB per page |
|
||||
//!
|
||||
//! Downloads that are not rewritten are streamed to a private (0600)
|
||||
//! temporary file in the temp directory, because the vault needs a file's
|
||||
//! size before storing it. On Unix the file is unlinked as soon as it is
|
||||
//! created, so its plaintext disappears with the process however it ends;
|
||||
//! elsewhere it is removed on every normal path. HTML and CSS for
|
||||
//! rewriting are held in memory instead.
|
||||
|
||||
pub mod html;
|
||||
pub mod url;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store::{Catalogue, FILE_LIMIT, Store, VaultName};
|
||||
use crate::util::Instant;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::io::{Read, Seek, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
pub use url::Url;
|
||||
|
||||
pub const CHROME_UA: &str =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Limits {
|
||||
pub connect: Duration,
|
||||
/// Until the response headers have arrived.
|
||||
pub response: Duration,
|
||||
/// End to end, body included, for each request.
|
||||
pub request: Duration,
|
||||
/// Body bytes: per download for `add`, in total for `mirror`.
|
||||
pub max_bytes: u64,
|
||||
pub max_redirects: u32,
|
||||
pub max_pages: u64,
|
||||
/// Largest HTML or CSS document held in memory for rewriting.
|
||||
pub max_text: u64,
|
||||
}
|
||||
|
||||
impl Limits {
|
||||
pub fn add_defaults() -> Limits {
|
||||
Limits {
|
||||
connect: Duration::from_secs(30),
|
||||
response: Duration::from_secs(60),
|
||||
request: Duration::from_secs(2 * 3600),
|
||||
max_bytes: 16 << 30,
|
||||
max_redirects: 10,
|
||||
max_pages: 1,
|
||||
max_text: 16 << 20,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mirror_defaults() -> Limits {
|
||||
Limits {
|
||||
request: Duration::from_secs(300),
|
||||
max_bytes: 4 << 30,
|
||||
max_pages: 10_000,
|
||||
..Limits::add_defaults()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_url(s: &str) -> bool {
|
||||
s.starts_with("https://") || s.starts_with("http://")
|
||||
}
|
||||
|
||||
/// Why a fetch failed: the byte budget ends a mirror, anything else only
|
||||
/// skips one resource.
|
||||
enum FetchError {
|
||||
Budget(u64),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FetchError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FetchError::Budget(n) => write!(f, "download exceeds {n} bytes"),
|
||||
FetchError::Other(m) => f.write_str(m),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for FetchError {
|
||||
fn from(s: String) -> Self {
|
||||
FetchError::Other(s)
|
||||
}
|
||||
}
|
||||
|
||||
type Response = ureq::http::Response<ureq::Body>;
|
||||
|
||||
struct Client {
|
||||
limits: Limits,
|
||||
agents: HashMap<Option<String>, ureq::Agent>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn new(limits: &Limits) -> Client {
|
||||
Client {
|
||||
limits: limits.clone(),
|
||||
agents: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn agent(&mut self, proxy: Option<String>) -> std::result::Result<&ureq::Agent, String> {
|
||||
if !self.agents.contains_key(&proxy) {
|
||||
let p = match &proxy {
|
||||
// The value may hold credentials: never echo it.
|
||||
Some(p) => Some(ureq::Proxy::new(p).map_err(|_| "invalid proxy setting in the environment".to_string())?),
|
||||
None => None,
|
||||
};
|
||||
let tls = ureq::tls::TlsConfig::builder()
|
||||
.root_certs(ureq::tls::RootCerts::PlatformVerifier)
|
||||
.build();
|
||||
let l = &self.limits;
|
||||
let cfg = ureq::Agent::config_builder()
|
||||
.tls_config(tls)
|
||||
.proxy(p)
|
||||
.max_redirects(0)
|
||||
.http_status_as_error(false)
|
||||
.user_agent(CHROME_UA)
|
||||
.timeout_connect(Some(l.connect))
|
||||
.timeout_recv_response(Some(l.response))
|
||||
.timeout_global(Some(l.request))
|
||||
.build();
|
||||
self.agents.insert(proxy.clone(), cfg.into());
|
||||
}
|
||||
Ok(&self.agents[&proxy])
|
||||
}
|
||||
|
||||
/// GETs `u`, following redirects; with `host`, every hop must stay on
|
||||
/// it. Returns the final URL and its response, whatever the status.
|
||||
fn get(&mut self, u: &Url, host: Option<&str>) -> std::result::Result<(Url, Response), String> {
|
||||
let mut cur = u.clone();
|
||||
for hop in 0..=self.limits.max_redirects {
|
||||
if cur.scheme != "http" && cur.scheme != "https" {
|
||||
return Err(format!("unsupported protocol scheme {:?}", cur.scheme));
|
||||
}
|
||||
let agent = self.agent(proxy_for(&cur))?;
|
||||
let resp = agent.get(cur.to_string()).call().map_err(ureq_reason)?;
|
||||
let code = resp.status().as_u16();
|
||||
let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_string);
|
||||
match (code, location) {
|
||||
(301 | 302 | 303 | 307 | 308, Some(loc)) => {
|
||||
if hop == self.limits.max_redirects {
|
||||
return Err(format!("stopped after {} redirects", self.limits.max_redirects));
|
||||
}
|
||||
cur = cur.join(&loc).map_err(|e| format!("bad redirect location {loc:?}: {e}"))?;
|
||||
if host.is_some_and(|h| cur.host.to_lowercase() != h) {
|
||||
return Err(format!("redirected to another host: {}", cur.redacted()));
|
||||
}
|
||||
}
|
||||
_ => return Ok((cur, resp)),
|
||||
}
|
||||
}
|
||||
unreachable!("the loop returns on its last hop")
|
||||
}
|
||||
}
|
||||
|
||||
/// A ureq error as a lower-case diagnostic.
|
||||
fn ureq_reason(e: ureq::Error) -> String {
|
||||
match e {
|
||||
ureq::Error::Io(io) => crate::util::io_reason(&io),
|
||||
e => {
|
||||
let s = e.to_string();
|
||||
let mut c = s.chars();
|
||||
c.next().map_or(s.clone(), |f| f.to_lowercase().chain(c).collect())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn status_error(resp: &Response) -> Option<String> {
|
||||
let s = resp.status();
|
||||
(s.as_u16() != 200).then(|| {
|
||||
format!("{} {}", s.as_u16(), s.canonical_reason().unwrap_or(""))
|
||||
.trim_end()
|
||||
.to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// Copies at most `budget` decoded body bytes to `dst`. Returns the bytes
|
||||
/// received, which count against budgets even when the copy fails.
|
||||
fn copy_body(resp: &mut Response, dst: &mut dyn Write, budget: u64) -> (u64, std::result::Result<(), FetchError>) {
|
||||
let mut r = resp.body_mut().with_config().limit(u64::MAX).reader();
|
||||
let mut buf = vec![0u8; 64 << 10];
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
let k = match r.read(&mut buf) {
|
||||
Ok(0) => return (n, Ok(())),
|
||||
Ok(k) => k,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return (n, Err(FetchError::Other(crate::util::io_reason(&e)))),
|
||||
};
|
||||
n += k as u64;
|
||||
if n > budget {
|
||||
return (n, Err(FetchError::Budget(budget)));
|
||||
}
|
||||
if let Err(e) = dst.write_all(&buf[..k]) {
|
||||
return (n, Err(FetchError::Other(crate::util::io_reason(&e))));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A private (0600) temporary file. On Unix its name is removed at once,
|
||||
/// so nothing is left behind even if the process is killed; elsewhere it
|
||||
/// is removed when dropped.
|
||||
struct TempFile {
|
||||
file: Option<std::fs::File>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl TempFile {
|
||||
fn new(prefix: &str) -> Result<TempFile> {
|
||||
let dir = std::env::temp_dir();
|
||||
loop {
|
||||
let r: [u8; 8] = crate::rng::os().array();
|
||||
let path = dir.join(format!("{prefix}{}", crate::store::hex(&r)));
|
||||
let mut o = std::fs::OpenOptions::new();
|
||||
o.read(true).write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
std::os::unix::fs::OpenOptionsExt::mode(&mut o, 0o600);
|
||||
match o.open(&path) {
|
||||
Ok(file) => {
|
||||
#[cfg(unix)]
|
||||
std::fs::remove_file(&path).map_err(|e| Error::io("remove", &path, &e))?;
|
||||
return Ok(TempFile { file: Some(file), path });
|
||||
}
|
||||
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => return Err(Error::io("create temporary file in", &dir, &e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn file(&mut self) -> &mut std::fs::File {
|
||||
self.file.as_mut().expect("open")
|
||||
}
|
||||
|
||||
/// Stores the `n` bytes written so far as `name`.
|
||||
fn store(mut self, store: &Store, cat: &mut Catalogue, n: u64, mtime: Instant, name: &VaultName) -> Result<crate::store::Added> {
|
||||
let f = self.file.as_mut().expect("open");
|
||||
f.rewind().map_err(|e| Error::io("seek", &self.path, &e))?;
|
||||
store.add_reader(cat, f, n, 0o600, mtime, name)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TempFile {
|
||||
fn drop(&mut self) {
|
||||
drop(self.file.take());
|
||||
#[cfg(not(unix))]
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn vault_name(dest: &str, name: &str) -> Result<VaultName> {
|
||||
let joined = crate::util::join(&[dest, name]);
|
||||
VaultName::new(&joined).ok_or_else(|| Error::new(format!("invalid vault path {joined:?}")))
|
||||
}
|
||||
|
||||
/// Downloads `url` and stores it beneath `dest`, named after the last
|
||||
/// element of the final (post-redirect) URL's path, or its host name.
|
||||
pub fn add_url(
|
||||
store: &Store,
|
||||
cat: &mut Catalogue,
|
||||
url: &str,
|
||||
dest: &str,
|
||||
limits: &Limits,
|
||||
report: &mut dyn FnMut(&VaultName, bool),
|
||||
) -> Result<()> {
|
||||
let u = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
let shown = u.redacted();
|
||||
let mut client = Client::new(limits);
|
||||
let (fin, mut resp) = client.get(&u, None).map_err(|e| Error::new(format!("{shown}: {e}")))?;
|
||||
if let Some(e) = status_error(&resp) {
|
||||
crate::bail!("{shown}: {e}");
|
||||
}
|
||||
let mut name = crate::util::base(&fin.path);
|
||||
// A path ending in "." or ".." (say from a redirect to /%2e%2e) must
|
||||
// not climb out of dest.
|
||||
if matches!(name.as_str(), "/" | "." | ".." | "") {
|
||||
name = fin.hostname().to_string();
|
||||
}
|
||||
let vn = vault_name(dest, &name)?;
|
||||
if !dest.is_empty() && !vn.as_str().starts_with(&format!("{dest}/")) {
|
||||
crate::bail!("{shown}: unsafe name {name:?}");
|
||||
}
|
||||
let mut tmp = TempFile::new("vault-download-")?;
|
||||
let (n, r) = copy_body(&mut resp, tmp.file(), limits.max_bytes.min(FILE_LIMIT));
|
||||
r.map_err(|e| Error::new(format!("{shown}: {e}")))?;
|
||||
let mtime = resp
|
||||
.headers()
|
||||
.get("last-modified")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(parse_http_time)
|
||||
.map_or_else(Instant::now, |secs| Instant { secs, nanos: 0 });
|
||||
let added = tmp.store(store, cat, n, mtime, &vn)?;
|
||||
report(&vn, added.replaced);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub enum MirrorOutcome {
|
||||
Complete { stored: u64 },
|
||||
Partial { stored: u64, skipped: u64 },
|
||||
}
|
||||
|
||||
/// Where `u` lives inside a mirror, relative to its root (Go's localPath).
|
||||
pub(crate) fn local_path(u: &Url) -> String {
|
||||
let mut p = u.path.strip_prefix('/').unwrap_or(&u.path).to_string();
|
||||
if p.is_empty() || p.ends_with('/') {
|
||||
p.push_str("index.html");
|
||||
}
|
||||
if !u.raw_query.is_empty() {
|
||||
p.push('_');
|
||||
p.push_str(&url::query_escape(&u.raw_query));
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// A page's identity, independent of its fragment (Go's key).
|
||||
/// Whether a local path stays inside the mirror: no ".", ".." or empty
|
||||
/// segments (a decoded `%2e%2e` would otherwise climb out).
|
||||
fn safe_local(local: &str) -> bool {
|
||||
!local.split('/').any(|s| s.is_empty() || s == "." || s == "..")
|
||||
}
|
||||
|
||||
pub(crate) fn key(u: &Url) -> String {
|
||||
let mut c = u.clone();
|
||||
c.clear_fragment();
|
||||
c.to_string()
|
||||
}
|
||||
|
||||
/// The link from the file `from` to the file `to`, both relative to the
|
||||
/// same root (Go's relative).
|
||||
pub(crate) fn relative(from: &str, to: &str) -> String {
|
||||
let dir = crate::util::clean(from.rsplit_once('/').map_or(".", |(d, _)| if d.is_empty() { "/" } else { d }));
|
||||
let mut f: Vec<&str> = dir.split('/').collect();
|
||||
let mut t: Vec<&str> = to.split('/').collect();
|
||||
if f[0] == "." {
|
||||
f.clear();
|
||||
}
|
||||
while !f.is_empty() && t.len() > 1 && f[0] == t[0] {
|
||||
f.remove(0);
|
||||
t.remove(0);
|
||||
}
|
||||
format!("{}{}", "../".repeat(f.len()), t.join("/"))
|
||||
}
|
||||
|
||||
struct Crawl<'a> {
|
||||
store: &'a Store,
|
||||
client: Client,
|
||||
host: String,
|
||||
root: String,
|
||||
seen: HashSet<String>,
|
||||
queue: VecDeque<Url>,
|
||||
owners: HashMap<String, String>,
|
||||
budget: u64,
|
||||
max_pages: u64,
|
||||
limit_hit: bool,
|
||||
max_text: u64,
|
||||
}
|
||||
|
||||
/// Crawls the site at `url`, one request at a time, staying on its host,
|
||||
/// and stores every page and asset beneath `dest/HOST`. Links in HTML and
|
||||
/// CSS that point into the mirror are rewritten to relative paths. A
|
||||
/// resource that fails is reported and skipped.
|
||||
pub fn mirror(
|
||||
store: &Store,
|
||||
cat: &mut Catalogue,
|
||||
url: &str,
|
||||
dest: &str,
|
||||
limits: &Limits,
|
||||
report: &mut dyn FnMut(&VaultName, bool),
|
||||
) -> Result<MirrorOutcome> {
|
||||
let mut start = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
if start.scheme != "http" && start.scheme != "https" {
|
||||
crate::bail!("mirror needs an http:// or https:// URL");
|
||||
}
|
||||
if start.path.is_empty() && start.raw_query.is_empty() {
|
||||
// The same page as a link to "/" would name.
|
||||
start = start.join("/").expect("valid reference");
|
||||
}
|
||||
let host = start.host.to_lowercase();
|
||||
let mut c = Crawl {
|
||||
store,
|
||||
client: Client::new(limits),
|
||||
root: crate::util::join(&[dest, &host]),
|
||||
host,
|
||||
seen: HashSet::new(),
|
||||
queue: VecDeque::new(),
|
||||
owners: HashMap::new(),
|
||||
budget: limits.max_bytes.min(FILE_LIMIT),
|
||||
max_pages: limits.max_pages,
|
||||
limit_hit: false,
|
||||
max_text: limits.max_text,
|
||||
};
|
||||
c.enqueue(start);
|
||||
let (mut stored, mut skipped) = (0u64, 0u64);
|
||||
while let Some(u) = c.queue.pop_front() {
|
||||
match c.one(cat, &u, report) {
|
||||
Ok(()) => stored += 1,
|
||||
Err(FetchError::Budget(n)) => {
|
||||
eprintln!("skipping {}: the mirror's {n}-byte budget is used up", u.redacted());
|
||||
skipped += 1 + c.queue.len() as u64;
|
||||
c.queue.clear();
|
||||
}
|
||||
Err(FetchError::Other(e)) => {
|
||||
eprintln!("skipping {}: {e}", u.redacted());
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.limit_hit {
|
||||
eprintln!("mirror: page limit of {} reached; further links were not followed", c.max_pages);
|
||||
skipped += 1;
|
||||
}
|
||||
Ok(if skipped == 0 {
|
||||
MirrorOutcome::Complete { stored }
|
||||
} else {
|
||||
MirrorOutcome::Partial { stored, skipped }
|
||||
})
|
||||
}
|
||||
|
||||
impl Crawl<'_> {
|
||||
fn enqueue(&mut self, u: Url) {
|
||||
let k = key(&u);
|
||||
if self.seen.contains(&k) {
|
||||
return;
|
||||
}
|
||||
if self.seen.len() as u64 >= self.max_pages {
|
||||
self.limit_hit = true;
|
||||
return;
|
||||
}
|
||||
self.seen.insert(k);
|
||||
self.queue.push_back(u);
|
||||
}
|
||||
|
||||
fn one(&mut self, cat: &mut Catalogue, u: &Url, report: &mut dyn FnMut(&VaultName, bool)) -> std::result::Result<(), FetchError> {
|
||||
let local = local_path(u);
|
||||
if !safe_local(&local) {
|
||||
return Err(format!("unsafe local name {local:?}").into());
|
||||
}
|
||||
let k = key(u);
|
||||
match self.owners.get(&local) {
|
||||
Some(owner) if *owner != k => return Err(format!("its local name {local} is already used by {owner}").into()),
|
||||
_ => {
|
||||
self.owners.insert(local.clone(), k);
|
||||
}
|
||||
}
|
||||
let name = vault_name(&self.root, &local).map_err(|e| e.msg)?;
|
||||
if !name.as_str().starts_with(&format!("{}/", self.root)) {
|
||||
return Err(format!("unsafe local name {local:?}").into());
|
||||
}
|
||||
let (fin, mut resp) = self.client.get(u, Some(&self.host))?;
|
||||
if let Some(e) = status_error(&resp) {
|
||||
return Err(e.into());
|
||||
}
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
let (is_html, is_css) = (ct.starts_with("text/html"), ct.starts_with("text/css"));
|
||||
let added = if is_html || is_css {
|
||||
let mut body = Vec::new();
|
||||
let cap = self.budget.min(self.max_text);
|
||||
let (n, r) = copy_body(&mut resp, &mut body, cap);
|
||||
self.budget = self.budget.saturating_sub(n);
|
||||
r.map_err(|e| match e {
|
||||
FetchError::Budget(_) if cap < self.budget + n => {
|
||||
FetchError::Other(format!("larger than the {}-byte limit for pages to rewrite", self.max_text))
|
||||
}
|
||||
e => e,
|
||||
})?;
|
||||
let body = self.rewrite(&body, is_html, &fin, &local)?;
|
||||
self.store
|
||||
.add_reader(cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name)
|
||||
.map_err(|e| FetchError::Other(e.msg))?
|
||||
} else {
|
||||
let mut tmp = TempFile::new("vault-mirror-").map_err(|e| FetchError::Other(e.msg))?;
|
||||
let (n, r) = copy_body(&mut resp, tmp.file(), self.budget);
|
||||
self.budget = self.budget.saturating_sub(n);
|
||||
r?;
|
||||
tmp.store(self.store, cat, n, Instant::now(), &name)
|
||||
.map_err(|e| FetchError::Other(e.msg))?
|
||||
};
|
||||
report(&name, added.replaced);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrites links that point into the mirror, queueing their targets.
|
||||
/// Refuses a page whose rewritten links would add more than four times
|
||||
/// the rewrite limit (deeply nested pages multiply every "../").
|
||||
fn rewrite(&mut self, body: &[u8], is_html: bool, page: &Url, local: &str) -> std::result::Result<Vec<u8>, FetchError> {
|
||||
let base = std::cell::RefCell::new(page.clone());
|
||||
let found = std::cell::RefCell::new((Vec::new(), HashSet::new()));
|
||||
let added = std::cell::Cell::new(0u64);
|
||||
let cap = self.max_text.saturating_mul(4);
|
||||
let host = self.host.clone();
|
||||
let mut link = |v: &str| -> Option<String> {
|
||||
let mut l = base.borrow().join(v).ok()?;
|
||||
if (l.scheme != "http" && l.scheme != "https") || l.host.to_lowercase() != host {
|
||||
return None;
|
||||
}
|
||||
let frag = l.fragment.clone();
|
||||
l.clear_fragment();
|
||||
if l.path.is_empty() && l.raw_query.is_empty() {
|
||||
l = l.join("/").ok()?;
|
||||
}
|
||||
let target = local_path(&l);
|
||||
{
|
||||
let (list, set) = &mut *found.borrow_mut();
|
||||
if set.insert(key(&l)) {
|
||||
// Queued even when unsafe, so it is reported as skipped.
|
||||
list.push(l);
|
||||
}
|
||||
}
|
||||
if !safe_local(&target) {
|
||||
return None;
|
||||
}
|
||||
let mut r = url::escape_path(&relative(local, &target));
|
||||
if r.split('/').next().is_some_and(|s| s.contains(':')) {
|
||||
r.insert_str(0, "./");
|
||||
}
|
||||
if !frag.is_empty() {
|
||||
r.push('#');
|
||||
r.push_str(&frag);
|
||||
}
|
||||
added.set(added.get() + r.len() as u64);
|
||||
(added.get() <= cap).then_some(r)
|
||||
};
|
||||
let out = if is_html {
|
||||
let mut on_base = |v: &str| -> Option<String> {
|
||||
let joined = base.borrow().join(v);
|
||||
if let Ok(b) = joined {
|
||||
*base.borrow_mut() = b;
|
||||
}
|
||||
// Point the copy's base at the page itself, whatever host it
|
||||
// named, so the relative links written here resolve offline.
|
||||
Some(relative(local, local))
|
||||
};
|
||||
html::rewrite_html(body, &mut on_base, &mut link)
|
||||
} else {
|
||||
html::rewrite_css(body, false, &mut link)
|
||||
};
|
||||
if added.get() > cap {
|
||||
return Err(FetchError::Other(format!("rewriting its links would grow it past {cap} bytes")));
|
||||
}
|
||||
for l in found.into_inner().0 {
|
||||
self.enqueue(l);
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `ProxyFromEnvironment`: the proxy URL for `u`, if any.
|
||||
fn proxy_for(u: &Url) -> Option<String> {
|
||||
let env = |names: &[&str]| names.iter().find_map(|n| std::env::var(n).ok().filter(|v| !v.is_empty()));
|
||||
let proxy = if u.scheme == "https" {
|
||||
env(&["HTTPS_PROXY", "https_proxy"])
|
||||
} else if std::env::var_os("REQUEST_METHOD").is_some_and(|v| !v.is_empty()) {
|
||||
None
|
||||
} else {
|
||||
env(&["HTTP_PROXY", "http_proxy"])
|
||||
}?;
|
||||
if !use_proxy(u, env(&["NO_PROXY", "no_proxy"]).as_deref().unwrap_or("")) {
|
||||
return None;
|
||||
}
|
||||
Some(if proxy.contains("://") { proxy } else { format!("http://{proxy}") })
|
||||
}
|
||||
|
||||
/// Whether `u` should go through a proxy given `no_proxy` (Go's
|
||||
/// httpproxy semantics). Known differences from Go, all in the direction of
|
||||
/// using the proxy: IP entries in `NO_PROXY` ignore any port, bracketed
|
||||
/// IPv6 entries with a port never match, and host names are not
|
||||
/// IDNA-normalised.
|
||||
fn use_proxy(u: &Url, no_proxy: &str) -> bool {
|
||||
let host = u.hostname().to_lowercase();
|
||||
let port = u.port_or_default();
|
||||
if host == "localhost" {
|
||||
return false;
|
||||
}
|
||||
let ip: Option<std::net::IpAddr> = host.parse().ok();
|
||||
let loopback = |ip: std::net::IpAddr| match ip {
|
||||
std::net::IpAddr::V6(v6) => v6.to_ipv4_mapped().map_or(v6.is_loopback(), |v4| v4.is_loopback()),
|
||||
ip => ip.is_loopback(),
|
||||
};
|
||||
if ip.is_some_and(loopback) {
|
||||
return false;
|
||||
}
|
||||
for entry in no_proxy.split(',').map(|e| e.trim().to_lowercase()).filter(|e| !e.is_empty()) {
|
||||
if entry == "*" {
|
||||
return false;
|
||||
}
|
||||
if let Some((net, bits)) = entry.split_once('/') {
|
||||
if let (Ok(net), Ok(bits), Some(ip)) = (net.parse::<std::net::IpAddr>(), bits.parse::<u32>(), ip)
|
||||
&& in_cidr(ip, net, bits)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Ok(eip) = entry.trim_start_matches('[').trim_end_matches(']').parse::<std::net::IpAddr>() {
|
||||
if ip == Some(eip) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let (mut phost, pport) = match entry.rsplit_once(':') {
|
||||
Some((h, p)) if !h.is_empty() && p.bytes().all(|c| c.is_ascii_digit()) => (h.to_string(), Some(p)),
|
||||
_ => (entry.clone(), None),
|
||||
};
|
||||
if let Some(rest) = phost.strip_prefix("*.") {
|
||||
phost = format!(".{rest}");
|
||||
}
|
||||
let match_host = !phost.starts_with('.');
|
||||
if match_host {
|
||||
phost.insert(0, '.');
|
||||
}
|
||||
if (host.ends_with(&phost) || (match_host && host == phost[1..])) && pport.is_none_or(|p| p == port) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn in_cidr(ip: std::net::IpAddr, net: std::net::IpAddr, bits: u32) -> bool {
|
||||
let (a, b, width): (u128, u128, u32) = match (ip, net) {
|
||||
(std::net::IpAddr::V4(a), std::net::IpAddr::V4(b)) => (u32::from(a) as u128, u32::from(b) as u128, 32),
|
||||
(std::net::IpAddr::V6(a), std::net::IpAddr::V6(b)) => (u128::from(a), u128::from(b), 128),
|
||||
_ => return false,
|
||||
};
|
||||
if bits > width {
|
||||
return false;
|
||||
}
|
||||
let shift = width - bits;
|
||||
shift >= width || (a >> shift) == (b >> shift)
|
||||
}
|
||||
|
||||
/// Go's `http.ParseTime`: RFC 1123 with "GMT", RFC 850, or ANSI C, as Unix
|
||||
/// seconds.
|
||||
fn parse_http_time(s: &str) -> Option<i64> {
|
||||
const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
const DAYS: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
|
||||
const LONG: [&str; 7] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
|
||||
let month = |m: &str| MONTHS.iter().position(|x| *x == m).map(|i| i as i64 + 1);
|
||||
let num = |t: &str, n: usize| {
|
||||
(t.len() == n && t.bytes().all(|c| c.is_ascii_digit()))
|
||||
.then(|| t.parse::<i64>().ok())
|
||||
.flatten()
|
||||
};
|
||||
let clock = |t: &str| -> Option<i64> {
|
||||
let p: Vec<&str> = t.split(':').collect();
|
||||
let (h, m, s) = (num(p.first()?, 2)?, num(p.get(1)?, 2)?, num(p.get(2)?, 2)?);
|
||||
(p.len() == 3 && h < 24 && m < 60 && s < 60).then_some(h * 3600 + m * 60 + s)
|
||||
};
|
||||
let date = |y: i64, mo: i64, d: i64| -> Option<i64> {
|
||||
let dim = [
|
||||
31,
|
||||
if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 { 29 } else { 28 },
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
];
|
||||
(d >= 1 && d <= dim[(mo - 1) as usize]).then(|| days_from_civil(y, mo, d) * 86400)
|
||||
};
|
||||
// Mon, 02 Jan 2006 15:04:05 GMT
|
||||
if let Some(rest) = s.get(..5).filter(|d| DAYS.contains(&&d[..3]) && &d[3..] == ", ").map(|_| &s[5..]) {
|
||||
let p: Vec<&str> = rest.split(' ').collect();
|
||||
if p.len() == 5 && p[4] == "GMT" {
|
||||
return Some(date(num(p[2], 4)?, month(p[1])?, num(p[0], 2)?)? + clock(p[3])?);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
// Monday, 02-Jan-06 15:04:05 MST
|
||||
if let Some((day, rest)) = s.split_once(", ") {
|
||||
if !LONG.contains(&day) {
|
||||
return None;
|
||||
}
|
||||
let p: Vec<&str> = rest.split(' ').collect();
|
||||
if p.len() != 3 || !(3..=5).contains(&p[2].len()) || !p[2].bytes().all(|c| c.is_ascii_uppercase()) {
|
||||
return None;
|
||||
}
|
||||
let d: Vec<&str> = p[0].split('-').collect();
|
||||
if d.len() != 3 {
|
||||
return None;
|
||||
}
|
||||
let yy = num(d[2], 2)?;
|
||||
return Some(date(if yy >= 69 { 1900 + yy } else { 2000 + yy }, month(d[1])?, num(d[0], 2)?)? + clock(p[1])?);
|
||||
}
|
||||
// Mon Jan _2 15:04:05 2006
|
||||
let (head, rest) = (s.get(..8)?, s.get(8..)?);
|
||||
if !DAYS.contains(&&head[..3]) || &head[3..4] != " " || &head[7..8] != " " {
|
||||
return None;
|
||||
}
|
||||
let mo = month(&head[4..7])?;
|
||||
let day = if let Some(r) = rest.strip_prefix(' ') {
|
||||
(r, num(r.get(..1)?, 1)?)
|
||||
} else {
|
||||
(rest, num(rest.get(..2)?, 2)?)
|
||||
};
|
||||
let (r, d) = (day.0, day.1);
|
||||
let after = &r[if rest.starts_with(' ') { 1 } else { 2 }..];
|
||||
let p: Vec<&str> = after.strip_prefix(' ')?.split(' ').collect();
|
||||
if p.len() != 2 {
|
||||
return None;
|
||||
}
|
||||
Some(date(num(p[1], 4)?, mo, d)? + clock(p[0])?)
|
||||
}
|
||||
|
||||
/// Days since 1970-01-01 of a proleptic Gregorian date.
|
||||
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
||||
let y = if m <= 2 { y - 1 } else { y };
|
||||
let era = y.div_euclid(400);
|
||||
let yoe = y - era * 400;
|
||||
let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1;
|
||||
era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Network tests against local HTTP fixtures; nothing leaves 127.0.0.1.
|
||||
|
||||
use super::*;
|
||||
use crate::crypto::kem::{Identity, SUITE_ID};
|
||||
use crate::pipeline::Pool;
|
||||
use std::io::BufRead;
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// What a fixture handler sends back for a request.
|
||||
enum Reply {
|
||||
/// Status line text, headers, body; Content-Length is added.
|
||||
Full(&'static str, Vec<(String, String)>, Vec<u8>),
|
||||
/// Raw bytes, then the connection closes.
|
||||
Raw(Vec<u8>),
|
||||
/// Headers then a stall.
|
||||
Stall,
|
||||
}
|
||||
|
||||
struct Server {
|
||||
addr: String,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Serves `handler(path, user_agent)` on 127.0.0.1, one thread per
|
||||
/// connection, for the life of the test process.
|
||||
fn start(handler: impl Fn(&str, &str) -> Reply + Send + Sync + 'static) -> Server {
|
||||
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = l.local_addr().unwrap().to_string();
|
||||
let handler = Arc::new(handler);
|
||||
std::thread::spawn(move || {
|
||||
for c in l.incoming().flatten() {
|
||||
let h = Arc::clone(&handler);
|
||||
std::thread::spawn(move || serve(c, &*h));
|
||||
}
|
||||
});
|
||||
Server { addr }
|
||||
}
|
||||
|
||||
fn url(&self, p: &str) -> String {
|
||||
format!("http://{}{p}", self.addr)
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(mut c: TcpStream, h: &dyn Fn(&str, &str) -> Reply) {
|
||||
let mut r = std::io::BufReader::new(c.try_clone().unwrap());
|
||||
let mut line = String::new();
|
||||
if r.read_line(&mut line).is_err() {
|
||||
return;
|
||||
}
|
||||
let path = line.split(' ').nth(1).unwrap_or("/").to_string();
|
||||
let mut ua = String::new();
|
||||
loop {
|
||||
let mut l = String::new();
|
||||
if r.read_line(&mut l).unwrap_or(0) == 0 || l == "\r\n" {
|
||||
break;
|
||||
}
|
||||
if let Some(v) = l.strip_prefix("user-agent: ").or_else(|| l.strip_prefix("User-Agent: ")) {
|
||||
ua = v.trim().to_string();
|
||||
}
|
||||
}
|
||||
let out = match h(&path, &ua) {
|
||||
Reply::Full(status, headers, body) => {
|
||||
let mut o = format!("HTTP/1.1 {status}\r\nConnection: close\r\nContent-Length: {}\r\n", body.len());
|
||||
for (k, v) in headers {
|
||||
o.push_str(&format!("{k}: {v}\r\n"));
|
||||
}
|
||||
o.push_str("\r\n");
|
||||
[o.into_bytes(), body].concat()
|
||||
}
|
||||
Reply::Raw(b) => b,
|
||||
Reply::Stall => {
|
||||
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc");
|
||||
std::thread::sleep(Duration::from_secs(5));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = c.write_all(&out);
|
||||
}
|
||||
|
||||
fn ok(ct: &str, body: &str) -> Reply {
|
||||
let headers = if ct.is_empty() {
|
||||
vec![]
|
||||
} else {
|
||||
vec![("Content-Type".to_string(), ct.to_string())]
|
||||
};
|
||||
Reply::Full("200 OK", headers, body.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn not_found() -> Reply {
|
||||
Reply::Full("404 Not Found", vec![], b"no".to_vec())
|
||||
}
|
||||
|
||||
fn redirect(to: &str) -> Reply {
|
||||
Reply::Full("302 Found", vec![("Location".into(), to.into())], vec![])
|
||||
}
|
||||
|
||||
struct Vault {
|
||||
dir: PathBuf,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
impl Drop for Vault {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn vault() -> Vault {
|
||||
let r: [u8; 8] = crate::rng::os().array();
|
||||
let dir = std::env::temp_dir().join(format!("vault-net-{}-{}", std::process::id(), crate::store::hex(&r)));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let root = dir.join("store");
|
||||
let pool = Arc::new(Pool::new(2));
|
||||
Store::init(&root, 0, Arc::clone(&pool)).unwrap();
|
||||
let store = Store::open(&root, Identity::from_suite_seed(SUITE_ID, &[1; 32]).unwrap(), pool).unwrap();
|
||||
Vault { dir, store }
|
||||
}
|
||||
|
||||
impl Vault {
|
||||
fn get(&self, name: &str) -> Vec<u8> {
|
||||
let cat = self.store.catalogue().unwrap();
|
||||
let rec = cat
|
||||
.current(name)
|
||||
.unwrap_or_else(|| panic!("{name} not stored; have {:?}", cat.names().collect::<Vec<_>>()));
|
||||
let out = format!("out-{}", rec.id.hex());
|
||||
self.store
|
||||
.extract(rec, &crate::platform::fs::Dir::open(&self.dir).unwrap(), std::ffi::OsStr::new(&out))
|
||||
.unwrap();
|
||||
std::fs::read(self.dir.join(out)).unwrap()
|
||||
}
|
||||
|
||||
fn names(&self) -> Vec<String> {
|
||||
self.store.catalogue().unwrap().names().map(String::from).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn mirror_with(v: &Vault, url: &str, limits: &Limits) -> MirrorOutcome {
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
mirror(&v.store, &mut cat, url, "m", limits, &mut |_, _| {}).unwrap()
|
||||
}
|
||||
|
||||
/// Staged downloads leave no name behind on Unix (unlinked at creation),
|
||||
/// and are removed on drop elsewhere — so counting files in the shared
|
||||
/// temp directory, which other tests use concurrently, is never needed.
|
||||
#[test]
|
||||
fn temp_files_leave_nothing_behind() {
|
||||
let mut t = TempFile::new("vault-test-").unwrap();
|
||||
let path = t.path.clone();
|
||||
t.file().write_all(b"plaintext").unwrap();
|
||||
#[cfg(unix)]
|
||||
assert!(!path.exists(), "the name should be gone at once");
|
||||
drop(t);
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_url_names_and_user_agent() {
|
||||
let srv = Server::start(|p, ua| {
|
||||
if ua != CHROME_UA {
|
||||
return Reply::Full("403 Forbidden", vec![], b"bad agent".to_vec());
|
||||
}
|
||||
match p {
|
||||
"/files/page.html" => Reply::Full(
|
||||
"200 OK",
|
||||
vec![("Last-Modified".into(), "Tue, 15 Nov 1994 08:12:31 GMT".into())],
|
||||
b"body".to_vec(),
|
||||
),
|
||||
"/old" => redirect("/files/new.txt"),
|
||||
"/files/new.txt" => ok("", "moved"),
|
||||
"/" => ok("", "root"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
let v = vault();
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
let mut reported = Vec::new();
|
||||
let lim = Limits::add_defaults();
|
||||
add_url(&v.store, &mut cat, &srv.url("/files/page.html"), "web", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
add_url(&v.store, &mut cat, &srv.url("/old"), "web", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
add_url(&v.store, &mut cat, &srv.url(""), "", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
reported,
|
||||
[
|
||||
("web/page.html".into(), false),
|
||||
("web/new.txt".into(), false),
|
||||
("127.0.0.1".into(), false)
|
||||
]
|
||||
);
|
||||
assert_eq!(v.get("web/page.html"), b"body");
|
||||
assert_eq!(v.get("web/new.txt"), b"moved", "named after the final URL");
|
||||
assert_eq!(
|
||||
v.store.catalogue().unwrap().current("web/page.html").unwrap().modified.secs,
|
||||
784887151
|
||||
);
|
||||
let e = add_url(&v.store, &mut cat, &srv.url("/missing"), "", &lim, &mut |_, _| {}).unwrap_err();
|
||||
assert!(e.msg.ends_with("404 Not Found"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_url_limits() {
|
||||
let gz = {
|
||||
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
|
||||
e.write_all(&vec![b'a'; 1 << 20]).unwrap();
|
||||
e.finish().unwrap()
|
||||
};
|
||||
let srv = Server::start(move |p, _| match p {
|
||||
"/stall" => Reply::Stall,
|
||||
"/long" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabcdefgh".to_vec()),
|
||||
"/short" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nabc".to_vec()),
|
||||
"/bomb" => Reply::Full("200 OK", vec![("Content-Encoding".into(), "gzip".into())], gz.clone()),
|
||||
"/loop" => redirect("/loop"),
|
||||
"/big" => ok("", &"x".repeat(5000)),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
let small = Limits {
|
||||
request: Duration::from_secs(2),
|
||||
max_bytes: 4096,
|
||||
..Limits::add_defaults()
|
||||
};
|
||||
let mut try_add = |p: &str| add_url(&v.store, &mut cat, &srv.url(p), "", &small, &mut |_, _| {});
|
||||
let e = try_add("/stall").unwrap_err();
|
||||
assert!(e.msg.contains("timeout") || e.msg.contains("timed out"), "{e}");
|
||||
// A server sending more than it declared is cut at its Content-Length.
|
||||
try_add("/long").unwrap();
|
||||
assert!(try_add("/short").is_err(), "a body shorter than declared is an error");
|
||||
let e = try_add("/bomb").unwrap_err();
|
||||
assert!(e.msg.contains("exceeds 4096 bytes"), "decoded bytes count: {e}");
|
||||
let e = try_add("/loop").unwrap_err();
|
||||
assert!(e.msg.contains("stopped after 10 redirects"), "{e}");
|
||||
assert!(try_add("/big").unwrap_err().msg.contains("exceeds"));
|
||||
assert_eq!(v.get("long"), b"abc");
|
||||
assert_eq!(v.names(), ["long"], "nothing else stored");
|
||||
}
|
||||
|
||||
/// A redirect to /%2e%2e must not name the download "..", which would
|
||||
/// replace the folder's parent entry.
|
||||
#[test]
|
||||
fn add_url_dot_names_fall_back_to_host() {
|
||||
let srv = Server::start(|p, _| match p {
|
||||
"/up" => redirect("/%2e%2e"),
|
||||
"/.." | "/%2e%2e" => ok("", "dots"),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
let mut got = Vec::new();
|
||||
add_url(&v.store, &mut cat, &srv.url("/up"), "a/b", &Limits::add_defaults(), &mut |n, _| {
|
||||
got.push(n.to_string())
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(got, ["a/b/127.0.0.1"]);
|
||||
}
|
||||
|
||||
/// Bytes received before a failure still count against the mirror budget.
|
||||
#[test]
|
||||
fn mirror_charges_failed_downloads() {
|
||||
let server = Server::start(|p, _| match p {
|
||||
"/" => ok("text/html", r#"<a href="/1">1</a><a href="/2">2</a>"#),
|
||||
"/1" => {
|
||||
let mut b = b"HTTP/1.1 200 OK\r\nContent-Length: 100000\r\nConnection: close\r\n\r\n".to_vec();
|
||||
b.extend_from_slice(&[b'y'; 2000]);
|
||||
Reply::Raw(b)
|
||||
}
|
||||
"/2" => ok("", &"z".repeat(1000)),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let lim = Limits {
|
||||
max_bytes: 2500,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
match mirror_with(&v, &server.url("/"), &lim) {
|
||||
MirrorOutcome::Partial { stored: 1, skipped: 2 } => {}
|
||||
MirrorOutcome::Partial { stored, skipped } => panic!("stored {stored}, skipped {skipped}"),
|
||||
MirrorOutcome::Complete { .. } => panic!("truncated body not charged"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Links that decode to "." or ".." segments never escape the mirror's
|
||||
/// folder, and rewritten links are escaped.
|
||||
#[test]
|
||||
fn mirror_refuses_escaping_paths() {
|
||||
let srv = Arc::new(std::sync::OnceLock::<String>::new());
|
||||
let s2 = Arc::clone(&srv);
|
||||
let server = Server::start(move |p, _| {
|
||||
let h = s2.get().unwrap().trim_start_matches("http:").to_string();
|
||||
match p {
|
||||
"/" => ok(
|
||||
"text/html",
|
||||
&format!(r#"<a href="/%2e%2e/x">a</a><a href="{h}/..%2f..%2fq">b</a><a href="/a%3Fb">c</a><a href="/c:d">d</a>"#),
|
||||
),
|
||||
"/a%3Fb" | "/c:d" => ok("", "fine"),
|
||||
_ => ok("", "should never be stored"),
|
||||
}
|
||||
});
|
||||
srv.set(server.url("")).unwrap();
|
||||
let v = vault();
|
||||
let out = mirror_with(&v, &server.url("/"), &Limits::mirror_defaults());
|
||||
assert!(matches!(out, MirrorOutcome::Partial { stored: 3, skipped: 2 }), "{:?}", v.names());
|
||||
let h = &server.addr;
|
||||
for n in v.names() {
|
||||
assert!(n.starts_with(&format!("m/{h}/")), "stored outside the mirror: {n}");
|
||||
}
|
||||
let index = String::from_utf8(v.get(&format!("m/{h}/index.html"))).unwrap();
|
||||
assert!(index.contains(r#"href="a%3Fb""#), "{index}");
|
||||
assert!(index.contains(r#"href="./c:d""#), "{index}");
|
||||
assert!(index.contains(r#"href="/%2e%2e/x""#), "unsafe links are left as they were: {index}");
|
||||
}
|
||||
|
||||
/// A deeply nested page full of links to the root cannot make the rewrite
|
||||
/// balloon: past four times the rewrite limit, the page is skipped.
|
||||
#[test]
|
||||
fn mirror_bounds_rewrite_growth() {
|
||||
let deep = format!("/{}p.html", "d/".repeat(40));
|
||||
let page = format!("<style>{}</style>", "a{b:url(/)}".repeat(80));
|
||||
let server = Server::start(move |p, _| match p {
|
||||
"/" => ok("text/html", "root"),
|
||||
_ => ok("text/html", &page),
|
||||
});
|
||||
let v = vault();
|
||||
let lim = Limits {
|
||||
max_text: 1000,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
assert!(matches!(
|
||||
mirror_with(&v, &server.url(&deep), &lim),
|
||||
MirrorOutcome::Partial { stored: 0, skipped: 1 }
|
||||
));
|
||||
}
|
||||
|
||||
/// Go's TestMirror: links in HTML and CSS become relative, off-host links
|
||||
/// stay, everything lands under host/.
|
||||
#[test]
|
||||
fn mirror_like_go() {
|
||||
let srv = Arc::new(std::sync::OnceLock::<String>::new());
|
||||
let s2 = Arc::clone(&srv);
|
||||
let server = Server::start(move |p, ua| {
|
||||
if ua != CHROME_UA {
|
||||
return Reply::Full("403 Forbidden", vec![], vec![]);
|
||||
}
|
||||
let base = s2.get().unwrap();
|
||||
match p {
|
||||
"/" => ok(
|
||||
"text/html",
|
||||
&format!(r#"<a href="{base}/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>"#),
|
||||
),
|
||||
"/sub/p.html" => ok("text/html", r#"<a href="/">home</a> <img src="../img/a.png">"#),
|
||||
"/s.css" => ok("text/css", "body{background:url(/img/a.png)}"),
|
||||
"/img/a.png" => ok("", "png"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
srv.set(server.url("")).unwrap();
|
||||
let v = vault();
|
||||
let out = mirror_with(&v, &server.url(""), &Limits::mirror_defaults());
|
||||
assert!(matches!(out, MirrorOutcome::Complete { stored: 4 }));
|
||||
let h = &server.addr;
|
||||
for (name, want) in [
|
||||
(
|
||||
format!("m/{h}/index.html"),
|
||||
r#"<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>"#,
|
||||
),
|
||||
(
|
||||
format!("m/{h}/sub/p.html"),
|
||||
r#"<a href="../index.html">home</a> <img src="../img/a.png">"#,
|
||||
),
|
||||
(format!("m/{h}/s.css"), "body{background:url(img/a.png)}"),
|
||||
(format!("m/{h}/img/a.png"), "png"),
|
||||
] {
|
||||
assert_eq!(String::from_utf8(v.get(&name)).unwrap(), want, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_policies() {
|
||||
let srv = Arc::new(std::sync::OnceLock::<String>::new());
|
||||
let s2 = Arc::clone(&srv);
|
||||
let server = Server::start(move |p, _| {
|
||||
let other = s2.get().unwrap().replace("127.0.0.1", "localhost");
|
||||
match p {
|
||||
"/" => ok(
|
||||
"text/html",
|
||||
r#"<head><base href="/d/"></head><a href=a.html>a</a> <a href="/away">x</a> <a href="/a_b%253Dc">c</a> <a href="/a?b=c">q</a>
|
||||
<script>var s = '<a href="/never">';</script><!-- <a href="/never2"> --><div style="background:url('/i.png')"><style>p{background:url("/j.png")}</style>"#,
|
||||
),
|
||||
"/d/a.html" => ok("text/html", "<p>a</p>"),
|
||||
"/away" => redirect(&format!("{other}/elsewhere")),
|
||||
"/a_b%253Dc" => ok("", "literal"),
|
||||
"/a?b=c" => ok("", "query"),
|
||||
"/i.png" | "/j.png" => ok("", "img"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
srv.set(server.url("")).unwrap();
|
||||
let v = vault();
|
||||
let out = mirror_with(&v, &server.url("/"), &Limits::mirror_defaults());
|
||||
// /away (off-host redirect) and one of the colliding names are skipped.
|
||||
match out {
|
||||
MirrorOutcome::Partial { skipped: 2, stored: 5 } => {}
|
||||
MirrorOutcome::Partial { stored, skipped } => panic!("stored {stored}, skipped {skipped}: {:?}", v.names()),
|
||||
MirrorOutcome::Complete { stored } => panic!("complete, stored {stored}"),
|
||||
}
|
||||
let h = &server.addr;
|
||||
let index = String::from_utf8(v.get(&format!("m/{h}/index.html"))).unwrap();
|
||||
assert!(index.contains(r#"<base href="index.html">"#), "{index}");
|
||||
assert!(index.contains("<a href=d/a.html>"), "base href honoured, unquoted kept: {index}");
|
||||
assert!(
|
||||
index.contains(r#"'<a href="/never">'"#) && index.contains(r#"<!-- <a href="/never2"> -->"#),
|
||||
"{index}"
|
||||
);
|
||||
assert!(index.contains("url('i.png')") && index.contains(r#"url("j.png")"#), "{index}");
|
||||
let names = v.names();
|
||||
assert!(names.contains(&format!("m/{h}/d/a.html")));
|
||||
assert!(!names.iter().any(|n| n.contains("never") || n.contains("elsewhere")), "{names:?}");
|
||||
let collided = [format!("m/{h}/a_b%3Dc")];
|
||||
assert_eq!(
|
||||
names.iter().filter(|n| collided.contains(n)).count(),
|
||||
1,
|
||||
"one of two URLs owns the name"
|
||||
);
|
||||
let kept = String::from_utf8(v.get(&collided[0])).unwrap();
|
||||
assert!(kept == "literal" || kept == "query");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_budgets() {
|
||||
let server = Server::start(|p, _| match p {
|
||||
"/" => ok("text/html", r#"<a href="/1">1</a><a href="/2">2</a><a href="/3">3</a>"#),
|
||||
"/1" | "/2" | "/3" => ok("", &"z".repeat(1000)),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let pages = Limits {
|
||||
max_pages: 2,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
match mirror_with(&v, &server.url("/"), &pages) {
|
||||
MirrorOutcome::Partial { stored: 2, .. } => {}
|
||||
_ => panic!("page limit not applied"),
|
||||
}
|
||||
let v = vault();
|
||||
let bytes = Limits {
|
||||
max_bytes: 1500,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
match mirror_with(&v, &server.url("/"), &bytes) {
|
||||
MirrorOutcome::Partial { stored: 2, skipped: 2 } => {}
|
||||
MirrorOutcome::Partial { stored, skipped } => panic!("byte budget: stored {stored}, skipped {skipped}"),
|
||||
MirrorOutcome::Complete { .. } => panic!("byte budget not applied"),
|
||||
}
|
||||
let v = vault();
|
||||
let text = Limits {
|
||||
max_text: 10,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
assert!(matches!(
|
||||
mirror_with(&v, &server.url("/"), &text),
|
||||
MirrorOutcome::Partial { stored: 0, skipped: 1 }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_matches_go() {
|
||||
for r in url::tests::vectors()["relative"].as_array().unwrap() {
|
||||
let r = r.as_array().unwrap();
|
||||
assert_eq!(
|
||||
relative(r[0].as_str().unwrap(), r[1].as_str().unwrap()),
|
||||
r[2].as_str().unwrap(),
|
||||
"{r:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_time_matches_go() {
|
||||
for (s, want) in url::tests::vectors()["http_time"].as_object().unwrap() {
|
||||
assert_eq!(parse_http_time(s), want.as_i64(), "{s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_rules() {
|
||||
let u = |s: &str| Url::parse(s).unwrap();
|
||||
assert!(!use_proxy(&u("http://localhost:8080/"), ""));
|
||||
assert!(!use_proxy(&u("http://127.0.0.1/"), ""));
|
||||
assert!(!use_proxy(&u("http://[::1]/"), ""));
|
||||
assert!(!use_proxy(&u("http://[::ffff:127.0.0.1]/"), ""));
|
||||
assert_eq!(
|
||||
u("http://user:secret@h/x").redacted(),
|
||||
"http://h/x",
|
||||
"credentials stay out of messages"
|
||||
);
|
||||
assert!(use_proxy(&u("http://example.com/"), "other.com"));
|
||||
assert!(!use_proxy(&u("http://example.com/"), "example.com"));
|
||||
assert!(!use_proxy(&u("http://a.example.com/"), "example.com"));
|
||||
assert!(use_proxy(&u("http://example.com/"), ".example.com"));
|
||||
assert!(!use_proxy(&u("http://a.example.com/"), "*.example.com"));
|
||||
assert!(!use_proxy(&u("http://x.com/"), "*"));
|
||||
assert!(use_proxy(&u("http://x.com:81/"), "x.com:80"));
|
||||
assert!(!use_proxy(&u("http://x.com/"), "x.com:80"));
|
||||
assert!(!use_proxy(&u("https://10.1.2.3/"), "10.0.0.0/8"));
|
||||
assert!(use_proxy(&u("https://11.1.2.3/"), "10.0.0.0/8, 192.168.1.1"));
|
||||
assert!(!use_proxy(&u("https://192.168.1.1/"), "10.0.0.0/8, 192.168.1.1"));
|
||||
}
|
||||
+500
@@ -0,0 +1,500 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Just enough of Go's `net/url` to crawl and name pages exactly as the Go
|
||||
//! version did: `Parse`, `ResolveReference` (RFC 3986 with dot-segment
|
||||
//! removal), `String` (re-escaping like `EscapedPath`), `Hostname` and
|
||||
//! `QueryEscape`. `path` is percent-decoded; `raw_query` is kept as sent.
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub scheme: String,
|
||||
pub opaque: String,
|
||||
/// Userinfo as written (still escaped), without the '@'.
|
||||
pub user: Option<String>,
|
||||
pub host: String,
|
||||
pub path: String,
|
||||
raw_path: String,
|
||||
pub force_query: bool,
|
||||
pub raw_query: String,
|
||||
pub fragment: String,
|
||||
raw_fragment: String,
|
||||
omit_host: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Mode {
|
||||
Path,
|
||||
Host,
|
||||
QueryComponent,
|
||||
Fragment,
|
||||
}
|
||||
|
||||
fn should_escape(c: u8, mode: Mode) -> bool {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
return false;
|
||||
}
|
||||
if mode == Mode::Host && b"!$&'()*+,;=:[]<>\"".contains(&c) {
|
||||
return false;
|
||||
}
|
||||
match c {
|
||||
b'-' | b'_' | b'.' | b'~' => return false,
|
||||
b'$' | b'&' | b'+' | b',' | b'/' | b':' | b';' | b'=' | b'?' | b'@' => {
|
||||
return match mode {
|
||||
Mode::Path => c == b'?',
|
||||
Mode::QueryComponent => true,
|
||||
Mode::Fragment => false,
|
||||
Mode::Host => true,
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if mode == Mode::Fragment && b"!()*".contains(&c) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
/// Percent-escapes `s` for use as a URL path, as Go's `PathEscape` does
|
||||
/// within a path (slashes kept).
|
||||
pub fn escape_path(s: &str) -> String {
|
||||
escape(s, Mode::Path)
|
||||
}
|
||||
|
||||
fn escape(s: &str, mode: Mode) -> String {
|
||||
let mut o = String::with_capacity(s.len());
|
||||
for &c in s.as_bytes() {
|
||||
if c == b' ' && mode == Mode::QueryComponent {
|
||||
o.push('+');
|
||||
} else if should_escape(c, mode) {
|
||||
o.push_str(&format!("%{c:02X}"));
|
||||
} else {
|
||||
o.push(c as char);
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
fn unhex(c: u8) -> Option<u8> {
|
||||
(c as char).to_digit(16).map(|d| d as u8)
|
||||
}
|
||||
|
||||
fn unescape(s: &str, mode: Mode) -> Result<String, String> {
|
||||
let b = s.as_bytes();
|
||||
let mut out = Vec::with_capacity(b.len());
|
||||
let mut i = 0;
|
||||
while i < b.len() {
|
||||
match b[i] {
|
||||
b'%' => {
|
||||
let (h, l) = match (b.get(i + 1).copied().and_then(unhex), b.get(i + 2).copied().and_then(unhex)) {
|
||||
(Some(h), Some(l)) => (h, l),
|
||||
_ => {
|
||||
return Err(format!(
|
||||
"invalid URL escape {:?}",
|
||||
String::from_utf8_lossy(&b[i..(i + 3).min(b.len())])
|
||||
));
|
||||
}
|
||||
};
|
||||
if mode == Mode::Host && h < 8 && &s[i..i + 3] != "%25" {
|
||||
return Err(format!("invalid URL escape {:?}", &s[i..i + 3]));
|
||||
}
|
||||
out.push(h << 4 | l);
|
||||
i += 3;
|
||||
}
|
||||
b'+' => {
|
||||
out.push(if mode == Mode::QueryComponent { b' ' } else { b'+' });
|
||||
i += 1;
|
||||
}
|
||||
c => {
|
||||
if mode == Mode::Host && c < 0x80 && should_escape(c, mode) {
|
||||
return Err(format!("invalid character {:?} in host name", c as char));
|
||||
}
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Go strings may hold any bytes; a path that decodes to invalid UTF-8
|
||||
// cannot become a vault name anyway, so it is refused here.
|
||||
String::from_utf8(out).map_err(|_| "invalid UTF-8 in URL".to_string())
|
||||
}
|
||||
|
||||
pub fn query_escape(s: &str) -> String {
|
||||
escape(s, Mode::QueryComponent)
|
||||
}
|
||||
|
||||
fn valid_encoded(s: &str, mode: Mode) -> bool {
|
||||
s.bytes()
|
||||
.all(|c| b"!$&'()*+,;=:@[]".contains(&c) || c == b'%' || !should_escape(c, mode))
|
||||
}
|
||||
|
||||
fn get_scheme(raw: &str) -> Result<(&str, &str), String> {
|
||||
for (i, c) in raw.bytes().enumerate() {
|
||||
match c {
|
||||
b'a'..=b'z' | b'A'..=b'Z' => {}
|
||||
b'0'..=b'9' | b'+' | b'-' | b'.' if i > 0 => {}
|
||||
b':' if i == 0 => return Err("missing protocol scheme".into()),
|
||||
b':' => return Ok((&raw[..i], &raw[i + 1..])),
|
||||
_ => return Ok(("", raw)),
|
||||
}
|
||||
}
|
||||
Ok(("", raw))
|
||||
}
|
||||
|
||||
fn valid_optional_port(p: &str) -> bool {
|
||||
p.is_empty() || (p.starts_with(':') && p[1..].bytes().all(|c| c.is_ascii_digit()))
|
||||
}
|
||||
|
||||
fn parse_host(host: &str) -> Result<String, String> {
|
||||
if let Some(rest) = host.strip_prefix('[') {
|
||||
let end = rest.find(']').ok_or("missing ']' in host")?;
|
||||
let port = &rest[end + 1..];
|
||||
if !valid_optional_port(port) {
|
||||
return Err(format!("invalid port {port:?} after host"));
|
||||
}
|
||||
} else if let Some(i) = host.rfind(':') {
|
||||
let port = &host[i..];
|
||||
if !valid_optional_port(port) {
|
||||
return Err(format!("invalid port {port:?} after host"));
|
||||
}
|
||||
}
|
||||
unescape(host, Mode::Host)
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(raw: &str) -> Result<Url, String> {
|
||||
let (rest, frag) = match raw.find('#') {
|
||||
Some(i) => (&raw[..i], Some(&raw[i + 1..])),
|
||||
None => (raw, None),
|
||||
};
|
||||
let mut u = Url::parse_without_fragment(rest)?;
|
||||
if let Some(f) = frag {
|
||||
u.fragment = unescape(f, Mode::Fragment)?;
|
||||
u.raw_fragment = if escape(&u.fragment, Mode::Fragment) == f {
|
||||
String::new()
|
||||
} else {
|
||||
f.to_string()
|
||||
};
|
||||
}
|
||||
Ok(u)
|
||||
}
|
||||
|
||||
fn parse_without_fragment(raw: &str) -> Result<Url, String> {
|
||||
if raw.bytes().any(|c| c < 0x20 || c == 0x7f) {
|
||||
return Err("net/url: invalid control character in URL".into());
|
||||
}
|
||||
let mut u = Url::default();
|
||||
if raw.is_empty() {
|
||||
return Ok(u);
|
||||
}
|
||||
let (scheme, mut rest) = get_scheme(raw)?;
|
||||
u.scheme = scheme.to_ascii_lowercase();
|
||||
if rest.ends_with('?') && rest.matches('?').count() == 1 {
|
||||
u.force_query = true;
|
||||
rest = &rest[..rest.len() - 1];
|
||||
} else if let Some(i) = rest.find('?') {
|
||||
u.raw_query = rest[i + 1..].to_string();
|
||||
rest = &rest[..i];
|
||||
}
|
||||
if !rest.starts_with('/') {
|
||||
if !u.scheme.is_empty() {
|
||||
u.opaque = rest.to_string();
|
||||
return Ok(u);
|
||||
}
|
||||
let seg = rest.split('/').next().unwrap_or("");
|
||||
if seg.contains(':') {
|
||||
return Err("first path segment in URL cannot contain colon".into());
|
||||
}
|
||||
}
|
||||
if (!u.scheme.is_empty() || !rest.starts_with("///")) && rest.starts_with("//") {
|
||||
let auth_and_rest = &rest[2..];
|
||||
let (authority, r) = match auth_and_rest.find('/') {
|
||||
Some(i) => (&auth_and_rest[..i], &auth_and_rest[i..]),
|
||||
None => (auth_and_rest, ""),
|
||||
};
|
||||
rest = r;
|
||||
let host = match authority.rfind('@') {
|
||||
Some(i) => {
|
||||
let ui = &authority[..i];
|
||||
if !ui.bytes().all(|c| c.is_ascii_alphanumeric() || b"-._:~!$&'()*+,;=%@".contains(&c)) {
|
||||
return Err("net/url: invalid userinfo".into());
|
||||
}
|
||||
u.user = Some(ui.to_string());
|
||||
&authority[i + 1..]
|
||||
}
|
||||
None => authority,
|
||||
};
|
||||
u.host = parse_host(host)?;
|
||||
} else if !u.scheme.is_empty() && rest.starts_with('/') {
|
||||
u.omit_host = true;
|
||||
}
|
||||
if !rest.is_empty() {
|
||||
u.set_path(rest)?;
|
||||
}
|
||||
Ok(u)
|
||||
}
|
||||
|
||||
fn set_path(&mut self, p: &str) -> Result<(), String> {
|
||||
self.path = unescape(p, Mode::Path)?;
|
||||
self.raw_path = if escape(&self.path, Mode::Path) == p {
|
||||
String::new()
|
||||
} else {
|
||||
p.to_string()
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn escaped_path(&self) -> String {
|
||||
if !self.raw_path.is_empty()
|
||||
&& valid_encoded(&self.raw_path, Mode::Path)
|
||||
&& unescape(&self.raw_path, Mode::Path).ok().as_deref() == Some(&self.path)
|
||||
{
|
||||
return self.raw_path.clone();
|
||||
}
|
||||
if self.path == "*" {
|
||||
return "*".into();
|
||||
}
|
||||
escape(&self.path, Mode::Path)
|
||||
}
|
||||
|
||||
fn escaped_fragment(&self) -> String {
|
||||
if !self.raw_fragment.is_empty()
|
||||
&& valid_encoded(&self.raw_fragment, Mode::Fragment)
|
||||
&& unescape(&self.raw_fragment, Mode::Fragment).ok().as_deref() == Some(&self.fragment)
|
||||
{
|
||||
return self.raw_fragment.clone();
|
||||
}
|
||||
escape(&self.fragment, Mode::Fragment)
|
||||
}
|
||||
|
||||
/// `ref` resolved against `self`, as Go's `URL.Parse`.
|
||||
pub fn join(&self, reference: &str) -> Result<Url, String> {
|
||||
Ok(self.resolve(&Url::parse(reference)?))
|
||||
}
|
||||
|
||||
/// Go's `ResolveReference`.
|
||||
pub fn resolve(&self, r: &Url) -> Url {
|
||||
let mut u = r.clone();
|
||||
if r.scheme.is_empty() {
|
||||
u.scheme = self.scheme.clone();
|
||||
}
|
||||
if !r.scheme.is_empty() || !r.host.is_empty() || r.user.is_some() {
|
||||
let p = resolve_path(&r.escaped_path(), "");
|
||||
let _ = u.set_path(&p);
|
||||
return u;
|
||||
}
|
||||
if !r.opaque.is_empty() {
|
||||
u.user = None;
|
||||
u.host.clear();
|
||||
u.path.clear();
|
||||
return u;
|
||||
}
|
||||
if r.path.is_empty() && !r.force_query && r.raw_query.is_empty() {
|
||||
u.raw_query = self.raw_query.clone();
|
||||
if r.fragment.is_empty() {
|
||||
u.fragment = self.fragment.clone();
|
||||
u.raw_fragment = self.raw_fragment.clone();
|
||||
}
|
||||
}
|
||||
if r.path.is_empty() && !self.opaque.is_empty() {
|
||||
u.opaque = self.opaque.clone();
|
||||
u.user = None;
|
||||
u.host.clear();
|
||||
u.path.clear();
|
||||
return u;
|
||||
}
|
||||
u.host = self.host.clone();
|
||||
u.user = self.user.clone();
|
||||
let p = resolve_path(&self.escaped_path(), &r.escaped_path());
|
||||
let _ = u.set_path(&p);
|
||||
u
|
||||
}
|
||||
|
||||
/// The URL without userinfo, for messages.
|
||||
pub fn redacted(&self) -> String {
|
||||
let mut c = self.clone();
|
||||
c.user = None;
|
||||
c.to_string()
|
||||
}
|
||||
|
||||
pub fn clear_fragment(&mut self) {
|
||||
self.fragment.clear();
|
||||
self.raw_fragment.clear();
|
||||
}
|
||||
|
||||
/// The host without any port, IPv6 brackets removed.
|
||||
pub fn hostname(&self) -> &str {
|
||||
let h = &self.host;
|
||||
if let Some(rest) = h.strip_prefix('[') {
|
||||
return rest.find(']').map_or(rest, |i| &rest[..i]);
|
||||
}
|
||||
match h.rfind(':') {
|
||||
Some(i) if valid_optional_port(&h[i..]) => &h[..i],
|
||||
_ => h,
|
||||
}
|
||||
}
|
||||
|
||||
/// The port, or the scheme's default.
|
||||
pub fn port_or_default(&self) -> &str {
|
||||
let h = &self.host;
|
||||
let after = if h.starts_with('[') {
|
||||
h.find(']').map_or("", |i| &h[i + 1..])
|
||||
} else {
|
||||
h.rfind(':').map_or("", |i| &h[i..])
|
||||
};
|
||||
match after.strip_prefix(':') {
|
||||
Some(p) if !p.is_empty() => p,
|
||||
_ if self.scheme == "https" => "443",
|
||||
_ => "80",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Url {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut b = String::new();
|
||||
if !self.scheme.is_empty() {
|
||||
b.push_str(&self.scheme);
|
||||
b.push(':');
|
||||
}
|
||||
if !self.opaque.is_empty() {
|
||||
b.push_str(&self.opaque);
|
||||
} else {
|
||||
if (!self.scheme.is_empty() || !self.host.is_empty() || self.user.is_some())
|
||||
&& !(self.omit_host && self.host.is_empty() && self.user.is_none())
|
||||
{
|
||||
if !self.host.is_empty() || !self.path.is_empty() || self.user.is_some() {
|
||||
b.push_str("//");
|
||||
}
|
||||
if let Some(ui) = &self.user {
|
||||
b.push_str(ui);
|
||||
b.push('@');
|
||||
}
|
||||
b.push_str(&escape(&self.host, Mode::Host));
|
||||
}
|
||||
let p = self.escaped_path();
|
||||
if !p.is_empty() && !p.starts_with('/') && !self.host.is_empty() {
|
||||
b.push('/');
|
||||
}
|
||||
if b.is_empty() && p.split('/').next().unwrap_or("").contains(':') {
|
||||
b.push_str("./");
|
||||
}
|
||||
b.push_str(&p);
|
||||
}
|
||||
if self.force_query || !self.raw_query.is_empty() {
|
||||
b.push('?');
|
||||
b.push_str(&self.raw_query);
|
||||
}
|
||||
if !self.fragment.is_empty() {
|
||||
b.push('#');
|
||||
b.push_str(&self.escaped_fragment());
|
||||
}
|
||||
f.write_str(&b)
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `resolvePath`: merge and remove dot segments.
|
||||
fn resolve_path(base: &str, r: &str) -> String {
|
||||
let full = if r.is_empty() {
|
||||
base.to_string()
|
||||
} else if !r.starts_with('/') {
|
||||
let i = base.rfind('/').map_or(0, |i| i + 1);
|
||||
format!("{}{}", &base[..i], r)
|
||||
} else {
|
||||
r.to_string()
|
||||
};
|
||||
if full.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let mut dst = String::from("/");
|
||||
let mut first = true;
|
||||
let mut last = "";
|
||||
for elem in full.split('/') {
|
||||
last = elem;
|
||||
if elem == "." {
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
if elem == ".." {
|
||||
let s = dst[1..].to_string();
|
||||
dst = String::from("/");
|
||||
match s.rfind('/') {
|
||||
None => first = true,
|
||||
Some(i) => dst.push_str(&s[..i]),
|
||||
}
|
||||
} else {
|
||||
if !first {
|
||||
dst.push('/');
|
||||
}
|
||||
dst.push_str(elem);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
if last == "." || last == ".." {
|
||||
dst.push('/');
|
||||
}
|
||||
if dst.len() > 1 && dst.as_bytes()[1] == b'/' {
|
||||
dst.remove(0);
|
||||
}
|
||||
dst
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use serde_json::Value;
|
||||
|
||||
pub(crate) fn vectors() -> Value {
|
||||
serde_json::from_str(include_str!("../../tests/vectors/net.json")).unwrap()
|
||||
}
|
||||
|
||||
fn check(v: &Value, got: Result<Url, String>, ctx: &str) {
|
||||
if v["err"].as_bool().unwrap() {
|
||||
assert!(got.is_err(), "{ctx}: accepted, Go refused: {:?}", got.map(|u| u.to_string()));
|
||||
return;
|
||||
}
|
||||
let u = got.unwrap_or_else(|e| panic!("{ctx}: refused ({e}), Go accepted"));
|
||||
let s = |k: &str| v[k].as_str().unwrap_or("").to_string();
|
||||
assert_eq!(u.to_string(), s("string"), "{ctx} String");
|
||||
assert_eq!(u.scheme, s("scheme"), "{ctx} Scheme");
|
||||
assert_eq!(u.host, s("host"), "{ctx} Host");
|
||||
assert_eq!(u.hostname(), s("hostname"), "{ctx} Hostname");
|
||||
assert_eq!(u.path, s("path"), "{ctx} Path");
|
||||
assert_eq!(u.raw_query, s("raw_query"), "{ctx} RawQuery");
|
||||
assert_eq!(u.fragment, s("fragment"), "{ctx} Fragment");
|
||||
assert_eq!(super::super::local_path(&u), s("local_path"), "{ctx} localPath");
|
||||
assert_eq!(super::super::key(&u), s("key"), "{ctx} key");
|
||||
}
|
||||
|
||||
/// Found by fuzzing: a bad escape before a multi-byte character used to
|
||||
/// slice through the character and panic.
|
||||
#[test]
|
||||
fn bad_escape_before_multibyte_character() {
|
||||
assert!(Url::parse("/K%\u{8e3}").is_err());
|
||||
assert!(Url::parse("http://h/%é").is_err());
|
||||
assert!(Url::parse("http://h/%2").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_matches_go() {
|
||||
for v in vectors()["parse"].as_array().unwrap() {
|
||||
let input = v["input"].as_str().unwrap();
|
||||
check(v, Url::parse(input), &format!("Parse({input:?})"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_matches_go() {
|
||||
for v in vectors()["resolve"].as_array().unwrap() {
|
||||
let (b, r) = (v["base"].as_str().unwrap(), v["ref"].as_str().unwrap());
|
||||
let base = Url::parse(b).unwrap();
|
||||
check(&v["result"], base.join(r), &format!("{b:?}.Parse({r:?})"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_escape_matches_go() {
|
||||
for (k, want) in vectors()["query_escape"].as_object().unwrap() {
|
||||
assert_eq!(query_escape(k), want.as_str().unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
+222
@@ -0,0 +1,222 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! One bounded worker pool per process, and the bounded buffer pool that
|
||||
//! object I/O borrows from.
|
||||
//!
|
||||
//! The pool runs a fixed number of threads fed from a bounded queue, so
|
||||
//! submitting blocks once the queue is full (backpressure). Coordinators —
|
||||
//! the code that reads a source, writes an output in order, or collects
|
||||
//! results — always run on the caller's thread, never on a worker, so no
|
||||
//! worker ever waits for another and nested waiting cannot deadlock.
|
||||
//! Dropping the pool closes the queue and joins every thread; jobs already
|
||||
//! queued still run (each is bounded file I/O and crypto), and their
|
||||
//! results are dropped if nobody waits for them. A panicking job is turned
|
||||
//! into an error for whoever waits on it and does not take the worker down.
|
||||
|
||||
use crate::secret::SecretVec;
|
||||
use std::sync::mpsc::{Receiver, SyncSender, sync_channel};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::JoinHandle;
|
||||
|
||||
/// The largest worker count accepted from the user.
|
||||
pub const MAX_WORKERS: usize = 256;
|
||||
/// The default cap: past this, the work is bound by storage, not CPU.
|
||||
pub const DEFAULT_MAX_WORKERS: usize = 8;
|
||||
/// Object-sized buffers in flight at once for one file.
|
||||
pub const OBJECT_BUFFERS: usize = 8;
|
||||
|
||||
type Job = Box<dyn FnOnce() + Send + 'static>;
|
||||
|
||||
pub struct Pool {
|
||||
tx: Option<SyncSender<Job>>,
|
||||
threads: Vec<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl Pool {
|
||||
pub fn new(workers: usize) -> Pool {
|
||||
assert!((1..=MAX_WORKERS).contains(&workers), "worker count out of range");
|
||||
let (tx, rx) = sync_channel::<Job>(workers * 2);
|
||||
let rx = Arc::new(Mutex::new(rx));
|
||||
let threads = (0..workers)
|
||||
.map(|i| {
|
||||
let rx = Arc::clone(&rx);
|
||||
std::thread::Builder::new()
|
||||
.name(format!("vault-worker-{i}"))
|
||||
.spawn(move || {
|
||||
loop {
|
||||
// Hold the lock only to take a job, not to run it.
|
||||
let job = match rx.lock() {
|
||||
Ok(r) => r.recv(),
|
||||
Err(p) => p.into_inner().recv(),
|
||||
};
|
||||
match job {
|
||||
Ok(job) => job(),
|
||||
Err(_) => return,
|
||||
}
|
||||
}
|
||||
})
|
||||
.expect("spawn worker thread")
|
||||
})
|
||||
.collect();
|
||||
Pool { tx: Some(tx), threads }
|
||||
}
|
||||
|
||||
/// The default worker count: available parallelism, at most 8.
|
||||
pub fn default_workers() -> usize {
|
||||
std::thread::available_parallelism().map_or(1, |n| n.get()).min(DEFAULT_MAX_WORKERS)
|
||||
}
|
||||
|
||||
pub fn workers(&self) -> usize {
|
||||
self.threads.len()
|
||||
}
|
||||
|
||||
/// Queues `f`, blocking while the queue is full.
|
||||
pub fn submit<T: Send + 'static>(&self, f: impl FnOnce() -> T + Send + 'static) -> Ticket<T> {
|
||||
let (rtx, rrx) = sync_channel(1);
|
||||
let job: Job = Box::new(move || {
|
||||
let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(f)).map_err(|_| "internal error: a worker panicked".to_string());
|
||||
let _ = rtx.send(r);
|
||||
});
|
||||
self.tx.as_ref().expect("pool open").send(job).expect("workers alive");
|
||||
Ticket(rrx)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Pool {
|
||||
fn drop(&mut self) {
|
||||
drop(self.tx.take());
|
||||
for t in self.threads.drain(..) {
|
||||
let _ = t.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The pending result of a submitted job.
|
||||
pub struct Ticket<T>(Receiver<Result<T, String>>);
|
||||
|
||||
impl<T> Ticket<T> {
|
||||
pub fn wait(self) -> Result<T, String> {
|
||||
self.0.recv().unwrap_or_else(|_| Err("internal error: a worker vanished".into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// A fixed set of reusable, equally sized secret buffers. Taking one blocks
|
||||
/// until one is free; a buffer is cleared when it comes back, and every
|
||||
/// buffer is cleared again when the pool is dropped.
|
||||
#[derive(Clone)]
|
||||
pub struct Buffers {
|
||||
tx: SyncSender<SecretVec>,
|
||||
rx: Arc<Mutex<Receiver<SecretVec>>>,
|
||||
}
|
||||
|
||||
impl Buffers {
|
||||
pub fn new(count: usize, capacity: usize) -> Buffers {
|
||||
let (tx, rx) = sync_channel(count);
|
||||
for _ in 0..count {
|
||||
tx.send(SecretVec::with_capacity(capacity)).expect("room for every buffer");
|
||||
}
|
||||
Buffers {
|
||||
tx,
|
||||
rx: Arc::new(Mutex::new(rx)),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn take(&self) -> Lease {
|
||||
let b = match self.rx.lock() {
|
||||
Ok(r) => r.recv(),
|
||||
Err(p) => p.into_inner().recv(),
|
||||
}
|
||||
.expect("buffer pool alive");
|
||||
Lease {
|
||||
buf: Some(b),
|
||||
home: self.tx.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A borrowed buffer, returned (cleared) on drop — including when a job
|
||||
/// fails or panics.
|
||||
pub struct Lease {
|
||||
buf: Option<SecretVec>,
|
||||
home: SyncSender<SecretVec>,
|
||||
}
|
||||
|
||||
impl Lease {
|
||||
pub fn buf(&mut self) -> &mut SecretVec {
|
||||
self.buf.as_mut().expect("leased")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Lease {
|
||||
fn drop(&mut self) {
|
||||
if let Some(mut b) = self.buf.take() {
|
||||
b.clear();
|
||||
let _ = self.home.send(b);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
#[test]
|
||||
fn results_in_submission_order_and_bounded_concurrency() {
|
||||
let pool = Pool::new(4);
|
||||
let running = Arc::new(AtomicUsize::new(0));
|
||||
let peak = Arc::new(AtomicUsize::new(0));
|
||||
let tickets: Vec<_> = (0..64)
|
||||
.map(|i| {
|
||||
let (running, peak) = (Arc::clone(&running), Arc::clone(&peak));
|
||||
pool.submit(move || {
|
||||
let now = running.fetch_add(1, Ordering::SeqCst) + 1;
|
||||
peak.fetch_max(now, Ordering::SeqCst);
|
||||
std::thread::sleep(Duration::from_millis((64 - i) % 5));
|
||||
running.fetch_sub(1, Ordering::SeqCst);
|
||||
i
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
let got: Vec<_> = tickets.into_iter().map(|t| t.wait().unwrap()).collect();
|
||||
assert_eq!(got, (0..64).collect::<Vec<_>>());
|
||||
assert!(peak.load(Ordering::SeqCst) <= 4);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn panics_become_errors() {
|
||||
let pool = Pool::new(2);
|
||||
let t = pool.submit(|| -> u8 { panic!("boom") });
|
||||
assert!(t.wait().is_err());
|
||||
assert_eq!(pool.submit(|| 5).wait().unwrap(), 5, "worker survived");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_tickets_and_pool_cleans_up() {
|
||||
for _ in 0..20 {
|
||||
let pool = Pool::new(3);
|
||||
for _ in 0..10 {
|
||||
drop(pool.submit(|| std::thread::sleep(Duration::from_millis(1))));
|
||||
}
|
||||
drop(pool); // joins: no threads outlive it
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn buffers_are_bounded_and_cleared() {
|
||||
let b = Buffers::new(2, 8);
|
||||
let mut l1 = b.take();
|
||||
l1.buf().extend_within_capacity(b"secret");
|
||||
let _l2 = b.take();
|
||||
let b2 = b.clone();
|
||||
let waiter = std::thread::spawn(move || {
|
||||
let mut l = b2.take();
|
||||
l.buf().len()
|
||||
});
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
assert!(!waiter.is_finished(), "a third lease should wait");
|
||||
drop(l1);
|
||||
assert_eq!(waiter.join().unwrap(), 0, "returned buffers are cleared");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,530 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Filesystem primitives with the guarantees the store and extraction need.
|
||||
//!
|
||||
//! - [`publish`] moves a finished temporary file to its final name without
|
||||
//! ever replacing anything there, symlinks (dangling or not) included:
|
||||
//! `renameat2(RENAME_NOREPLACE)` on Linux, `renameatx_np(RENAME_EXCL)` on
|
||||
//! macOS, `link`+`unlink` on other Unix systems and when a Linux
|
||||
//! filesystem lacks RENAME_NOREPLACE, and `MoveFileExW` without
|
||||
//! `MOVEFILE_REPLACE_EXISTING` on Windows. There is no check-then-rename
|
||||
//! fallback: if no atomic primitive works, publishing fails.
|
||||
//! - [`Dir`] creates and publishes files relative to an open directory. On
|
||||
//! Unix every path component below the chosen destination is opened with
|
||||
//! `O_NOFOLLOW|O_DIRECTORY`, so a symlink planted inside the destination
|
||||
//! cannot redirect extraction elsewhere. Windows has no `openat`; there
|
||||
//! each component is checked not to be a symlink or junction before use,
|
||||
//! which narrows but does not close the race (see SECURITY.md).
|
||||
//! - [`sync_dir`] makes a directory's entries durable where the OS allows
|
||||
//! it (Unix). Windows offers no directory fsync through these APIs; NTFS
|
||||
//! journals metadata, and that is the durability on offer there.
|
||||
|
||||
use std::ffi::OsStr;
|
||||
use std::fs::{File, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt};
|
||||
|
||||
pub fn create_private_dir_all(p: &Path) -> io::Result<()> {
|
||||
let mut b = std::fs::DirBuilder::new();
|
||||
b.recursive(true);
|
||||
#[cfg(unix)]
|
||||
b.mode(0o700);
|
||||
b.create(p)
|
||||
}
|
||||
|
||||
pub fn create_private_dir(p: &Path) -> io::Result<()> {
|
||||
#[cfg_attr(not(unix), allow(unused_mut))]
|
||||
let mut b = std::fs::DirBuilder::new();
|
||||
#[cfg(unix)]
|
||||
b.mode(0o700);
|
||||
b.create(p)
|
||||
}
|
||||
|
||||
/// Creates a new file readable only by its owner; fails if `p` exists.
|
||||
pub fn create_new_private(p: &Path) -> io::Result<File> {
|
||||
let mut o = OpenOptions::new();
|
||||
o.write(true).create_new(true);
|
||||
#[cfg(unix)]
|
||||
o.mode(0o600);
|
||||
o.open(p)
|
||||
}
|
||||
|
||||
pub fn sync_dir(dir: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
File::open(dir)?.sync_all()
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let _ = dir;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets a path's access and modification times to the Unix epoch,
|
||||
/// following symlinks, as Go's `os.Chtimes(p, epoch, epoch)`.
|
||||
pub fn set_epoch(p: &Path) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{AtFlags, CWD, Timespec, Timestamps, utimensat};
|
||||
let z = Timespec { tv_sec: 0, tv_nsec: 0 };
|
||||
utimensat(
|
||||
CWD,
|
||||
p,
|
||||
&Timestamps {
|
||||
last_access: z,
|
||||
last_modification: z,
|
||||
},
|
||||
AtFlags::empty(),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let f = open_for_times(p)?;
|
||||
f.set_times(
|
||||
std::fs::FileTimes::new()
|
||||
.set_accessed(std::time::UNIX_EPOCH)
|
||||
.set_modified(std::time::UNIX_EPOCH),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn open_for_times(p: &Path) -> io::Result<File> {
|
||||
use std::os::windows::fs::OpenOptionsExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{FILE_FLAG_BACKUP_SEMANTICS, FILE_WRITE_ATTRIBUTES};
|
||||
OpenOptions::new()
|
||||
.access_mode(FILE_WRITE_ATTRIBUTES)
|
||||
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS)
|
||||
.open(p)
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn open_for_times(p: &Path) -> io::Result<File> {
|
||||
OpenOptions::new().write(true).open(p)
|
||||
}
|
||||
|
||||
fn random_suffix() -> String {
|
||||
let b: [u8; 8] = crate::rng::os().array();
|
||||
b.iter().map(|x| format!("{x:02x}")).collect()
|
||||
}
|
||||
|
||||
/// Creates `dir/<prefix><random>` exclusively, mode 0600.
|
||||
pub fn create_temp(dir: &Path, prefix: &str) -> io::Result<(File, PathBuf)> {
|
||||
loop {
|
||||
let p = dir.join(format!("{prefix}{}", random_suffix()));
|
||||
match create_new_private(&p) {
|
||||
Ok(f) => return Ok((f, p)),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn exists_error(dst: &Path) -> io::Error {
|
||||
io::Error::new(io::ErrorKind::AlreadyExists, format!("{} already exists", dst.display()))
|
||||
}
|
||||
|
||||
/// Renames `tmp` to `dst` only if `dst` does not exist (in any form). On
|
||||
/// failure `tmp` is left for the caller to remove.
|
||||
pub fn publish(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
let r = publish_inner(tmp, dst);
|
||||
match r {
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => Err(exists_error(dst)),
|
||||
r => r,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))]
|
||||
fn publish_inner(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
use rustix::fs::{CWD, RenameFlags, renameat_with};
|
||||
match renameat_with(CWD, tmp, CWD, dst, RenameFlags::NOREPLACE) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(e) if e == rustix::io::Errno::INVAL || e == rustix::io::Errno::NOSYS || e == rustix::io::Errno::NOTSUP => {
|
||||
link_publish(tmp, dst)
|
||||
}
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(unix, not(any(target_os = "linux", target_os = "android", target_vendor = "apple"))))]
|
||||
fn publish_inner(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
link_publish(tmp, dst)
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn publish_inner(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
link_publish(tmp, dst)
|
||||
}
|
||||
|
||||
/// `link` never replaces an existing name, so link-then-unlink is an
|
||||
/// atomic no-clobber publish wherever hard links work.
|
||||
#[cfg(not(windows))]
|
||||
fn link_publish(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
std::fs::hard_link(tmp, dst).map_err(no_atomic)?;
|
||||
// The data is safely at dst; failing to drop the temporary name is only
|
||||
// worth a warning.
|
||||
if let Err(e) = std::fs::remove_file(tmp) {
|
||||
eprintln!(
|
||||
"vault: could not remove temporary file {}: {}",
|
||||
tmp.display(),
|
||||
crate::util::io_reason(&e)
|
||||
);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A filesystem with neither RENAME_NOREPLACE nor hard links cannot publish
|
||||
/// without risking a replace; say so rather than fall back.
|
||||
#[cfg(not(windows))]
|
||||
fn no_atomic(e: io::Error) -> io::Error {
|
||||
if e.kind() == io::ErrorKind::AlreadyExists {
|
||||
return e;
|
||||
}
|
||||
io::Error::new(
|
||||
e.kind(),
|
||||
format!(
|
||||
"this filesystem offers no atomic no-replace rename or hard link ({})",
|
||||
crate::util::io_reason(&e)
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn publish_inner(tmp: &Path, dst: &Path) -> io::Result<()> {
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{MOVEFILE_WRITE_THROUGH, MoveFileExW};
|
||||
let w = |p: &Path| -> Vec<u16> { p.as_os_str().encode_wide().chain(Some(0)).collect() };
|
||||
let (a, b) = (w(tmp), w(dst));
|
||||
// SAFETY: both arguments are NUL-terminated UTF-16 strings that outlive
|
||||
// the call; MoveFileExW does not retain them.
|
||||
let ok = unsafe { MoveFileExW(a.as_ptr(), b.as_ptr(), MOVEFILE_WRITE_THROUGH) };
|
||||
if ok == 0 { Err(io::Error::last_os_error()) } else { Ok(()) }
|
||||
}
|
||||
|
||||
/// Sets permission bits as Go's `File.Chmod` does: all of `mode & 0o777`
|
||||
/// on Unix, only the read-only attribute on Windows.
|
||||
pub fn set_mode(f: &File, mode: u32) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
f.set_permissions(std::fs::Permissions::from_mode(mode & 0o777))
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
let mut p = f.metadata()?.permissions();
|
||||
p.set_readonly(mode & 0o200 == 0);
|
||||
f.set_permissions(p)
|
||||
}
|
||||
}
|
||||
|
||||
/// A destination directory that files are created and published inside
|
||||
/// without following symlinks below it.
|
||||
pub struct Dir {
|
||||
path: PathBuf,
|
||||
#[cfg(unix)]
|
||||
fd: rustix::fd::OwnedFd,
|
||||
}
|
||||
|
||||
impl Dir {
|
||||
/// Opens the chosen destination itself; a symlink here is followed,
|
||||
/// because the user named it. An empty path is the current directory
|
||||
/// (and names inside it are shown without a "./").
|
||||
pub fn open(path: &Path) -> io::Result<Dir> {
|
||||
let real = if path.as_os_str().is_empty() { Path::new(".") } else { path };
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{CWD, Mode, OFlags, openat};
|
||||
let fd = openat(CWD, real, OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty())?;
|
||||
Ok(Dir {
|
||||
path: path.to_path_buf(),
|
||||
fd,
|
||||
})
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
if !std::fs::metadata(real)?.is_dir() {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::NotADirectory,
|
||||
format!("{} is not a directory", path.display()),
|
||||
));
|
||||
}
|
||||
Ok(Dir { path: path.to_path_buf() })
|
||||
}
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Opens (creating with mode 0700 if missing) the subdirectory `name`,
|
||||
/// refusing a symlink or anything that is not a directory.
|
||||
pub fn subdir(&self, name: &OsStr) -> io::Result<Dir> {
|
||||
check_component(name)?;
|
||||
let path = self.path.join(name);
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{Mode, OFlags, mkdirat, openat};
|
||||
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
|
||||
let open = || openat(&self.fd, name, flags, Mode::empty());
|
||||
let fd = match open() {
|
||||
Ok(fd) => fd,
|
||||
Err(rustix::io::Errno::NOENT) => {
|
||||
match mkdirat(&self.fd, name, Mode::from_raw_mode(0o700)) {
|
||||
Ok(()) | Err(rustix::io::Errno::EXIST) => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
open().map_err(|e| refuse(e, &path))?
|
||||
}
|
||||
Err(e) => return Err(refuse(e, &path)),
|
||||
};
|
||||
Ok(Dir { path, fd })
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
match std::fs::symlink_metadata(&path) {
|
||||
Ok(m) if m.is_dir() && !m.file_type().is_symlink() && !is_reparse(&m) => {}
|
||||
Ok(_) => {
|
||||
return Err(io::Error::other(format!("{} is not a directory, or is a link", path.display())));
|
||||
}
|
||||
Err(e) if e.kind() == io::ErrorKind::NotFound => match create_private_dir(&path) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => return self.subdir(name),
|
||||
Err(e) => return Err(e),
|
||||
},
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
Ok(Dir { path })
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a private temporary file in this directory.
|
||||
pub fn create_temp(&self, prefix: &str) -> io::Result<(File, std::ffi::OsString)> {
|
||||
loop {
|
||||
let name: std::ffi::OsString = format!("{prefix}{}", random_suffix()).into();
|
||||
#[cfg(unix)]
|
||||
let r = {
|
||||
use rustix::fs::{Mode, OFlags, openat};
|
||||
openat(
|
||||
&self.fd,
|
||||
&name,
|
||||
OFlags::WRONLY | OFlags::CREATE | OFlags::EXCL | OFlags::NOFOLLOW | OFlags::CLOEXEC,
|
||||
Mode::from_raw_mode(0o600),
|
||||
)
|
||||
.map(File::from)
|
||||
.map_err(io::Error::from)
|
||||
};
|
||||
#[cfg(not(unix))]
|
||||
let r = create_new_private(&self.path.join(&name));
|
||||
match r {
|
||||
Ok(f) => return Ok((f, name)),
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => continue,
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes `tmp` as `name` in this directory, never replacing
|
||||
/// anything already there.
|
||||
pub fn publish(&self, tmp: &OsStr, name: &OsStr) -> io::Result<()> {
|
||||
check_component(name)?;
|
||||
let dst = self.path.join(name);
|
||||
#[cfg(any(target_os = "linux", target_os = "android", target_vendor = "apple"))]
|
||||
{
|
||||
use rustix::fs::{RenameFlags, renameat_with};
|
||||
match renameat_with(&self.fd, tmp, &self.fd, name, RenameFlags::NOREPLACE) {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(rustix::io::Errno::EXIST) => return Err(exists_error(&dst)),
|
||||
Err(e) if e == rustix::io::Errno::INVAL || e == rustix::io::Errno::NOSYS || e == rustix::io::Errno::NOTSUP => {}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use rustix::fs::{AtFlags, linkat, unlinkat};
|
||||
match linkat(&self.fd, tmp, &self.fd, name, AtFlags::empty()) {
|
||||
Ok(()) => {}
|
||||
Err(rustix::io::Errno::EXIST) => return Err(exists_error(&dst)),
|
||||
Err(e) => return Err(no_atomic(e.into())),
|
||||
}
|
||||
if let Err(e) = unlinkat(&self.fd, tmp, AtFlags::empty()) {
|
||||
eprintln!("vault: could not remove temporary file {}: {e}", self.path.join(tmp).display());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
publish(&self.path.join(tmp), &dst)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn remove(&self, name: &OsStr) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
rustix::fs::unlinkat(&self.fd, name, rustix::fs::AtFlags::empty())?;
|
||||
Ok(())
|
||||
}
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
std::fs::remove_file(self.path.join(name))
|
||||
}
|
||||
}
|
||||
|
||||
pub fn sync(&self) -> io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
rustix::fs::fsync(&self.fd)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn refuse(e: rustix::io::Errno, path: &Path) -> io::Error {
|
||||
if e == rustix::io::Errno::LOOP || e == rustix::io::Errno::NOTDIR {
|
||||
io::Error::other(format!(
|
||||
"{} is not a directory, or is a symlink; refusing to extract through it",
|
||||
path.display()
|
||||
))
|
||||
} else {
|
||||
e.into()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn is_reparse(m: &std::fs::Metadata) -> bool {
|
||||
use std::os::windows::fs::MetadataExt;
|
||||
m.file_attributes() & windows_sys::Win32::Storage::FileSystem::FILE_ATTRIBUTE_REPARSE_POINT != 0
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn is_reparse(_: &std::fs::Metadata) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// On Windows, also refuse names NTFS would treat specially: alternate
|
||||
/// data streams (`a:b`), reserved device names, trailing dots or spaces.
|
||||
fn windows_safe(name: &OsStr) -> bool {
|
||||
if !cfg!(windows) {
|
||||
return true;
|
||||
}
|
||||
let Some(n) = name.to_str() else { return false };
|
||||
let stem = n.split('.').next().unwrap_or("").trim_end().to_ascii_uppercase();
|
||||
let reserved = matches!(stem.as_str(), "CON" | "PRN" | "AUX" | "NUL" | "CONIN$" | "CONOUT$")
|
||||
|| ((stem.starts_with("COM") || stem.starts_with("LPT")) && stem.len() == 4 && stem.as_bytes()[3].is_ascii_digit());
|
||||
!n.contains([':', '<', '>', '"', '|', '?', '*']) && !n.ends_with(['.', ' ']) && !reserved
|
||||
}
|
||||
|
||||
/// A single, plain path component: no separators, not "." or "..".
|
||||
fn check_component(name: &OsStr) -> io::Result<()> {
|
||||
let ok = !name.is_empty()
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& Path::new(name).components().count() == 1
|
||||
&& matches!(Path::new(name).components().next(), Some(std::path::Component::Normal(_)))
|
||||
&& windows_safe(name);
|
||||
if ok {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!("invalid path component {:?}", name),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn tmpdir(tag: &str) -> PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("vault-fs-{tag}-{}-{}", std::process::id(), random_suffix()));
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn publish_never_replaces() {
|
||||
let d = tmpdir("pub");
|
||||
let (_, t1) = create_temp(&d, ".t-").unwrap();
|
||||
std::fs::write(&t1, b"one").unwrap();
|
||||
publish(&t1, &d.join("out")).unwrap();
|
||||
let (_, t2) = create_temp(&d, ".t-").unwrap();
|
||||
std::fs::write(&t2, b"two").unwrap();
|
||||
let e = publish(&t2, &d.join("out")).unwrap_err();
|
||||
assert_eq!(e.kind(), io::ErrorKind::AlreadyExists);
|
||||
assert_eq!(std::fs::read(d.join("out")).unwrap(), b"one");
|
||||
#[cfg(unix)]
|
||||
{
|
||||
std::os::unix::fs::symlink(d.join("nowhere"), d.join("dangling")).unwrap();
|
||||
assert_eq!(publish(&t2, &d.join("dangling")).unwrap_err().kind(), io::ErrorKind::AlreadyExists);
|
||||
assert!(!d.join("nowhere").exists(), "followed a dangling symlink");
|
||||
}
|
||||
std::fs::remove_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
/// Many threads race to publish the same name: exactly one wins and
|
||||
/// every loser's file is left for it to clean up.
|
||||
#[test]
|
||||
fn concurrent_publishers() {
|
||||
let d = tmpdir("race");
|
||||
let wins = std::sync::atomic::AtomicUsize::new(0);
|
||||
std::thread::scope(|s| {
|
||||
for i in 0..16 {
|
||||
let (d, wins) = (&d, &wins);
|
||||
s.spawn(move || {
|
||||
let dir = Dir::open(d).unwrap();
|
||||
let (mut f, t) = dir.create_temp(".t-").unwrap();
|
||||
std::io::Write::write_all(&mut f, format!("{i}").as_bytes()).unwrap();
|
||||
match dir.publish(&t, OsStr::new("same")) {
|
||||
Ok(()) => {
|
||||
wins.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
Err(e) => {
|
||||
assert_eq!(e.kind(), io::ErrorKind::AlreadyExists);
|
||||
dir.remove(&t).unwrap();
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
assert_eq!(wins.into_inner(), 1);
|
||||
assert_eq!(std::fs::read_dir(&d).unwrap().count(), 1);
|
||||
std::fs::remove_dir_all(d).unwrap();
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn subdir_refuses_symlinks() {
|
||||
let d = tmpdir("conf");
|
||||
let outside = tmpdir("outside");
|
||||
std::os::unix::fs::symlink(&outside, d.join("evil")).unwrap();
|
||||
let dir = Dir::open(&d).unwrap();
|
||||
assert!(dir.subdir(OsStr::new("evil")).is_err());
|
||||
assert!(dir.subdir(OsStr::new("..")).is_err());
|
||||
assert!(dir.subdir(OsStr::new("a/b")).is_err());
|
||||
let sub = dir.subdir(OsStr::new("ok")).unwrap();
|
||||
assert!(sub.path().ends_with("ok"));
|
||||
std::fs::write(d.join("file"), b"").unwrap();
|
||||
assert!(dir.subdir(OsStr::new("file")).is_err());
|
||||
assert_eq!(std::fs::read_dir(&outside).unwrap().count(), 0);
|
||||
std::fs::remove_dir_all(d).unwrap();
|
||||
std::fs::remove_dir_all(outside).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn epoch_times() {
|
||||
let d = tmpdir("epoch");
|
||||
let f = d.join("f");
|
||||
std::fs::write(&f, b"x").unwrap();
|
||||
set_epoch(&f).unwrap();
|
||||
set_epoch(&d).unwrap();
|
||||
for p in [&f, &d] {
|
||||
assert_eq!(std::fs::metadata(p).unwrap().modified().unwrap(), std::time::UNIX_EPOCH);
|
||||
}
|
||||
std::fs::remove_dir_all(d).unwrap();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! The vault lock on `DIR/.lock`, interoperable with the Go version on the
|
||||
//! same store:
|
||||
//!
|
||||
//! - Unix: classic POSIX record locks (`fcntl(F_SETLK/F_SETLKW)` over the
|
||||
//! whole file), exactly what Go uses. Not `flock(2)`, which does not
|
||||
//! exclude `fcntl` locks on Linux, and not OFD locks. The kernel drops
|
||||
//! them when the process dies. POSIX record locks belong to the process,
|
||||
//! so this process must hold the lock file open only once.
|
||||
//! - Windows: `LockFileEx` on byte 0, as Go does; released on death.
|
||||
//! - Elsewhere: an exclusive `O_EXCL` lock file. A crash leaves it behind;
|
||||
//! after a minute of waiting the error names it for manual removal.
|
||||
//!
|
||||
//! Readers take the lock shared, writers exclusive. If it is held in a
|
||||
//! conflicting mode, `waiting` is called once and the call blocks.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
|
||||
pub const LOCK_NAME: &str = ".lock";
|
||||
|
||||
/// A held lock; dropping it releases the lock.
|
||||
pub struct Lock {
|
||||
_file: Option<File>,
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
remove: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
impl Drop for Lock {
|
||||
fn drop(&mut self) {
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
if let Some(p) = self.remove.take() {
|
||||
let _ = std::fs::remove_file(p);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
pub fn acquire(path: &Path, exclusive: bool, waiting: &mut dyn FnMut()) -> io::Result<Lock> {
|
||||
use rustix::fs::{FlockOperation, fcntl_lock};
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
let f = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.mode(0o600)
|
||||
.open(path)?;
|
||||
let (try_op, wait_op) = if exclusive {
|
||||
(FlockOperation::NonBlockingLockExclusive, FlockOperation::LockExclusive)
|
||||
} else {
|
||||
(FlockOperation::NonBlockingLockShared, FlockOperation::LockShared)
|
||||
};
|
||||
match fcntl_lock(&f, try_op) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e == rustix::io::Errno::AGAIN || e == rustix::io::Errno::ACCESS => {
|
||||
waiting();
|
||||
loop {
|
||||
match fcntl_lock(&f, wait_op) {
|
||||
Ok(()) => break,
|
||||
Err(rustix::io::Errno::INTR) => continue,
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
}
|
||||
Ok(Lock { _file: Some(f) })
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
pub fn acquire(path: &Path, exclusive: bool, waiting: &mut dyn FnMut()) -> io::Result<Lock> {
|
||||
use std::os::windows::io::AsRawHandle;
|
||||
use windows_sys::Win32::Foundation::ERROR_LOCK_VIOLATION;
|
||||
use windows_sys::Win32::Storage::FileSystem::{LOCKFILE_EXCLUSIVE_LOCK, LOCKFILE_FAIL_IMMEDIATELY, LockFileEx};
|
||||
use windows_sys::Win32::System::IO::OVERLAPPED;
|
||||
let f = std::fs::OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(path)?;
|
||||
let flags = if exclusive { LOCKFILE_EXCLUSIVE_LOCK } else { 0 };
|
||||
let lock = |flags: u32| -> io::Result<()> {
|
||||
// SAFETY: the handle is valid for the life of `f`; the OVERLAPPED is
|
||||
// zeroed (offset 0) and outlives this synchronous call.
|
||||
let mut ov: OVERLAPPED = unsafe { std::mem::zeroed() };
|
||||
let ok = unsafe { LockFileEx(f.as_raw_handle() as _, flags, 0, 1, 0, &mut ov) };
|
||||
if ok == 0 { Err(io::Error::last_os_error()) } else { Ok(()) }
|
||||
};
|
||||
match lock(flags | LOCKFILE_FAIL_IMMEDIATELY) {
|
||||
Ok(()) => {}
|
||||
Err(e) if e.raw_os_error() == Some(ERROR_LOCK_VIOLATION as i32) => {
|
||||
waiting();
|
||||
lock(flags)?;
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
Ok(Lock { _file: Some(f) })
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub fn acquire(path: &Path, _exclusive: bool, waiting: &mut dyn FnMut()) -> io::Result<Lock> {
|
||||
for i in 0.. {
|
||||
match super::fs::create_new_private(path) {
|
||||
Ok(_) => {
|
||||
return Ok(Lock {
|
||||
_file: None,
|
||||
remove: Some(path.to_path_buf()),
|
||||
});
|
||||
}
|
||||
Err(e) if e.kind() == io::ErrorKind::AlreadyExists => {}
|
||||
Err(e) => return Err(e),
|
||||
}
|
||||
if i == 0 {
|
||||
waiting();
|
||||
}
|
||||
if i == 600 {
|
||||
return Err(io::Error::other(format!(
|
||||
"vault still locked after a minute; if no other vault process is running, remove {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(100));
|
||||
}
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
/// Locks the vault at `root`. A shared (reader) lock on a read-only
|
||||
/// filesystem, where nothing can be writing, proceeds without a lock; every
|
||||
/// other failure is reported.
|
||||
pub fn lock_vault(root: &Path, exclusive: bool) -> Result<Lock, String> {
|
||||
let path = root.join(LOCK_NAME);
|
||||
let mut waiting = || eprintln!("waiting for another vault process to finish");
|
||||
match acquire(&path, exclusive, &mut waiting) {
|
||||
Ok(l) => Ok(l),
|
||||
Err(e) if !exclusive && e.kind() == io::ErrorKind::ReadOnlyFilesystem => Ok(Lock {
|
||||
_file: None,
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
remove: None,
|
||||
}),
|
||||
Err(e) => Err(format!("cannot lock the vault: {}", crate::util::io_msg("lock", &path, &e))),
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user