Crash and leak tests; platform matrix; ppc64 via cross; CI runs builds under qemu

Co-Authored-By: Claude <noreply@anthropic.com>

.github/workflows/ci.yml

@@ -51,6 +51,10 @@ jobs:
     runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v5
+      - uses: actions/setup-go@v6
+        with:
+          go-version: "1.26.x"
+          cache-dependency-path: oracle/go/go.sum
       - run: rustup show active-toolchain || rustup toolchain install
       - name: zig and cargo-zigbuild
         run: |
@@ -59,22 +63,28 @@ jobs:
           mkdir -p "$HOME/zig" && tar -xJf zig.tar.xz -C "$HOME/zig" --strip-components=1
           echo "$HOME/zig" >>"$GITHUB_PATH"
           cargo install cargo-zigbuild --version 0.23.4 --locked
+          cargo install cross --version 0.2.5 --locked
       - name: targets
         run: |
           rustup target add x86_64-unknown-linux-musl aarch64-unknown-linux-musl i686-unknown-linux-musl \
             arm-unknown-linux-musleabihf armv7-unknown-linux-musleabihf riscv64gc-unknown-linux-musl \
             loongarch64-unknown-linux-musl powerpc64le-unknown-linux-musl powerpc64-unknown-linux-gnu \
-            s390x-unknown-linux-gnu x86_64-pc-windows-gnu i686-pc-windows-gnu aarch64-pc-windows-gnullvm wasm32-wasip1
+            s390x-unknown-linux-gnu x86_64-pc-windows-gnu i686-pc-windows-gnu aarch64-pc-windows-gnullvm wasm32-wasip1 \
+            x86_64-unknown-freebsd i686-unknown-freebsd x86_64-unknown-netbsd x86_64-unknown-illumos x86_64-pc-solaris
       - name: build
         run: |
+          # Everything but macOS, which the test job builds on a Mac.
           TARGETS="linux/amd64 linux/arm64 linux/386 linux/armv6 linux/armv7 linux/riscv64 linux/loong64 \
-            linux/ppc64le linux/ppc64 linux/s390x windows/amd64 windows/386 windows/arm64 wasip1/wasm" ./build.sh
+            linux/ppc64le linux/ppc64 linux/s390x android/arm64 windows/amd64 windows/386 windows/arm64 \
-      - name: run the arm64 and big-endian s390x builds under qemu
+            freebsd/amd64 freebsd/386 netbsd/amd64 illumos/amd64 solaris/amd64 wasip1/wasm" ./build.sh
+      - name: run builds under qemu, cross-checked with the Go oracle
         run: |
-          sudo apt-get update -q && sudo apt-get install -yq qemu-user
+          sudo apt-get update -q && sudo apt-get install -yq qemu-user libc6-s390x-cross libc6-ppc64-cross
-          cargo build --locked --release
+          (cd oracle/go && go build -o ../../target/vault-go .)
-          EMU=qemu-aarch64 scripts/smoke.sh dist/vault-linux-arm64 target/release/vault
+          for t in "arm64 qemu-aarch64" "armv7 qemu-arm" "riscv64 qemu-riscv64" "ppc64le qemu-ppc64le" \
-          EMU="qemu-s390x -L /usr/s390x-linux-gnu" scripts/smoke.sh dist/vault-linux-s390x target/release/vault || echo "s390x needs a glibc sysroot; build-only"
+                   "s390x qemu-s390x -L /usr/s390x-linux-gnu" "ppc64 qemu-ppc64 -L /usr/powerpc64-linux-gnu"; do
+            EMU="${t#* }" scripts/smoke.sh "dist/vault-linux-${t%% *}" target/vault-go
+          done
       - uses: actions/upload-artifact@v4
         with:
           name: dist

build.sh

@@ -15,7 +15,9 @@
 # How each target is built, and what the host needs:
 #   zig    cargo-zigbuild and zig on PATH  (Linux, Windows GNU ABI)
 #   mac    a Mac with Xcode                (macOS)
-#   cross  `cross` and Docker               (FreeBSD, NetBSD, illumos, Solaris)
+#   cross  `cross` and Docker               (FreeBSD, NetBSD, illumos, Solaris,
+#                                            and big-endian ppc64, which zig
+#                                            links for the wrong ABI)
 #   cargo  nothing extra                    (WASI)
 # A requested target the host cannot build is reported, the others are still
 # built, and the script exits non-zero. See docs/PLATFORMS.md for the Go
@@ -38,7 +40,7 @@ table() {
 	linux/riscv64     riscv64gc-unknown-linux-musl      zig
 	linux/loong64     loongarch64-unknown-linux-musl    zig
 	linux/ppc64le     powerpc64le-unknown-linux-musl    zig
-	linux/ppc64       powerpc64-unknown-linux-gnu       zig
+	linux/ppc64       powerpc64-unknown-linux-gnu       cross
 	linux/s390x       s390x-unknown-linux-gnu           zig
 	android/arm64     aarch64-unknown-linux-musl        zig
 	windows/amd64     x86_64-pc-windows-gnu             zig

docs/PLATFORMS.md

@@ -0,0 +1,85 @@
+# Platforms
+
+The Go version was built for every target of `go tool dist list` that needs
+no cgo: 43 binaries. Rust supports fewer operating systems and CPUs with a
+prebuilt, stable standard library. This is the target-by-target record:
+how each is built, and how far it has actually been verified. "Runtime" means
+a binary was executed; "build" means it only compiled and linked.
+
+Tooling used for verification on the development machine (x86-64 Linux):
+the pinned Rust 1.99.0, zig 0.16.0 with cargo-zigbuild 0.23.4, qemu-user
+10.2.1 (Ubuntu package, unpacked locally, no system changes), glibc cross
+sysroots from Ubuntu's `libc6-*-cross` packages, and the Go oracle built for
+x86-64. `scripts/smoke.sh` is the runtime check: create a key and vault, store
+files across object boundaries, read them back, then have the Go oracle read
+what that binary wrote and the binary read what the oracle wrote.
+
+## Supported
+
+| Go name | Rust target | Linking | Built by | Verified |
+| --- | --- | --- | --- | --- |
+| linux/amd64 | x86_64-unknown-linux-musl | static | zig | runtime: full test suite (glibc host build) and smoke test of the static binary |
+| linux/arm64 | aarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-aarch64, cross-checked with the Go oracle |
+| linux/386 | i686-unknown-linux-musl | static | zig | runtime: smoke test under qemu-i386, cross-checked |
+| linux/armv6 | arm-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
+| linux/armv7 | armv7-unknown-linux-musleabihf | static | zig | runtime: smoke test under qemu-arm, cross-checked |
+| linux/riscv64 | riscv64gc-unknown-linux-musl | static | zig | runtime: smoke test under qemu-riscv64, cross-checked; unit tests under qemu (see note 1) |
+| linux/loong64 | loongarch64-unknown-linux-musl | static | zig | runtime: smoke test under qemu-loongarch64, cross-checked |
+| linux/ppc64le | powerpc64le-unknown-linux-musl | static | zig | runtime: smoke test under qemu-ppc64le, cross-checked |
+| linux/s390x | s390x-unknown-linux-gnu | glibc, dynamic | zig | runtime: smoke test under qemu-s390x (big-endian) with an Ubuntu glibc sysroot, cross-checked |
+| linux/ppc64 | powerpc64-unknown-linux-gnu | glibc, dynamic | cross (Docker) | see note 2 |
+| android/arm64 | aarch64-unknown-linux-musl | static | zig | build only on Android; it is the linux/arm64 binary (see note 3) |
+| windows/amd64 | x86_64-pc-windows-gnu | static CRT | zig | build; tests cross-compile cleanly; runtime in CI only (windows-2025 job, not yet run) |
+| windows/386 | i686-pc-windows-gnu | static CRT | zig | build only |
+| windows/arm64 | aarch64-pc-windows-gnullvm | static CRT | zig | build only |
+| darwin/amd64 | x86_64-apple-darwin | system | a Mac (Xcode) | CI only (macos-15-intel job, not yet run) |
+| darwin/arm64 | aarch64-apple-darwin | system | a Mac (Xcode) | CI only (macos-15 job, not yet run) |
+| freebsd/amd64 | x86_64-unknown-freebsd | system | cross (Docker) | see note 4 |
+| freebsd/386 | i686-unknown-freebsd | system | cross (Docker) | see note 4 |
+| netbsd/amd64 | x86_64-unknown-netbsd | system | cross (Docker) | see note 4 |
+| illumos/amd64 | x86_64-unknown-illumos | system | cross (Docker) | see note 4 |
+| solaris/amd64 | x86_64-pc-solaris | system | cross (Docker) | see note 4 |
+| wasip1/wasm | wasm32-wasip1 | n/a | cargo | build only; no agent, no terminal prompts (passphrase from the environment), lock file instead of locks |
+
+Feature notes: the `unlock` agent works on Unix only (Linux, macOS, the BSDs,
+illumos, Solaris); Windows asks for the passphrase every time. Full agent
+hardening (`PR_SET_DUMPABLE`) is Linux-only; elsewhere the agent locks its
+memory and disables core dumps. See `SECURITY.md`.
+
+1. zig's bundled musl `getrandom()` never reaches the kernel on riscv64 (the
+   process spins). `build.sh` and `packaging/package.sh` build that target
+   with `--cfg getrandom_backend="linux_raw"`, which makes the `getrandom`
+   crate issue the system call itself; with that flag the binary passes.
+2. zig links big-endian ppc64 for the ELFv2 ABI (`/lib64/ld64.so.2`), but
+   big-endian glibc distributions use ELFv1 (`ld64.so.1`), so the zig build
+   would not start there. `build.sh` builds it with `cross` (GCC toolchain)
+   instead.
+3. Go's android/arm64 binary was a static Linux binary used in Termux; this
+   is the same. Name resolution for `add URL`/`mirror` reads
+   `/etc/resolv.conf`, which Android does not have, so those two commands
+   may fail to resolve hosts there; everything else is unaffected.
+4. Built with `cross` 0.2.5 images; not executed (no emulator). The
+   dependency graph supports these systems (rustix, ring, getrandom).
+
+## Not supported
+
+These Go targets have no Rust target with a prebuilt standard library on the
+stable toolchain (Rust tier 3, built only with nightly `-Zbuild-std`, or no
+Rust target at all). Building them would need a nightly toolchain and
+untested standard libraries, which this project does not ship.
+
+| Go name | Rust status |
+| --- | --- |
+| aix/ppc64 | tier 3 (powerpc64-ibm-aix) |
+| dragonfly/amd64 | tier 3 |
+| freebsd/arm, freebsd/arm64, freebsd/riscv64 | tier 3 |
+| netbsd/386, netbsd/arm, netbsd/arm64 | tier 3 |
+| openbsd/386, amd64, arm, arm64, ppc64, riscv64 | tier 3 (OpenBSD packages Rust itself; building in an OpenBSD VM would work) |
+| linux/mips, mipsle, mips64, mips64le | tier 3 since Rust 1.72 |
+| plan9/386, amd64, arm | no Rust target |
+| js/wasm | wasm32-unknown-unknown has no filesystem or OS access; not meaningful for vault |
+
+Dropping these is a scope change from the Go release (43 binaries) and needs
+an explicit decision before a release: either accept the narrower matrix,
+or add nightly/native builds for chosen targets (OpenBSD in a VM is the most
+practical).

src/store/tests.rs

@@ -451,3 +451,53 @@ fn duplicate_records_recover() {
     add_bytes(&s, &mut cat, "f", b"new");
     assert_eq!(s.catalogue().unwrap().records("f").len(), 1, "next write cleans up");
 }
+
+/// Repeated failures mid-pipeline — a source that errors part-way, objects
+/// that fail to authenticate — leave no extra threads behind and return
+/// every buffer (the next large operation still gets all eight).
+#[cfg(target_os = "linux")]
+#[test]
+fn repeated_failures_do_not_leak() {
+    let threads = || std::fs::read_dir("/proc/self/task").unwrap().count();
+    let (d, s) = vault("leak");
+    let mut cat = s.catalogue().unwrap();
+    let body = vec![3u8; 3 * CHUNK_DATA as usize];
+    add_bytes(&s, &mut cat, "f", &body);
+    let r = cat.current("f").unwrap();
+    let obj = s.path(OBJECTS, &r.seed.object_id(1));
+    let good = std::fs::read(&obj).unwrap();
+    let mut bad = good.clone();
+    let n = bad.len() / 2;
+    bad[n] ^= 1;
+    std::fs::write(&obj, &bad).unwrap();
+    struct Failing(usize);
+    impl Read for Failing {
+        fn read(&mut self, b: &mut [u8]) -> std::io::Result<usize> {
+            if self.0 == 0 {
+                return Err(std::io::Error::other("source failed"));
+            }
+            let k = b.len().min(self.0);
+            self.0 -= k;
+            b[..k].fill(9);
+            Ok(k)
+        }
+    }
+    let mut other = s.catalogue().unwrap();
+    let before = threads();
+    for i in 0..10 {
+        assert!(get_bytes(&s, r, &d.0, &format!("o{i}")).is_err());
+        let e = s.add_reader(
+            &mut other,
+            &mut Failing(2 * CHUNK_DATA as usize + 5),
+            3 * CHUNK_DATA,
+            0o600,
+            Instant::now(),
+            &name("g"),
+        );
+        assert!(e.is_err());
+    }
+    assert_eq!(threads(), before, "threads accumulated");
+    std::fs::write(&obj, &good).unwrap();
+    assert_eq!(get_bytes(&s, r, &d.0, "ok").unwrap(), body, "buffers were not all returned");
+    assert!(!s.catalogue().unwrap().contains("g"));
+}

tests/cli.rs

@@ -585,3 +585,55 @@ fn broken_pipe_is_quiet() {
         String::from_utf8_lossy(&o.stderr)
     );
 }
+
+/// A crash part-way through writing (the kernel kills vault with SIGXFSZ
+/// once it passes a file-size limit) leaves the vault consistent: a file
+/// being replaced keeps its old contents, nothing half-written appears, and
+/// migrate deletes no original it had not finished storing.
+#[cfg(target_os = "linux")]
+#[test]
+fn crash_mid_write_keeps_the_vault_consistent() {
+    if Command::new("prlimit").arg("--version").output().is_err() {
+        return common::skip("prlimit not installed");
+    }
+    let e = Env::new("crash");
+    genkey(&e, RUST, false);
+    e.ok(RUST, Some(REAL), &["init"]);
+    e.write("v1/f", b"version one");
+    e.ok(RUST, Some(REAL), &["add", "v1/f"]);
+    let big = pattern(6 << 20, 9);
+    e.write("v2/f", &big);
+    e.write("tree/a", b"a");
+    e.write("tree/b", &big);
+    for args in [vec!["add", "v2/f"], vec!["migrate", "--yes", "tree"]] {
+        // Objects are about 4 MiB; a 2 MiB file-size limit kills vault while
+        // it writes the first one.
+        let o = Command::new("prlimit")
+            .arg("--fsize=2000000")
+            .arg(RUST)
+            .args(&args)
+            .current_dir(&e.dir)
+            .env("VAULT_KEY", e.dir.join("key"))
+            .env("VAULT_DIR", e.dir.join("store"))
+            .env("VAULT_PASSPHRASE", REAL)
+            .output()
+            .unwrap();
+        assert!(!o.status.success(), "{args:?} survived the size limit");
+    }
+    assert_eq!(e.get(RUST, REAL, "f"), b"version one", "replace interrupted, old version lost");
+    let names = e.names(RUST, REAL);
+    assert!(names.iter().all(|n| n == "f" || n == "tree/a"), "half-written entries: {names:?}");
+    assert_eq!(
+        std::fs::read(e.dir.join("tree/b")).unwrap(),
+        big,
+        "original deleted before it was stored"
+    );
+    if names.iter().any(|n| n == "tree/a") {
+        assert_eq!(e.get(RUST, REAL, "tree/a"), b"a");
+    } else {
+        assert!(e.dir.join("tree/a").exists());
+    }
+    // The vault still works normally afterwards.
+    e.ok(RUST, Some(REAL), &["add", "v2/f"]);
+    assert_eq!(e.get(RUST, REAL, "f"), big);
+}