Add vault locking and a cross-platform build script

Commands that change the vault hold an exclusive lock on DIR/.lock and
readers hold a shared one, taken after the passphrase prompt. A blocked
process says it is waiting. Unix uses fcntl locks and Windows uses
LockFileEx, both released by the OS if the process dies. Plan 9 and
WebAssembly fall back to an exclusive lock file.

build.sh cross-compiles static binaries for every Go target that builds
without cgo (43 today, armv6 and armv7 for 32-bit ARM Linux) and writes
SHA256SUMS.
This commit is contained in:
Andy Dixon
2026-09-26 14:36:44 +01:00
parent be09659f8f
commit 5aa60d7bc7
10 changed files with 273 additions and 8 deletions
+1
View File
@@ -1,3 +1,4 @@
/vault /vault
/vault.exe /vault.exe
.claude/ .claude/
/dist
+9 -1
View File
@@ -14,6 +14,11 @@ root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
- `store.go` - on-disk vault: `records/` holds fixed-size encrypted catalogue - `store.go` - on-disk vault: `records/` holds fixed-size encrypted catalogue
records, `objects/` holds padded encrypted file bodies. Records a key cannot records, `objects/` holds padded encrypted file bodies. Records a key cannot
open are skipped, not treated as errors. open are skipped, not treated as errors.
- `lock*.go` - vault lock on `DIR/.lock`: fcntl on unix, LockFileEx on
Windows, an exclusive lock file elsewhere. Writers lock exclusively,
readers shared, taken after the passphrase prompt.
- `build.sh` - cross-compiles every `go tool dist list` target except
iOS and android/386, amd64, arm (cgo-only).
- `internal/pqcrypt` - file encryption (header + STREAM ChaCha20-Poly1305), - `internal/pqcrypt` - file encryption (header + STREAM ChaCha20-Poly1305),
suite registry, hybrid FrodoKEM-640 + ML-KEM-768 default suite. suite registry, hybrid FrodoKEM-640 + ML-KEM-768 default suite.
- `internal/format`, `internal/stream`, `internal/term` - header encoding, - `internal/format`, `internal/stream`, `internal/term` - header encoding,
@@ -24,6 +29,8 @@ root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
```sh ```sh
go test ./... go test ./...
go build . go build .
./build.sh # all platforms, into ./dist
TARGETS="linux/amd64" ./build.sh # a subset
``` ```
End-to-end check: End-to-end check:
@@ -43,5 +50,6 @@ export VAULT_PASSPHRASE=test VAULT_KEY=$PWD/k VAULT_DIR=$PWD/s
- `get` publishes output only after the entire object has authenticated. - `get` publishes output only after the entire object has authenticated.
- Decoy deniability: both key slots stay the same size and are both always - Decoy deniability: both key slots stay the same size and are both always
tried; `init` always writes filler; foreign records are skipped silently. tried; `init` always writes filler; foreign records are skipped silently.
- Keep dependencies pure Go. - Every command that changes the vault holds the exclusive lock.
- Keep dependencies pure Go and CGO_ENABLED=0 builds working.
- Diagnostics go to standard error, lower-case, no trailing full stops. - Diagnostics go to standard error, lower-case, no trailing full stops.
+13 -1
View File
@@ -13,8 +13,14 @@ buckets.
```sh ```sh
go build . # produces ./vault go build . # produces ./vault
go test ./... go test ./...
./build.sh # every OS and CPU architecture, into ./dist
``` ```
`build.sh` cross-compiles static binaries for every target in
`go tool dist list` except iOS and 32-bit or x86 Android, which need cgo. It builds 32-bit ARM Linux for
both armv6 and armv7, and writes `dist/SHA256SUMS`. Set `TARGETS` to build a
subset, for example `TARGETS="linux/amd64 darwin/arm64" ./build.sh`.
## Use ## Use
```sh ```sh
@@ -31,6 +37,13 @@ vault mv report.pdf old/report.pdf
vault rm old/report.pdf vault rm old/report.pdf
``` ```
Commands that change the vault take an exclusive lock on `DIR/.lock`, and
commands that read it take a shared one. A second process waits, and says
so, until the first finishes. The operating system releases the lock if a
process dies. Plan 9 and WebAssembly have no file locks, so there the lock
file itself is the lock. After a crash on those platforms you remove it by
hand; the error message names it.
Adding a name that already exists replaces it. `get` refuses to overwrite an Adding a name that already exists replaces it. `get` refuses to overwrite an
existing file and writes nothing until the whole object has authenticated. existing file and writes nothing until the whole object has authenticated.
@@ -67,7 +80,6 @@ vault over time can still see it grow when real files are added.
## Limits ## Limits
- One writer at a time. Concurrent writes to the same vault are not locked.
- The storage host can see how many files exist, their size buckets and when - The storage host can see how many files exist, their size buckets and when
the vault changes. the vault changes.
- The key file is the only way in. Losing it or its passphrase loses the data. - The key file is the only way in. Losing it or its passphrase loses the data.
Executable
+48
View File
@@ -0,0 +1,48 @@
#!/usr/bin/env bash
# Build vault for every OS and CPU architecture Go can target without cgo.
#
# ./build.sh # everything, into ./dist
# TARGETS="linux/amd64 darwin/arm64" ./build.sh
# OUT=/tmp/release ./build.sh
#
# Binaries are static, stripped and reproducible (CGO_ENABLED=0, -trimpath).
# iOS and most Android targets are skipped (they need cgo); see below.
# 32-bit ARM Linux is built twice: armv6 (Raspberry Pi Zero/1) and armv7.
# SHA256SUMS is written next to the binaries.
set -euo pipefail
cd "$(dirname "$0")"
out=${OUT:-dist}
# iOS and 32-bit/x86 Android need cgo and a vendor SDK. android/arm64 builds
# a static binary that runs in Termux.
targets=${TARGETS:-$(go tool dist list | grep -vE '^ios/|^android/(386|amd64|arm)$')}
jobs=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}
rm -rf "$out"
mkdir -p "$out"
build_one() {
local os=${1%/*} arch=${1#*/} variants=("") name ext=""
[[ $os == windows ]] && ext=.exe
[[ $arch == wasm ]] && ext=.wasm
[[ $os/$arch == linux/arm ]] && variants=(6 7)
for v in "${variants[@]}"; do
name=vault-$os-$arch${v:+v$v}$ext
if ! GOOS=$os GOARCH=$arch GOARM=${v:-} CGO_ENABLED=0 \
go build -trimpath -buildvcs=false -ldflags='-s -w -buildid=' -o "$OUT_DIR/$name" . 2>"$OUT_DIR/$name.log"; then
echo "FAILED $name" >&2
cat "$OUT_DIR/$name.log" >&2
return 1
fi
rm "$OUT_DIR/$name.log"
echo "built $name"
done
}
export -f build_one
export OUT_DIR=$out
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}'
cd "$out"
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
echo "$(ls vault-* | wc -l | tr -d ' ') binaries in $out"
+28
View File
@@ -0,0 +1,28 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"fmt"
"os"
"path/filepath"
)
const lockName = ".lock"
// lockVault takes the vault lock: exclusive for commands that change the
// vault, shared for commands that only read it. If another vault process
// holds a conflicting lock it says so and waits. The returned function
// releases the lock.
func lockVault(root string, exclusive bool) (func(), error) {
release, err := acquireLock(filepath.Join(root, lockName), exclusive, func() {
fmt.Fprintln(os.Stderr, "waiting for another vault process to finish")
})
if err != nil && !exclusive {
// A reader on read-only storage cannot create the lock file, and
// nothing can be writing there either.
return func() {}, nil
}
return release, err
}
+37
View File
@@ -0,0 +1,37 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build !unix && !windows
package main
import (
"errors"
"fmt"
"io/fs"
"os"
"time"
)
// acquireLock falls back to an exclusive lock file where the OS offers no
// file locks. Every lock is exclusive here.
// ponytail: a crash leaves the lock file behind; the error names it for manual removal.
func acquireLock(path string, _ bool, waiting func()) (func(), error) {
for i := 0; ; i++ {
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
if err == nil {
f.Close()
return func() { os.Remove(path) }, nil
}
if !errors.Is(err, fs.ErrExist) {
return nil, err
}
if i == 0 {
waiting()
}
if i == 600 {
return nil, fmt.Errorf("vault still locked after a minute; if no other vault process is running, remove %s", path)
}
time.Sleep(100 * time.Millisecond)
}
}
+39
View File
@@ -0,0 +1,39 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build unix
package main
import (
"errors"
"io"
"os"
"golang.org/x/sys/unix"
)
// acquireLock uses fcntl record locks, which the kernel releases if the
// process dies, and which also work on NFS.
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
lk := unix.Flock_t{Type: unix.F_RDLCK, Whence: io.SeekStart}
if exclusive {
lk.Type = unix.F_WRLCK
}
err = unix.FcntlFlock(f.Fd(), unix.F_SETLK, &lk)
if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EACCES) {
waiting()
for err = unix.EINTR; errors.Is(err, unix.EINTR); {
err = unix.FcntlFlock(f.Fd(), unix.F_SETLKW, &lk)
}
}
if err != nil {
f.Close()
return nil, err
}
return func() { f.Close() }, nil
}
+36
View File
@@ -0,0 +1,36 @@
// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
//go:build windows
package main
import (
"errors"
"os"
"golang.org/x/sys/windows"
)
// acquireLock uses LockFileEx, which Windows releases if the process dies.
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
if err != nil {
return nil, err
}
h := windows.Handle(f.Fd())
var flags uint32
if exclusive {
flags = windows.LOCKFILE_EXCLUSIVE_LOCK
}
err = windows.LockFileEx(h, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped))
if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
waiting()
err = windows.LockFileEx(h, flags, 0, 1, 0, new(windows.Overlapped))
}
if err != nil {
f.Close()
return nil, err
}
return func() { f.Close() }, nil
}
+16 -6
View File
@@ -46,7 +46,12 @@ const (
decoyEnv = "VAULT_DECOY_PASSPHRASE" decoyEnv = "VAULT_DECOY_PASSPHRASE"
) )
// unlock releases the vault lock taken by open. fail calls it too, because
// os.Exit skips deferred calls.
var unlock = func() {}
func main() { func main() {
defer func() { unlock() }()
if len(os.Args) < 2 { if len(os.Args) < 2 {
fmt.Fprintln(os.Stderr, usage) fmt.Fprintln(os.Stderr, usage)
os.Exit(2) os.Exit(2)
@@ -90,7 +95,7 @@ func main() {
if len(pos) == 2 { if len(pos) == 2 {
dest = strings.Trim(pos[1], "/") dest = strings.Trim(pos[1], "/")
} }
s := open(*keyPath, *vaultDir) s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue() cat, err := s.catalogue()
check(err) check(err)
check(addPath(s, cat, pos[0], dest)) check(addPath(s, cat, pos[0], dest))
@@ -100,7 +105,7 @@ func main() {
if len(pos) == 1 { if len(pos) == 1 {
prefix = strings.Trim(pos[0], "/") prefix = strings.Trim(pos[0], "/")
} }
cat, err := open(*keyPath, *vaultDir).catalogue() cat, err := open(*keyPath, *vaultDir, false).catalogue()
check(err) check(err)
list(cat, prefix) list(cat, prefix)
case "get": case "get":
@@ -109,13 +114,13 @@ func main() {
if out == "" { if out == "" {
out = path.Base(pos[0]) out = path.Base(pos[0])
} }
check(open(*keyPath, *vaultDir).get(pos[0], out)) check(open(*keyPath, *vaultDir, false).get(pos[0], out))
case "mv", "rename": case "mv", "rename":
want(2, 2) want(2, 2)
check(open(*keyPath, *vaultDir).rename(pos[0], pos[1])) check(open(*keyPath, *vaultDir, true).rename(pos[0], pos[1]))
case "rm", "delete": case "rm", "delete":
want(1, 1) want(1, 1)
check(open(*keyPath, *vaultDir).delete(pos[0])) check(open(*keyPath, *vaultDir, true).delete(pos[0]))
case "help", "-h", "--help": case "help", "-h", "--help":
fmt.Println(usage) fmt.Println(usage)
default: default:
@@ -150,13 +155,17 @@ func defaultPath(env, name string) string {
return filepath.Join(home, ".vault", name) return filepath.Join(home, ".vault", name)
} }
func open(keyPath, vaultDir string) *store { // open unlocks the key and opens the vault, holding the vault lock
// exclusively for writers and shared for readers.
func open(keyPath, vaultDir string, exclusive bool) *store {
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv) pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
check(err) check(err)
id, err := unlockKeyFile(keyPath, pass) id, err := unlockKeyFile(keyPath, pass)
check(err) check(err)
s, err := openStore(vaultDir, id) s, err := openStore(vaultDir, id)
check(err) check(err)
unlock, err = lockVault(vaultDir, exclusive)
check(err)
return s return s
} }
@@ -271,6 +280,7 @@ func check(err error) {
} }
func fail(format string, args ...any) { func fail(format string, args ...any) {
unlock()
fmt.Fprintf(os.Stderr, "vault: "+format+"\n", args...) fmt.Fprintf(os.Stderr, "vault: "+format+"\n", args...)
os.Exit(1) os.Exit(1)
} }
+46
View File
@@ -4,10 +4,14 @@
package main package main
import ( import (
"bufio"
"bytes" "bytes"
"io"
"os" "os"
"os/exec"
"path/filepath" "path/filepath"
"testing" "testing"
"time"
) )
func TestKeyFile(t *testing.T) { func TestKeyFile(t *testing.T) {
@@ -129,3 +133,45 @@ func TestDecoy(t *testing.T) {
t.Fatalf("only %d records; filler missing", len(entries)) t.Fatalf("only %d records; filler missing", len(entries))
} }
} }
// TestLockExcludesOtherProcess runs a copy of the test binary that holds the
// exclusive lock, then checks this process has to wait for it.
func TestLockExcludesOtherProcess(t *testing.T) {
if dir := os.Getenv("VAULT_TEST_LOCK_HOLDER"); dir != "" {
release, err := lockVault(dir, true)
if err != nil {
os.Exit(1)
}
os.Stdout.WriteString("locked\n")
io.Copy(io.Discard, os.Stdin) // hold until the parent closes stdin
release()
os.Exit(0)
}
dir := t.TempDir()
holder := exec.Command(os.Args[0], "-test.run=^TestLockExcludesOtherProcess$")
holder.Env = append(os.Environ(), "VAULT_TEST_LOCK_HOLDER="+dir)
stdin, _ := holder.StdinPipe()
stdout, _ := holder.StdoutPipe()
if err := holder.Start(); err != nil {
t.Fatal(err)
}
defer holder.Wait()
if line, _ := bufio.NewReader(stdout).ReadString('\n'); line != "locked\n" {
t.Fatalf("holder said %q", line)
}
waited := make(chan bool, 1)
go func() {
time.Sleep(200 * time.Millisecond)
stdin.Close() // let the holder go only after we have started waiting
}()
for _, exclusive := range []bool{false, true} {
release, err := acquireLock(filepath.Join(dir, lockName), exclusive, func() { waited <- true })
if err != nil {
t.Fatal(err)
}
release()
if !exclusive && len(waited) == 0 {
t.Fatal("shared lock did not wait for the exclusive holder")
}
}
}