Add vault locking and a cross-platform build script
Commands that change the vault hold an exclusive lock on DIR/.lock and readers hold a shared one, taken after the passphrase prompt. A blocked process says it is waiting. Unix uses fcntl locks and Windows uses LockFileEx, both released by the OS if the process dies. Plan 9 and WebAssembly fall back to an exclusive lock file. build.sh cross-compiles static binaries for every Go target that builds without cgo (43 today, armv6 and armv7 for 32-bit ARM Linux) and writes SHA256SUMS.
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
/vault
|
||||
/vault.exe
|
||||
.claude/
|
||||
/dist
|
||||
|
||||
@@ -14,6 +14,11 @@ root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
|
||||
- `store.go` - on-disk vault: `records/` holds fixed-size encrypted catalogue
|
||||
records, `objects/` holds padded encrypted file bodies. Records a key cannot
|
||||
open are skipped, not treated as errors.
|
||||
- `lock*.go` - vault lock on `DIR/.lock`: fcntl on unix, LockFileEx on
|
||||
Windows, an exclusive lock file elsewhere. Writers lock exclusively,
|
||||
readers shared, taken after the passphrase prompt.
|
||||
- `build.sh` - cross-compiles every `go tool dist list` target except
|
||||
iOS and android/386, amd64, arm (cgo-only).
|
||||
- `internal/pqcrypt` - file encryption (header + STREAM ChaCha20-Poly1305),
|
||||
suite registry, hybrid FrodoKEM-640 + ML-KEM-768 default suite.
|
||||
- `internal/format`, `internal/stream`, `internal/term` - header encoding,
|
||||
@@ -24,6 +29,8 @@ root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
|
||||
```sh
|
||||
go test ./...
|
||||
go build .
|
||||
./build.sh # all platforms, into ./dist
|
||||
TARGETS="linux/amd64" ./build.sh # a subset
|
||||
```
|
||||
|
||||
End-to-end check:
|
||||
@@ -43,5 +50,6 @@ export VAULT_PASSPHRASE=test VAULT_KEY=$PWD/k VAULT_DIR=$PWD/s
|
||||
- `get` publishes output only after the entire object has authenticated.
|
||||
- Decoy deniability: both key slots stay the same size and are both always
|
||||
tried; `init` always writes filler; foreign records are skipped silently.
|
||||
- Keep dependencies pure Go.
|
||||
- Every command that changes the vault holds the exclusive lock.
|
||||
- Keep dependencies pure Go and CGO_ENABLED=0 builds working.
|
||||
- Diagnostics go to standard error, lower-case, no trailing full stops.
|
||||
|
||||
@@ -13,8 +13,14 @@ buckets.
|
||||
```sh
|
||||
go build . # produces ./vault
|
||||
go test ./...
|
||||
./build.sh # every OS and CPU architecture, into ./dist
|
||||
```
|
||||
|
||||
`build.sh` cross-compiles static binaries for every target in
|
||||
`go tool dist list` except iOS and 32-bit or x86 Android, which need cgo. It builds 32-bit ARM Linux for
|
||||
both armv6 and armv7, and writes `dist/SHA256SUMS`. Set `TARGETS` to build a
|
||||
subset, for example `TARGETS="linux/amd64 darwin/arm64" ./build.sh`.
|
||||
|
||||
## Use
|
||||
|
||||
```sh
|
||||
@@ -31,6 +37,13 @@ vault mv report.pdf old/report.pdf
|
||||
vault rm old/report.pdf
|
||||
```
|
||||
|
||||
Commands that change the vault take an exclusive lock on `DIR/.lock`, and
|
||||
commands that read it take a shared one. A second process waits, and says
|
||||
so, until the first finishes. The operating system releases the lock if a
|
||||
process dies. Plan 9 and WebAssembly have no file locks, so there the lock
|
||||
file itself is the lock. After a crash on those platforms you remove it by
|
||||
hand; the error message names it.
|
||||
|
||||
Adding a name that already exists replaces it. `get` refuses to overwrite an
|
||||
existing file and writes nothing until the whole object has authenticated.
|
||||
|
||||
@@ -67,7 +80,6 @@ vault over time can still see it grow when real files are added.
|
||||
|
||||
## Limits
|
||||
|
||||
- One writer at a time. Concurrent writes to the same vault are not locked.
|
||||
- The storage host can see how many files exist, their size buckets and when
|
||||
the vault changes.
|
||||
- The key file is the only way in. Losing it or its passphrase loses the data.
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# Build vault for every OS and CPU architecture Go can target without cgo.
|
||||
#
|
||||
# ./build.sh # everything, into ./dist
|
||||
# TARGETS="linux/amd64 darwin/arm64" ./build.sh
|
||||
# OUT=/tmp/release ./build.sh
|
||||
#
|
||||
# Binaries are static, stripped and reproducible (CGO_ENABLED=0, -trimpath).
|
||||
# iOS and most Android targets are skipped (they need cgo); see below.
|
||||
# 32-bit ARM Linux is built twice: armv6 (Raspberry Pi Zero/1) and armv7.
|
||||
# SHA256SUMS is written next to the binaries.
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
out=${OUT:-dist}
|
||||
# iOS and 32-bit/x86 Android need cgo and a vendor SDK. android/arm64 builds
|
||||
# a static binary that runs in Termux.
|
||||
targets=${TARGETS:-$(go tool dist list | grep -vE '^ios/|^android/(386|amd64|arm)$')}
|
||||
jobs=${JOBS:-$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)}
|
||||
|
||||
rm -rf "$out"
|
||||
mkdir -p "$out"
|
||||
|
||||
build_one() {
|
||||
local os=${1%/*} arch=${1#*/} variants=("") name ext=""
|
||||
[[ $os == windows ]] && ext=.exe
|
||||
[[ $arch == wasm ]] && ext=.wasm
|
||||
[[ $os/$arch == linux/arm ]] && variants=(6 7)
|
||||
for v in "${variants[@]}"; do
|
||||
name=vault-$os-$arch${v:+v$v}$ext
|
||||
if ! GOOS=$os GOARCH=$arch GOARM=${v:-} CGO_ENABLED=0 \
|
||||
go build -trimpath -buildvcs=false -ldflags='-s -w -buildid=' -o "$OUT_DIR/$name" . 2>"$OUT_DIR/$name.log"; then
|
||||
echo "FAILED $name" >&2
|
||||
cat "$OUT_DIR/$name.log" >&2
|
||||
return 1
|
||||
fi
|
||||
rm "$OUT_DIR/$name.log"
|
||||
echo "built $name"
|
||||
done
|
||||
}
|
||||
export -f build_one
|
||||
export OUT_DIR=$out
|
||||
|
||||
printf '%s\n' $targets | xargs -P "$jobs" -I{} bash -c 'build_one {}'
|
||||
|
||||
cd "$out"
|
||||
if command -v sha256sum >/dev/null; then sha256sum vault-* >SHA256SUMS; else shasum -a 256 vault-* >SHA256SUMS; fi
|
||||
echo "$(ls vault-* | wc -l | tr -d ' ') binaries in $out"
|
||||
@@ -0,0 +1,28 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
)
|
||||
|
||||
const lockName = ".lock"
|
||||
|
||||
// lockVault takes the vault lock: exclusive for commands that change the
|
||||
// vault, shared for commands that only read it. If another vault process
|
||||
// holds a conflicting lock it says so and waits. The returned function
|
||||
// releases the lock.
|
||||
func lockVault(root string, exclusive bool) (func(), error) {
|
||||
release, err := acquireLock(filepath.Join(root, lockName), exclusive, func() {
|
||||
fmt.Fprintln(os.Stderr, "waiting for another vault process to finish")
|
||||
})
|
||||
if err != nil && !exclusive {
|
||||
// A reader on read-only storage cannot create the lock file, and
|
||||
// nothing can be writing there either.
|
||||
return func() {}, nil
|
||||
}
|
||||
return release, err
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//go:build !unix && !windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
// acquireLock falls back to an exclusive lock file where the OS offers no
|
||||
// file locks. Every lock is exclusive here.
|
||||
// ponytail: a crash leaves the lock file behind; the error names it for manual removal.
|
||||
func acquireLock(path string, _ bool, waiting func()) (func(), error) {
|
||||
for i := 0; ; i++ {
|
||||
f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err == nil {
|
||||
f.Close()
|
||||
return func() { os.Remove(path) }, nil
|
||||
}
|
||||
if !errors.Is(err, fs.ErrExist) {
|
||||
return nil, err
|
||||
}
|
||||
if i == 0 {
|
||||
waiting()
|
||||
}
|
||||
if i == 600 {
|
||||
return nil, fmt.Errorf("vault still locked after a minute; if no other vault process is running, remove %s", path)
|
||||
}
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//go:build unix
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// acquireLock uses fcntl record locks, which the kernel releases if the
|
||||
// process dies, and which also work on NFS.
|
||||
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
lk := unix.Flock_t{Type: unix.F_RDLCK, Whence: io.SeekStart}
|
||||
if exclusive {
|
||||
lk.Type = unix.F_WRLCK
|
||||
}
|
||||
err = unix.FcntlFlock(f.Fd(), unix.F_SETLK, &lk)
|
||||
if errors.Is(err, unix.EAGAIN) || errors.Is(err, unix.EACCES) {
|
||||
waiting()
|
||||
for err = unix.EINTR; errors.Is(err, unix.EINTR); {
|
||||
err = unix.FcntlFlock(f.Fd(), unix.F_SETLKW, &lk)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return func() { f.Close() }, nil
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//go:build windows
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
|
||||
"golang.org/x/sys/windows"
|
||||
)
|
||||
|
||||
// acquireLock uses LockFileEx, which Windows releases if the process dies.
|
||||
func acquireLock(path string, exclusive bool, waiting func()) (func(), error) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0o600)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h := windows.Handle(f.Fd())
|
||||
var flags uint32
|
||||
if exclusive {
|
||||
flags = windows.LOCKFILE_EXCLUSIVE_LOCK
|
||||
}
|
||||
err = windows.LockFileEx(h, flags|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, new(windows.Overlapped))
|
||||
if errors.Is(err, windows.ERROR_LOCK_VIOLATION) {
|
||||
waiting()
|
||||
err = windows.LockFileEx(h, flags, 0, 1, 0, new(windows.Overlapped))
|
||||
}
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
return func() { f.Close() }, nil
|
||||
}
|
||||
@@ -46,7 +46,12 @@ const (
|
||||
decoyEnv = "VAULT_DECOY_PASSPHRASE"
|
||||
)
|
||||
|
||||
// unlock releases the vault lock taken by open. fail calls it too, because
|
||||
// os.Exit skips deferred calls.
|
||||
var unlock = func() {}
|
||||
|
||||
func main() {
|
||||
defer func() { unlock() }()
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, usage)
|
||||
os.Exit(2)
|
||||
@@ -90,7 +95,7 @@ func main() {
|
||||
if len(pos) == 2 {
|
||||
dest = strings.Trim(pos[1], "/")
|
||||
}
|
||||
s := open(*keyPath, *vaultDir)
|
||||
s := open(*keyPath, *vaultDir, true)
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
check(addPath(s, cat, pos[0], dest))
|
||||
@@ -100,7 +105,7 @@ func main() {
|
||||
if len(pos) == 1 {
|
||||
prefix = strings.Trim(pos[0], "/")
|
||||
}
|
||||
cat, err := open(*keyPath, *vaultDir).catalogue()
|
||||
cat, err := open(*keyPath, *vaultDir, false).catalogue()
|
||||
check(err)
|
||||
list(cat, prefix)
|
||||
case "get":
|
||||
@@ -109,13 +114,13 @@ func main() {
|
||||
if out == "" {
|
||||
out = path.Base(pos[0])
|
||||
}
|
||||
check(open(*keyPath, *vaultDir).get(pos[0], out))
|
||||
check(open(*keyPath, *vaultDir, false).get(pos[0], out))
|
||||
case "mv", "rename":
|
||||
want(2, 2)
|
||||
check(open(*keyPath, *vaultDir).rename(pos[0], pos[1]))
|
||||
check(open(*keyPath, *vaultDir, true).rename(pos[0], pos[1]))
|
||||
case "rm", "delete":
|
||||
want(1, 1)
|
||||
check(open(*keyPath, *vaultDir).delete(pos[0]))
|
||||
check(open(*keyPath, *vaultDir, true).delete(pos[0]))
|
||||
case "help", "-h", "--help":
|
||||
fmt.Println(usage)
|
||||
default:
|
||||
@@ -150,13 +155,17 @@ func defaultPath(env, name string) string {
|
||||
return filepath.Join(home, ".vault", name)
|
||||
}
|
||||
|
||||
func open(keyPath, vaultDir string) *store {
|
||||
// open unlocks the key and opens the vault, holding the vault lock
|
||||
// exclusively for writers and shared for readers.
|
||||
func open(keyPath, vaultDir string, exclusive bool) *store {
|
||||
pass, err := term.ReadPassphrase("Enter passphrase:", passphraseEnv)
|
||||
check(err)
|
||||
id, err := unlockKeyFile(keyPath, pass)
|
||||
check(err)
|
||||
s, err := openStore(vaultDir, id)
|
||||
check(err)
|
||||
unlock, err = lockVault(vaultDir, exclusive)
|
||||
check(err)
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -271,6 +280,7 @@ func check(err error) {
|
||||
}
|
||||
|
||||
func fail(format string, args ...any) {
|
||||
unlock()
|
||||
fmt.Fprintf(os.Stderr, "vault: "+format+"\n", args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -4,10 +4,14 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestKeyFile(t *testing.T) {
|
||||
@@ -129,3 +133,45 @@ func TestDecoy(t *testing.T) {
|
||||
t.Fatalf("only %d records; filler missing", len(entries))
|
||||
}
|
||||
}
|
||||
|
||||
// TestLockExcludesOtherProcess runs a copy of the test binary that holds the
|
||||
// exclusive lock, then checks this process has to wait for it.
|
||||
func TestLockExcludesOtherProcess(t *testing.T) {
|
||||
if dir := os.Getenv("VAULT_TEST_LOCK_HOLDER"); dir != "" {
|
||||
release, err := lockVault(dir, true)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
os.Stdout.WriteString("locked\n")
|
||||
io.Copy(io.Discard, os.Stdin) // hold until the parent closes stdin
|
||||
release()
|
||||
os.Exit(0)
|
||||
}
|
||||
dir := t.TempDir()
|
||||
holder := exec.Command(os.Args[0], "-test.run=^TestLockExcludesOtherProcess$")
|
||||
holder.Env = append(os.Environ(), "VAULT_TEST_LOCK_HOLDER="+dir)
|
||||
stdin, _ := holder.StdinPipe()
|
||||
stdout, _ := holder.StdoutPipe()
|
||||
if err := holder.Start(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer holder.Wait()
|
||||
if line, _ := bufio.NewReader(stdout).ReadString('\n'); line != "locked\n" {
|
||||
t.Fatalf("holder said %q", line)
|
||||
}
|
||||
waited := make(chan bool, 1)
|
||||
go func() {
|
||||
time.Sleep(200 * time.Millisecond)
|
||||
stdin.Close() // let the holder go only after we have started waiting
|
||||
}()
|
||||
for _, exclusive := range []bool{false, true} {
|
||||
release, err := acquireLock(filepath.Join(dir, lockName), exclusive, func() { waited <- true })
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
release()
|
||||
if !exclusive && len(waited) == 0 {
|
||||
t.Fatal("shared lock did not wait for the exclusive holder")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user