Files
vault/vault_test.go
T
Andy DixonandClaude Opus 5.5 eea16c9076 Chunk objects, shard the store, and make filler configurable
File bodies are now cut into 4 MiB objects, the last padded to its
power-of-two bucket, so every object is 64 KiB to 4 MiB whatever the file
size. Object IDs are HMAC'd from a secret per-record seed, so nothing on
disk links a file's pieces, and each object still leads with its own ID,
binding it to its record and position. Chunks are encrypted and synced by
a small worker pool so their fsyncs overlap.

records/ and objects/ are sharded into 256 subdirectories by ID prefix,
all created at init, with every shard's mtime pinned to the epoch.

vault init --filler SIZE writes at least SIZE of filler. Filler entries
are now real entries for a throwaway key, of log-uniform random sizes, so
they carry multi-chunk files like a real collection.

The marker becomes VAULT 2; earlier vaults are not supported.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-26 18:20:51 +01:00

298 lines
8.4 KiB
Go

// Copyright 2026 The vault Authors.
// SPDX-License-Identifier: GPL-3.0-or-later
package main
import (
"bufio"
"bytes"
"crypto/rand"
"io"
"io/fs"
"os"
"os/exec"
"path/filepath"
"testing"
"time"
)
func TestKeyFile(t *testing.T) {
path := filepath.Join(t.TempDir(), "key")
if err := writeKeyFile(path, "pass", ""); err != nil {
t.Fatal(err)
}
if _, err := unlockKeyFile(path, "wrong"); err == nil {
t.Fatal("wrong passphrase unlocked the key")
}
if _, err := unlockKeyFile(path, "pass"); err != nil {
t.Fatal(err)
}
}
func newTestStore(t *testing.T, keyPath, pass string) *store {
t.Helper()
id, err := unlockKeyFile(keyPath, pass)
if err != nil {
t.Fatal(err)
}
s, err := openStore(filepath.Join(filepath.Dir(keyPath), "store"), id)
if err != nil {
t.Fatal(err)
}
return s
}
func TestStoreLifecycle(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
src := filepath.Join(dir, "in")
want := bytes.Repeat([]byte("x"), 70000)
os.WriteFile(src, want, 0o640)
cat, _ := s.catalogue()
if _, err := s.add(cat, src, "a/b"); err != nil {
t.Fatal(err)
}
if replaced, err := s.add(cat, src, "a/b"); err != nil || !replaced {
t.Fatalf("replace: %v %v", replaced, err)
}
if _, err := s.add(cat, src, "c"); err != nil {
t.Fatal(err)
}
if cat, _ = s.catalogue(); len(cat) != 2 || len(cat["a/b"]) != 1 {
t.Fatalf("catalogue = %v", cat)
}
if err := s.rename("a/b", "d"); err != nil {
t.Fatal(err)
}
out := filepath.Join(dir, "out")
if err := s.get("d", out); err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(out); !bytes.Equal(got, want) {
t.Fatal("round trip mismatch")
}
if err := s.get("d", out); err == nil {
t.Fatal("get overwrote an existing file")
}
// Swapping one object for another must be detected.
cat, _ = s.catalogue()
d, c := cat["d"][0], cat["c"][0]
data, _ := os.ReadFile(s.path(objectsDir, objectID(c.Seed, 0)))
os.WriteFile(s.path(objectsDir, objectID(d.Seed, 0)), data, 0o600)
if err := s.get("d", filepath.Join(dir, "swapped")); err == nil {
t.Fatal("swapped object was accepted")
}
if err := s.delete("c"); err != nil {
t.Fatal(err)
}
if cat, _ = s.catalogue(); len(cat) != 1 {
t.Fatalf("after delete: %v", cat)
}
}
func TestDecoy(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "real", "decoy"); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
src := filepath.Join(dir, "in")
os.WriteFile(src, []byte("data"), 0o600)
real, decoy := newTestStore(t, keyPath, "real"), newTestStore(t, keyPath, "decoy")
for s, name := range map[*store]string{real: "secret", decoy: "shopping"} {
cat, err := s.catalogue()
if err != nil || len(cat) != 0 {
t.Fatalf("fresh vault catalogue = %v, %v", cat, err)
}
if _, err := s.add(cat, src, name); err != nil {
t.Fatal(err)
}
}
for s, name := range map[*store]string{real: "secret", decoy: "shopping"} {
if cat, _ := s.catalogue(); len(cat) != 1 || cat[name] == nil {
t.Fatalf("catalogue = %v, want only %s", cat, name)
}
}
if _, err := unlockKeyFile(keyPath, "neither"); err == nil {
t.Fatal("wrong passphrase unlocked the key")
}
if n := len(ciphertexts(t, filepath.Join(dir, "store", recordsDir))); n < fillerMin+2 {
t.Fatalf("only %d records; filler missing", n)
}
}
// ciphertexts maps every file under dir to its size.
func ciphertexts(t *testing.T, dir string) map[string]int64 {
t.Helper()
sizes := map[string]int64{}
err := filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error {
if err != nil || d.IsDir() {
return err
}
info, err := d.Info()
sizes[p] = info.Size()
return err
})
if err != nil {
t.Fatal(err)
}
return sizes
}
// TestChunks checks a file bigger than one chunk becomes unlinkable 4 MiB
// objects plus a bucket-sized tail, all in their shards, and that reordering
// the chunks is caught.
func TestChunks(t *testing.T) {
dir := t.TempDir()
keyPath := filepath.Join(dir, "key")
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
t.Fatal(err)
}
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
t.Fatal(err)
}
s := newTestStore(t, keyPath, "pass")
objects := filepath.Join(s.root, objectsDir)
before := ciphertexts(t, objects)
src := filepath.Join(dir, "in")
want := make([]byte, 2*chunkData+100)
rand.Read(want)
os.WriteFile(src, want, 0o600)
cat, _ := s.catalogue()
if _, err := s.add(cat, src, "big"); err != nil {
t.Fatal(err)
}
cat, _ = s.catalogue()
r := cat["big"][0]
var sizes []int64
for i := range objectCount(r.Size) {
p := s.path(objectsDir, objectID(r.Seed, i))
if filepath.Base(filepath.Dir(p)) != filepath.Base(p)[:2] {
t.Fatalf("%s is in the wrong shard", p)
}
sizes = append(sizes, ciphertexts(t, p)[p])
}
if len(sizes) != 3 || sizes[0] != sizes[1] || sizes[2] >= sizes[0] {
t.Fatalf("object sizes = %v, want two full chunks and a smaller tail", sizes)
}
if n := len(ciphertexts(t, objects)) - len(before); n != 3 {
t.Fatalf("add wrote %d objects, want 3", n)
}
for p, size := range ciphertexts(t, objects) {
if size > sizes[0] {
t.Fatalf("%s is %d bytes, larger than a chunk", p, size)
}
}
out := filepath.Join(dir, "out")
if err := s.get("big", out); err != nil {
t.Fatal(err)
}
if got, _ := os.ReadFile(out); !bytes.Equal(got, want) {
t.Fatal("round trip mismatch")
}
a, b := s.path(objectsDir, objectID(r.Seed, 0)), s.path(objectsDir, objectID(r.Seed, 1))
os.Rename(a, a+".x")
os.Rename(b, a)
os.Rename(a+".x", b)
if err := s.get("big", filepath.Join(dir, "swapped")); err == nil {
t.Fatal("reordered chunks were accepted")
}
if err := s.delete("big"); err != nil {
t.Fatal(err)
}
if n := len(ciphertexts(t, objects)); n != len(before) {
t.Fatalf("%d objects after rm, want %d", n, len(before))
}
}
// TestLockExcludesOtherProcess runs a copy of the test binary that holds the
// exclusive lock, then checks this process has to wait for it.
func TestLockExcludesOtherProcess(t *testing.T) {
if dir := os.Getenv("VAULT_TEST_LOCK_HOLDER"); dir != "" {
release, err := lockVault(dir, true)
if err != nil {
os.Exit(1)
}
os.Stdout.WriteString("locked\n")
io.Copy(io.Discard, os.Stdin) // hold until the parent closes stdin
release()
os.Exit(0)
}
dir := t.TempDir()
holder := exec.Command(os.Args[0], "-test.run=^TestLockExcludesOtherProcess$")
holder.Env = append(os.Environ(), "VAULT_TEST_LOCK_HOLDER="+dir)
stdin, _ := holder.StdinPipe()
stdout, _ := holder.StdoutPipe()
if err := holder.Start(); err != nil {
t.Fatal(err)
}
defer holder.Wait()
if line, _ := bufio.NewReader(stdout).ReadString('\n'); line != "locked\n" {
t.Fatalf("holder said %q", line)
}
waited := make(chan bool, 1)
go func() {
time.Sleep(200 * time.Millisecond)
stdin.Close() // let the holder go only after we have started waiting
}()
for _, exclusive := range []bool{false, true} {
release, err := acquireLock(filepath.Join(dir, lockName), exclusive, func() { waited <- true })
if err != nil {
t.Fatal(err)
}
release()
if !exclusive && len(waited) == 0 {
t.Fatal("shared lock did not wait for the exclusive holder")
}
}
}
func TestParseSize(t *testing.T) {
for in, want := range map[string]int64{"0": 0, "512": 512, "2k": 2 << 10, "500M": 500 << 20, "2G": 2 << 30, "1TB": 1 << 40} {
if got, err := parseSize(in); err != nil || got != want {
t.Errorf("parseSize(%q) = %d, %v; want %d", in, got, err, want)
}
}
for _, in := range []string{"", "G", "-1", "2X", "1.5G", "2T"} {
if _, err := parseSize(in); err == nil {
t.Errorf("parseSize(%q) accepted", in)
}
}
}
// TestFiller checks a filler target is met with entries shaped like real
// files: fixed-size records and objects no bigger than a chunk.
func TestFiller(t *testing.T) {
root := filepath.Join(t.TempDir(), "store")
if err := initStore(root, 24<<20); err != nil {
t.Fatal(err)
}
if n := len(ciphertexts(t, filepath.Join(root, recordsDir))); n < fillerMin {
t.Fatalf("%d filler records, want at least %d", n, fillerMin)
}
var total, largest int64
for _, size := range ciphertexts(t, filepath.Join(root, objectsDir)) {
total, largest = total+size, max(largest, size)
}
if total < 24<<20 || largest > chunkSize+chunkSize/64 {
t.Fatalf("filler objects total %d, largest %d", total, largest)
}
}