Chunk objects, shard the store, and make filler configurable

File bodies are now cut into 4 MiB objects, the last padded to its
power-of-two bucket, so every object is 64 KiB to 4 MiB whatever the file
size. Object IDs are HMAC'd from a secret per-record seed, so nothing on
disk links a file's pieces, and each object still leads with its own ID,
binding it to its record and position. Chunks are encrypted and synced by
a small worker pool so their fsyncs overlap.

records/ and objects/ are sharded into 256 subdirectories by ID prefix,
all created at init, with every shard's mtime pinned to the epoch.

vault init --filler SIZE writes at least SIZE of filler. Filler entries
are now real entries for a throwaway key, of log-uniform random sizes, so
they carry multi-chunk files like a real collection.

The marker becomes VAULT 2; earlier vaults are not supported.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Andy Dixon
2026-09-26 18:20:51 +01:00
co-authored by Claude Opus 5.5
parent 8f3a755bf9
commit eea16c9076
17 changed files with 556 additions and 175 deletions
+23 -2
View File
@@ -13,6 +13,7 @@ import (
"path"
"path/filepath"
"sort"
"strconv"
"strings"
"github.com/andydixon/vault/internal/term"
@@ -20,7 +21,8 @@ import (
const usage = `Usage:
vault genkey create a new passphrase-protected key
vault init create a new, empty vault
vault init [--filler SIZE] create a new, empty vault, with at least
SIZE of filler (e.g. 500M, 2G)
vault add FILE|DIR [FOLDER] encrypt a file or directory into the vault,
optionally under a folder inside the vault
vault list [PREFIX] list stored files
@@ -62,6 +64,7 @@ func main() {
keyPath := fset.String("key", defaultPath("VAULT_KEY", "key"), "")
vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "")
output := fset.String("o", "", "")
filler := fset.String("filler", "0", "")
pos, err := parseInterleaved(fset, args)
if err != nil {
fail("%v\n\n%s", err, usage)
@@ -87,7 +90,9 @@ func main() {
fmt.Fprintf(os.Stderr, "key written to %s\n", *keyPath)
case "init":
want(0, 0)
check(initStore(*vaultDir))
size, err := parseSize(*filler)
check(err)
check(initStore(*vaultDir, size))
fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir)
case "add":
want(1, 2)
@@ -144,6 +149,22 @@ func parseInterleaved(fset *flag.FlagSet, args []string) ([]string, error) {
}
}
// parseSize reads a byte count with an optional K, M, G or T suffix, in
// powers of 1024.
func parseSize(s string) (int64, error) {
num, shift := strings.TrimSuffix(strings.ToUpper(s), "B"), 0
for i, unit := range "KMGT" {
if strings.HasSuffix(num, string(unit)) {
num, shift = strings.TrimSuffix(num, string(unit)), 10*(i+1)
}
}
n, err := strconv.ParseInt(num, 10, 64)
if err != nil || n < 0 || n > fileLimit>>shift {
return 0, fmt.Errorf("invalid size %q", s)
}
return n << shift, nil
}
func defaultPath(env, name string) string {
if v := os.Getenv(env); v != "" {
return v