Chunk objects, shard the store, and make filler configurable
File bodies are now cut into 4 MiB objects, the last padded to its power-of-two bucket, so every object is 64 KiB to 4 MiB whatever the file size. Object IDs are HMAC'd from a secret per-record seed, so nothing on disk links a file's pieces, and each object still leads with its own ID, binding it to its record and position. Chunks are encrypted and synced by a small worker pool so their fsyncs overlap. records/ and objects/ are sharded into 256 subdirectories by ID prefix, all created at init, with every shard's mtime pinned to the epoch. vault init --filler SIZE writes at least SIZE of filler. Filler entries are now real entries for a throwaway key, of log-uniform random sizes, so they carry multi-chunk files like a real collection. The marker becomes VAULT 2; earlier vaults are not supported. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
8f3a755bf9
commit
eea16c9076
@@ -13,6 +13,7 @@ import (
|
||||
"path"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/andydixon/vault/internal/term"
|
||||
@@ -20,7 +21,8 @@ import (
|
||||
|
||||
const usage = `Usage:
|
||||
vault genkey create a new passphrase-protected key
|
||||
vault init create a new, empty vault
|
||||
vault init [--filler SIZE] create a new, empty vault, with at least
|
||||
SIZE of filler (e.g. 500M, 2G)
|
||||
vault add FILE|DIR [FOLDER] encrypt a file or directory into the vault,
|
||||
optionally under a folder inside the vault
|
||||
vault list [PREFIX] list stored files
|
||||
@@ -62,6 +64,7 @@ func main() {
|
||||
keyPath := fset.String("key", defaultPath("VAULT_KEY", "key"), "")
|
||||
vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "")
|
||||
output := fset.String("o", "", "")
|
||||
filler := fset.String("filler", "0", "")
|
||||
pos, err := parseInterleaved(fset, args)
|
||||
if err != nil {
|
||||
fail("%v\n\n%s", err, usage)
|
||||
@@ -87,7 +90,9 @@ func main() {
|
||||
fmt.Fprintf(os.Stderr, "key written to %s\n", *keyPath)
|
||||
case "init":
|
||||
want(0, 0)
|
||||
check(initStore(*vaultDir))
|
||||
size, err := parseSize(*filler)
|
||||
check(err)
|
||||
check(initStore(*vaultDir, size))
|
||||
fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir)
|
||||
case "add":
|
||||
want(1, 2)
|
||||
@@ -144,6 +149,22 @@ func parseInterleaved(fset *flag.FlagSet, args []string) ([]string, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// parseSize reads a byte count with an optional K, M, G or T suffix, in
|
||||
// powers of 1024.
|
||||
func parseSize(s string) (int64, error) {
|
||||
num, shift := strings.TrimSuffix(strings.ToUpper(s), "B"), 0
|
||||
for i, unit := range "KMGT" {
|
||||
if strings.HasSuffix(num, string(unit)) {
|
||||
num, shift = strings.TrimSuffix(num, string(unit)), 10*(i+1)
|
||||
}
|
||||
}
|
||||
n, err := strconv.ParseInt(num, 10, 64)
|
||||
if err != nil || n < 0 || n > fileLimit>>shift {
|
||||
return 0, fmt.Errorf("invalid size %q", s)
|
||||
}
|
||||
return n << shift, nil
|
||||
}
|
||||
|
||||
func defaultPath(env, name string) string {
|
||||
if v := os.Getenv(env); v != "" {
|
||||
return v
|
||||
|
||||
Reference in New Issue
Block a user