diff --git a/CLAUDE.md b/CLAUDE.md index 759f6d1..b3c10f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,7 +8,8 @@ vault is a post-quantum encrypted file vault: one `package main` at the repo root that builds a single `vault` binary. Module `github.com/andydixon/vault`. - `main.go` - CLI: `genkey`, `init`, `add` (file, directory - or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`. + or URL), `migrate` (add, then delete originals), `mirror`, `absorb` (tar + on stdin, `absorb.go`), `list`, `get`, `mv`, `rm`. - `keyfile.go` - key file with two passphrase slots (Argon2id + XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second slot is either a decoy identity or random bytes; slot order is random. diff --git a/README.md b/README.md index 9644238..abbcfdb 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,7 @@ vault add ./photos # every regular file under photos/ vault add https://example.com/a.pdf # download and store a URL vault migrate ./photos # like add, then delete the originals vault mirror https://example.com/ # crawl a site into example.com/, links rewritten +tar -cz photos | vault absorb backup # every file in a tar (or .tar.gz) under backup/ vault list # everything vault list work # everything under work/ vault get work/2026/report.pdf # writes ./report.pdf diff --git a/absorb.go b/absorb.go new file mode 100644 index 0000000..0a5acee --- /dev/null +++ b/absorb.go @@ -0,0 +1,75 @@ +// Copyright 2026 The vault Authors. +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "archive/tar" + "bufio" + "bytes" + "compress/gzip" + "errors" + "fmt" + "io" + "os" + "path" + "strings" +) + +// absorb reads a tar archive, gzipped or not, from r and adds every regular +// file in it beneath dest, streaming each straight into the vault so no +// plaintext touches the disk. Other entries, and names that would escape +// dest, are reported and skipped. +func absorb(s *store, cat map[string][]*record, r io.Reader, dest string) error { + br := bufio.NewReader(r) + if magic, _ := br.Peek(2); bytes.Equal(magic, []byte{0x1f, 0x8b}) { + zr, err := gzip.NewReader(br) + if err != nil { + return err + } + defer zr.Close() + r = zr + } else { + r = br + } + tr := tar.NewReader(r) + n := 0 + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return fmt.Errorf("reading archive: %w", err) + } + if hdr.Typeflag != tar.TypeReg { + if hdr.Typeflag != tar.TypeDir { + fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", hdr.Name) + } + continue + } + name := strings.TrimPrefix(path.Clean("/"+hdr.Name), "/") + if strings.Contains(hdr.Name, "\\") || name == "" || name != path.Clean(strings.TrimPrefix(strings.TrimPrefix(hdr.Name, "./"), "/")) { + fmt.Fprintf(os.Stderr, "skipping %s: unsafe name\n", hdr.Name) + continue + } + name = path.Join(dest, name) + replaced, err := s.addReader(cat, tr, hdr.Size, hdr.FileInfo().Mode(), hdr.ModTime, name) + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + err = fmt.Errorf("%s: archive ends early", hdr.Name) + } + if err != nil { + return err + } + verb := "added" + if replaced { + verb = "replaced" + } + fmt.Fprintf(os.Stderr, "%s %s\n", verb, name) + n++ + } + if n == 0 { + return errors.New("no files in archive") + } + return nil +} diff --git a/docs/guide/img/genkey.png b/docs/guide/img/genkey.png index 68080ec..1d4786c 100644 Binary files a/docs/guide/img/genkey.png and b/docs/guide/img/genkey.png differ diff --git a/docs/guide/img/help.png b/docs/guide/img/help.png index 0825e65..45d79c8 100644 Binary files a/docs/guide/img/help.png and b/docs/guide/img/help.png differ diff --git a/docs/guide/img/init.png b/docs/guide/img/init.png index 581262c..b99ec09 100644 Binary files a/docs/guide/img/init.png and b/docs/guide/img/init.png differ diff --git a/docs/guide/shots.py b/docs/guide/shots.py index 95d7361..edac702 100755 --- a/docs/guide/shots.py +++ b/docs/guide/shots.py @@ -315,6 +315,35 @@ def main(): t.cmd("ls Scans") shoot("migrate", t) + # absorb: a tar archive piped in, here pulled from a gzipped tarball + box.write("Projects/website/index.html", "

hello

\n") + box.write("Projects/website/css/site.css", "body{}\n") + box.write("Projects/website/img/logo.png", os.urandom(24_311)) + t = Term(box) + t.cmd("tar -czf website.tar.gz -C Projects website") + t.cmd("vault absorb backups < website.tar.gz", P) + t.cmd("tar -c recipes | vault absorb old", P) + t.cmd("vault list backups", P) + shoot("absorb", t) + + # mirror: a tiny local site, served on localhost + site = os.path.join(box.root, "site") + for rel, body in {"index.html": 'About ', + "about.html": 'Home ', + "style.css": "body{background:url(/img/bg.png)}", + "img/me.png": "png", "img/bg.png": "png"}.items(): + os.makedirs(os.path.dirname(os.path.join(site, rel)), exist_ok=True) + with open(os.path.join(site, rel), "w") as f: + f.write(body) + srv = subprocess.Popen([sys.executable, "-m", "http.server", "8000", "--bind", "127.0.0.1"], + cwd=site, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + time.sleep(1) + t = Term(box) + t.cmd("vault mirror http://localhost:8000/ sites", P) + t.cmd("vault get sites/localhost:8000/about.html && cat about.html", P) + shoot("mirror", t) + srv.terminate() + # errors t = Term(box) t.cmd("vault list", "hunter2", highlight=("incorrect",)) diff --git a/docs/guide/vault-guide.tex b/docs/guide/vault-guide.tex index 011a192..3d231dc 100644 --- a/docs/guide/vault-guide.tex +++ b/docs/guide/vault-guide.tex @@ -567,6 +567,22 @@ the whole site, one request at a time, into a folder named after its host. Links inside pages and stylesheets are rewritten to relative paths, so the copy still browses once you get it out. Only the same host is followed. +\subsection{From a tar archive: \texttt{absorb}} +\cmd{absorb} reads a tar archive, gzipped or not, from standard input and +stores every plain file in it, optionally under a folder: + +\begin{verbatim} +tar -cz photos | vault absorb backup +ssh server 'tar -c /etc' | vault absorb server1 +vault absorb old < archive.tar.gz +\end{verbatim} + +Each file is encrypted straight from the stream, so nothing is unpacked to +disk first. Directories need no entry of their own; links, devices and any +name that tries to climb out with \file{..} are skipped with a note. If the +archive stops halfway through a file, that file isn't stored and vault +stops; everything before it is already safely in. + \subsection{Moving in: \texttt{migrate}} \cmd{migrate} works exactly like \cmd{add}, but deletes each original once it's safely stored, then tidies away any folders left empty. It won't take a diff --git a/docs/vault-absorb.1 b/docs/vault-absorb.1 new file mode 100644 index 0000000..15837cd --- /dev/null +++ b/docs/vault-absorb.1 @@ -0,0 +1,73 @@ +.TH VAULT-ABSORB 1 "October 2026" "vault" "User Commands" +.SH NAME +vault-absorb \- add the files in a tar archive to an encrypted vault +.SH SYNOPSIS +.B vault absorb +.RB [ \-\-key +.IR PATH ] +.RB [ \-\-vault +.IR DIR ] +.RI [ FOLDER ] +.B < +.I ARCHIVE +.SH DESCRIPTION +.B vault absorb +reads a tar archive from standard input and stores every regular file in +it, beneath +.I FOLDER +if one is given, keeping the paths, permissions and modification times +recorded in the archive. A gzip-compressed archive is detected and +unpacked automatically. +.PP +Each file is encrypted straight from the stream; nothing is unpacked to +disk. Anything already stored under the same name is replaced. +.PP +Directory entries are not stored (directories exist implicitly). Symbolic +and hard links, devices and other special entries are reported on standard +error and skipped, as is any entry whose name would escape +.I FOLDER +through +.BR .. . +A leading +.B ./ +or +.B / +is dropped. +.PP +The passphrase prompt reads from the terminal, not standard input, so +it still works with an archive piped in. The vault is locked exclusively +for the whole run. +.SH OPTIONS +.TP +.BI \-\-key " PATH" +Key file. Default: $VAULT_KEY, else ~/.vault/key. +.TP +.BI \-\-vault " DIR" +Vault directory. Default: $VAULT_DIR, else ~/.vault/store. +.SH ENVIRONMENT +.TP +.B VAULT_PASSPHRASE +Passphrase, instead of prompting on the terminal. +.SH EXIT STATUS +0 if at least one file was stored and the archive was read to the end. +1 if the archive is malformed or ends early, holds no regular files, or +anything fails; files stored before the failure stay in the vault. +.SH EXAMPLES +Store a directory under +.BR backup/photos/ : +.PP +.RS +tar \-cz photos | vault absorb backup +.RE +.PP +Pull a remote directory straight into the vault: +.PP +.RS +ssh server 'tar \-C /etc \-c .' | vault absorb server1/etc +.RE +.SH SEE ALSO +.BR vault (1), +.BR vault\-add (1), +.BR tar (1) +.SH AUTHOR +Andy Dixon , https://www.dixon.cx diff --git a/docs/vault-add.1 b/docs/vault-add.1 index f89845a..6e3d204 100644 --- a/docs/vault-add.1 +++ b/docs/vault-add.1 @@ -41,6 +41,7 @@ vault add https://example.com/paper.pdf papers .SH SEE ALSO .BR vault (1), .BR vault\-migrate (1), -.BR vault\-mirror (1) +.BR vault\-mirror (1), +.BR vault\-absorb (1) .SH AUTHOR Andy Dixon , https://www.dixon.cx diff --git a/docs/vault.1 b/docs/vault.1 index 439903f..fe3f381 100644 --- a/docs/vault.1 +++ b/docs/vault.1 @@ -33,6 +33,9 @@ Like add, then delete each original. .BR vault\-mirror (1) Crawl a website into the vault. .TP +.BR vault\-absorb (1) +Add every file in a tar archive read from standard input. +.TP .BR vault\-list (1) List stored files (alias: .BR ls ). diff --git a/main.go b/main.go index ceda30f..179aa43 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,9 @@ const usage = `Usage: vault mirror URL [FOLDER] crawl a website (same host only, one request at a time) into FOLDER/HOST, rewriting its links to relative paths + vault absorb [FOLDER] read a tar archive (optionally gzipped) from + standard input and add every file in it, + optionally under a folder inside the vault vault list [PREFIX] list stored files vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name) vault mv OLD NEW rename a stored file @@ -146,6 +149,19 @@ func main() { cat, err := s.catalogue() check(err) check(mirror(s, cat, pos[0], dest)) + case "absorb": + want(0, 1) + if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 { + fail("absorb reads a tar archive from standard input; pipe one in") + } + dest := "" + if len(pos) == 1 { + dest = strings.Trim(pos[0], "/") + } + s := open(*keyPath, *vaultDir, true) + cat, err := s.catalogue() + check(err) + check(absorb(s, cat, os.Stdin, dest)) case "list", "ls": want(0, 1) prefix := "" diff --git a/store.go b/store.go index e02533a..907c114 100644 --- a/store.go +++ b/store.go @@ -14,6 +14,7 @@ import ( "errors" "fmt" "io" + "io/fs" "math" mrand "math/rand/v2" "os" @@ -382,30 +383,43 @@ func (s *store) add(cat map[string][]*record, src, name string) (bool, error) { if !info.Mode().IsRegular() || info.Size() > fileLimit { return false, fmt.Errorf("%s: not a regular file of at most 1 TiB", src) } + replaced, err := s.addReader(cat, f, info.Size(), info.Mode().Perm(), info.ModTime(), name) + if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { + err = fmt.Errorf("%s changed while being added", src) + } + return replaced, err +} + +// addReader stores size bytes from r as name, like add. Reading fewer than +// size bytes stores nothing and fails with io.ErrUnexpectedEOF or io.EOF. +func (s *store) addReader(cat map[string][]*record, r io.Reader, size int64, mode fs.FileMode, mtime time.Time, name string) (bool, error) { + if !validName(name) { + return false, fmt.Errorf("invalid vault path %q", name) + } + if size < 0 || size > fileLimit { + return false, fmt.Errorf("%s: larger than 1 TiB", name) + } seed, err := randomHex(objectSeedSize) if err != nil { return false, err } - if err := s.writeObjects(f, seed, info.Size()); err != nil { - s.dropObjects(seed, info.Size()) - if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) { - err = fmt.Errorf("%s changed while being added", src) - } + if err := s.writeObjects(r, seed, size); err != nil { + s.dropObjects(seed, size) return false, err } - r := &record{ - Name: name, Seed: seed, Size: info.Size(), Mode: uint32(info.Mode().Perm()), - Modified: info.ModTime().UTC(), Added: time.Now().UTC(), + rec := &record{ + Name: name, Seed: seed, Size: size, Mode: uint32(mode.Perm()), + Modified: mtime.UTC(), Added: time.Now().UTC(), } - if err := s.writeRecord(r); err != nil { - s.dropObjects(seed, info.Size()) + if err := s.writeRecord(rec); err != nil { + s.dropObjects(seed, size) return false, err } replaced := len(cat[name]) > 0 if err := s.remove(cat, name, true); err != nil { return replaced, err } - cat[name] = []*record{r} + cat[name] = []*record{rec} return replaced, nil } diff --git a/vault_test.go b/vault_test.go index 05ae53c..146a3a1 100644 --- a/vault_test.go +++ b/vault_test.go @@ -4,8 +4,10 @@ package main import ( + "archive/tar" "bufio" "bytes" + "compress/gzip" "crypto/rand" "io" "io/fs" @@ -430,3 +432,50 @@ func TestMirror(t *testing.T) { os.Remove(out) } } + +func TestAbsorb(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "key") + if err := writeKeyFile(keyPath, "pass", ""); err != nil { + t.Fatal(err) + } + if err := initStore(filepath.Join(dir, "store"), 0); err != nil { + t.Fatal(err) + } + s := newTestStore(t, keyPath, "pass") + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + tw := tar.NewWriter(zw) + mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC) + for _, h := range []*tar.Header{ + {Name: "./d/", Typeflag: tar.TypeDir, Mode: 0o755}, + {Name: "./d/a.txt", Size: 5, Mode: 0o600, ModTime: mtime}, + {Name: "../escape", Size: 5, Mode: 0o600}, + {Name: "./d/link", Typeflag: tar.TypeSymlink, Linkname: "a.txt"}, + } { + tw.WriteHeader(h) + if h.Size > 0 { + io.WriteString(tw, "hello") + } + } + tw.Close() + zw.Close() + + cat, _ := s.catalogue() + if err := absorb(s, cat, &buf, "in"); err != nil { + t.Fatal(err) + } + if len(cat) != 1 || len(cat["in/d/a.txt"]) != 1 { + t.Fatalf("catalogue = %v, want only in/d/a.txt", cat) + } + if r := cat["in/d/a.txt"][0]; !r.Modified.Equal(mtime) || r.Mode != 0o600 { + t.Errorf("record = %v, %o", r.Modified, r.Mode) + } + out := filepath.Join(dir, "out") + if err := s.get("in/d/a.txt", out); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile(out); string(b) != "hello" { + t.Errorf("got %q", b) + } +}