Absorb: ask for the passphrase once the archive starts arriving

With gpg --decrypt x.tar.gpg | vault absorb, vault printed its prompt
while gpg was still writing to the terminal, which buried it, so the
command looked hung. Peek at standard input first: the upstream program
has finished with the terminal by the time data arrives, and an empty
pipe now fails without asking at all.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CLAUDE.md

@@ -8,7 +8,8 @@ vault is a post-quantum encrypted file vault: one `package main` at the repo
 root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
 
 - `main.go` - CLI: `genkey`, `init`, `add` (file, directory
-  or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`.
+  or URL), `migrate` (add, then delete originals), `mirror`, `absorb` (tar
+  on stdin, `absorb.go`), `list`, `get`, `mv`, `rm`.
 - `keyfile.go` - key file with two passphrase slots (Argon2id +
   XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second
   slot is either a decoy identity or random bytes; slot order is random.

README.md

@@ -30,6 +30,12 @@ A full illustrated guide lives in `docs/guide`. Build the PDF with
 `docs/guide/build-pdf.sh`, which fetches Tectonic if it isn't installed.
 `docs/guide/shots.py` regenerates its screenshots from the real binary.
 
+## Man pages
+
+`docs/*.1` holds `vault(1)` and one page per command (`vault-add(1)`,
+`vault-mirror(1)` and so on). Install them with
+`cp docs/*.1 ~/.local/share/man/man1/`.
+
 ## Use
 
 ```sh
@@ -39,6 +45,11 @@ vault init --filler 2G           # the same, with at least 2 GiB of filler
 vault add report.pdf             # stored as report.pdf
 vault add report.pdf work/2026   # stored as work/2026/report.pdf
 vault add ./photos               # every regular file under photos/
+vault add https://example.com/a.pdf   # download and store a URL
+vault migrate ./photos           # like add, then delete the originals
+vault mirror https://example.com/     # crawl a site into example.com/, links rewritten
+tar -cz photos | vault absorb backup  # every file in a tar (or .tar.gz) under backup/
+vault absorb backup < photos.tar.gz   # same, from an existing archive
 vault list                       # everything
 vault list work                  # everything under work/
 vault get work/2026/report.pdf   # writes ./report.pdf

absorb.go

@@ -0,0 +1,75 @@
+// Copyright 2026 The vault Authors.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import (
+	"archive/tar"
+	"bufio"
+	"bytes"
+	"compress/gzip"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"strings"
+)
+
+// absorb reads a tar archive, gzipped or not, from r and adds every regular
+// file in it beneath dest, streaming each straight into the vault so no
+// plaintext touches the disk. Other entries, and names that would escape
+// dest, are reported and skipped.
+func absorb(s *store, cat map[string][]*record, r io.Reader, dest string) error {
+	br := bufio.NewReader(r)
+	if magic, _ := br.Peek(2); bytes.Equal(magic, []byte{0x1f, 0x8b}) {
+		zr, err := gzip.NewReader(br)
+		if err != nil {
+			return err
+		}
+		defer zr.Close()
+		r = zr
+	} else {
+		r = br
+	}
+	tr := tar.NewReader(r)
+	n := 0
+	for {
+		hdr, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return fmt.Errorf("reading archive: %w", err)
+		}
+		if hdr.Typeflag != tar.TypeReg {
+			if hdr.Typeflag != tar.TypeDir {
+				fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", hdr.Name)
+			}
+			continue
+		}
+		name := strings.TrimPrefix(path.Clean("/"+hdr.Name), "/")
+		if strings.Contains(hdr.Name, "\\") || name == "" || name != path.Clean(strings.TrimPrefix(strings.TrimPrefix(hdr.Name, "./"), "/")) {
+			fmt.Fprintf(os.Stderr, "skipping %s: unsafe name\n", hdr.Name)
+			continue
+		}
+		name = path.Join(dest, name)
+		replaced, err := s.addReader(cat, tr, hdr.Size, hdr.FileInfo().Mode(), hdr.ModTime, name)
+		if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
+			err = fmt.Errorf("%s: archive ends early", hdr.Name)
+		}
+		if err != nil {
+			return err
+		}
+		verb := "added"
+		if replaced {
+			verb = "replaced"
+		}
+		fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
+		n++
+	}
+	if n == 0 {
+		return errors.New("no files in archive")
+	}
+	return nil
+}

docs/guide/shots.py

@@ -315,6 +315,35 @@ def main():
     t.cmd("ls Scans")
     shoot("migrate", t)
 
+    # absorb: a tar archive piped in, here pulled from a gzipped tarball
+    box.write("Projects/website/index.html", "<h1>hello</h1>\n")
+    box.write("Projects/website/css/site.css", "body{}\n")
+    box.write("Projects/website/img/logo.png", os.urandom(24_311))
+    t = Term(box)
+    t.cmd("tar -czf website.tar.gz -C Projects website")
+    t.cmd("vault absorb backups < website.tar.gz", P)
+    t.cmd("tar -c recipes | vault absorb old", P)
+    t.cmd("vault list backups", P)
+    shoot("absorb", t)
+
+    # mirror: a tiny local site, served on localhost
+    site = os.path.join(box.root, "site")
+    for rel, body in {"index.html": '<a href="/about.html">About</a> <link href="/style.css">',
+                      "about.html": '<a href="/">Home</a> <img src="/img/me.png">',
+                      "style.css": "body{background:url(/img/bg.png)}",
+                      "img/me.png": "png", "img/bg.png": "png"}.items():
+        os.makedirs(os.path.dirname(os.path.join(site, rel)), exist_ok=True)
+        with open(os.path.join(site, rel), "w") as f:
+            f.write(body)
+    srv = subprocess.Popen([sys.executable, "-m", "http.server", "8000", "--bind", "127.0.0.1"],
+                           cwd=site, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+    time.sleep(1)
+    t = Term(box)
+    t.cmd("vault mirror http://localhost:8000/ sites", P)
+    t.cmd("vault get sites/localhost:8000/about.html && cat about.html", P)
+    shoot("mirror", t)
+    srv.terminate()
+
     # errors
     t = Term(box)
     t.cmd("vault list", "hunter2", highlight=("incorrect",))

docs/guide/vault-guide.tex

@@ -561,6 +561,36 @@ downloads it (as Chrome, so fussy sites play along) and stores it under the
 last part of the address, such as \file{report.pdf}. The download sits in a
 private temporary file only until it's encrypted.
 
+\subsection{Copying a website: \texttt{mirror}}
+\cmd{mirror} takes an \file{http://} or \file{https://} address and copies
+the whole site, one request at a time, into a folder named after its host.
+Links inside pages and stylesheets are rewritten to relative paths, so the
+copy still browses once you get it out. Only the same host is followed.
+
+\shot{mirror}{Mirroring a small site. The links in each page now point at the copy.}
+
+\subsection{From a tar archive: \texttt{absorb}}
+\cmd{absorb} reads a tar archive, gzipped or not, from standard input and
+stores every plain file in it, optionally under a folder:
+
+\begin{verbatim}
+tar -cz photos | vault absorb backup
+ssh server 'tar -c /etc' | vault absorb server1
+vault absorb old < archive.tar.gz
+\end{verbatim}
+
+Each file is encrypted straight from the stream, so nothing is unpacked to
+disk first. Directories need no entry of their own; links, devices and any
+name that tries to climb out with \file{..} are skipped with a note.
+vault asks for your passphrase only once the archive starts arriving, so
+when it comes out of something that asks first, like
+\cmd{gpg --decrypt secrets.tar.gpg | vault absorb}, gpg's prompt comes
+and goes before vault's appears. If the
+archive stops halfway through a file, that file isn't stored and vault
+stops; everything before it is already safely in.
+
+\shot{absorb}{A gzipped tarball and a plain tar stream, both absorbed.}
+
 \subsection{Moving in: \texttt{migrate}}
 \cmd{migrate} works exactly like \cmd{add}, but deletes each original once
 it's safely stored, then tidies away any folders left empty. It won't take a
@@ -934,7 +964,8 @@ covered.
 
 \section{Locking is automatic}
 
-Commands that change the vault (\cmd{add}, \cmd{mv}, \cmd{rm}) take an
+Commands that change the vault (\cmd{add}, \cmd{migrate}, \cmd{mirror},
+\cmd{absorb}, \cmd{mv}, \cmd{rm}) take an
 \hl{exclusive} lock. Commands that only read (\cmd{list}, \cmd{get}) take a
 \hl{shared} one, so several can read at once. If a command has to wait, it
 says so, then carries on by itself:
@@ -1238,12 +1269,16 @@ passphrase. Nothing in this chapter helps with that. Back up your key.
   enhanced,colback=vmist,colframe=vline,boxrule=0.4pt,arc=4pt,left=9pt,right=9pt,top=7pt,bottom=7pt]
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault genkey}\\[2pt]\small Make a key. Asks for a passphrase and an optional decoy.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault init}\\[2pt]\small Make an empty vault, with filler. \texttt{--filler 2G} for more.\end{tcolorbox}
-\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault add FILE|DIR [FOLDER]}\\[2pt]\small Encrypt in, keeping the name. Replaces an existing name.\end{tcolorbox}
+\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault add FILE|DIR|URL [FOLDER]}\\[2pt]\small Encrypt in, keeping the name. Replaces an existing name.\end{tcolorbox}
+\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault migrate FILE|DIR [FOLDER]}\\[2pt]\small Like add, then delete the originals. Asks before replacing.\end{tcolorbox}
+\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault mirror URL [FOLDER]}\\[2pt]\small Crawl a website in, links rewritten to work offline.\end{tcolorbox}
+\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault absorb [FOLDER] < TAR}\\[2pt]\small Every file in a tar or .tar.gz from standard input.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault list [FOLDER]}\\[2pt]\small Size, modified time and name, sorted.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault get NAME [-o OUT]}\\[2pt]\small Decrypt out. Never overwrites. Verifies first.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault mv OLD NEW}\\[2pt]\small Rename, using the full new name. Instant.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault rm NAME}\\[2pt]\small Delete one file. No undo.\end{tcolorbox}
 \begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries vault help}\\[2pt]\small Show the command summary.\end{tcolorbox}
+\begin{tcolorbox}{\ttfamily\color{vtealdeep}\bfseries man vault}\\[2pt]\small A manual page for vault and for every command.\end{tcolorbox}
 \end{tcbraster}
 
 \section*{Options and variables}
@@ -1255,6 +1290,7 @@ passphrase. Nothing in this chapter helps with that. Back up your key.
 \textrm{(genkey prompt)} & VAULT\_DECOY\_PASSPHRASE & asked; empty means none \\
 -o OUT & \textrm{---} & the file's own name, here \\
 --filler SIZE & \textrm{---} & a few tens of MB, for \cmd{init} \\
+--yes & \textrm{---} & ask, for \cmd{migrate} \\
 \end{vtable}
 
 \section*{Numbers worth knowing}

docs/vault-absorb.1

@@ -0,0 +1,84 @@
+.TH VAULT-ABSORB 1 "October 2026" "vault" "User Commands"
+.SH NAME
+vault-absorb \- add the files in a tar archive to an encrypted vault
+.SH SYNOPSIS
+.B vault absorb
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.RI [ FOLDER ]
+.B <
+.I ARCHIVE
+.SH DESCRIPTION
+.B vault absorb
+reads a tar archive from standard input and stores every regular file in
+it, beneath
+.I FOLDER
+if one is given, keeping the paths, permissions and modification times
+recorded in the archive. A gzip-compressed archive is detected and
+unpacked automatically.
+.PP
+Each file is encrypted straight from the stream; nothing is unpacked to
+disk. Anything already stored under the same name is replaced.
+.PP
+Directory entries are not stored (directories exist implicitly). Symbolic
+and hard links, devices and other special entries are reported on standard
+error and skipped, as is any entry whose name would escape
+.I FOLDER
+through
+.BR .. .
+A leading
+.B ./
+or
+.B /
+is dropped.
+.PP
+The passphrase prompt reads from the terminal, not standard input, so
+it still works with an archive piped in. It waits until the archive starts
+arriving, so a program feeding the pipe, such as
+.BR gpg (1)
+asking for its own passphrase, has finished with the terminal first.
+An empty pipe fails without asking at all.
+The vault is locked exclusively for the whole run.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH EXIT STATUS
+0 if at least one file was stored and the archive was read to the end.
+1 if the archive is malformed or ends early, holds no regular files, or
+anything fails; files stored before the failure stay in the vault.
+.SH EXAMPLES
+Unpack an existing archive into the vault under
+.BR backup/ :
+.PP
+.RS
+vault absorb backup < photos.tar.gz
+.RE
+.PP
+Store a directory under
+.BR backup/photos/ :
+.PP
+.RS
+tar \-cz photos | vault absorb backup
+.RE
+.PP
+Pull a remote directory straight into the vault:
+.PP
+.RS
+ssh server 'tar \-C /etc \-c .' | vault absorb server1/etc
+.RE
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-add (1),
+.BR tar (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-add.1

@@ -0,0 +1,47 @@
+.TH VAULT-ADD 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-add \- encrypt files, directories or URLs into the vault
+.SH SYNOPSIS
+.B vault add
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.IR FILE | DIR | URL
+.RI [ FOLDER ]
+.SH DESCRIPTION
+Encrypts a file into the vault, named after its base name. A directory is
+added recursively, keeping its structure beneath its own name; a directory
+named "." or ".." adds its contents directly. Non-regular files, the vault
+and the key are skipped. A stored name that already exists is replaced.
+.PP
+An \fBhttp://\fR or \fBhttps://\fR \fIURL\fR is downloaded with a Chrome user
+agent and stored under the last element of its final (post-redirect) path, or
+the host name if there is none. Only a 200 response is accepted.
+.PP
+\fIFOLDER\fR places the result inside a folder in the vault. File contents
+are cut into 4 MiB chunks, the last bucket-padded.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH EXAMPLES
+.nf
+vault add report.pdf work/2026
+vault add ./photos
+vault add https://example.com/paper.pdf papers
+.fi
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-migrate (1),
+.BR vault\-mirror (1),
+.BR vault\-absorb (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-genkey.1

@@ -0,0 +1,32 @@
+.TH VAULT-GENKEY 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-genkey \- create a new vault key
+.SH SYNOPSIS
+.B vault genkey
+.RB [ \-\-key
+.IR PATH ]
+.SH DESCRIPTION
+Creates the key file, prompting twice for a passphrase, then for an optional
+decoy passphrase (empty for none). Both passphrase slots are sealed with
+Argon2id and XChaCha20-Poly1305, are the same size, and are stored in random
+order; the second holds either a decoy identity or random bytes. It refuses to
+overwrite an existing file. No seed or private key is ever written unencrypted.
+.PP
+When \fBVAULT_PASSPHRASE\fR is set, no prompts are made and the decoy comes
+from \fBVAULT_DECOY_PASSPHRASE\fR, or is omitted.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Where to write the key. Default: $VAULT_KEY, else ~/.vault/key.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.TP
+.B VAULT_DECOY_PASSPHRASE
+Decoy passphrase; must differ from the passphrase.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-init (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-get.1

@@ -0,0 +1,32 @@
+.TH VAULT-GET 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-get \- decrypt a stored file
+.SH SYNOPSIS
+.B vault get
+.RB [ \-o
+.IR OUTPUT ]
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.I NAME
+.SH DESCRIPTION
+Decrypts \fINAME\fR. Nothing appears at the output path until every object has
+authenticated. The stored mode and modification time are restored. It refuses
+to overwrite an existing file. Takes a shared lock.
+.SH OPTIONS
+.TP
+.BI \-o " OUTPUT"
+Output path. Default: the base name of \fINAME\fR in the current directory.
+.PP
+See also \fB\-\-key\fR and \fB\-\-vault\fR in
+.BR vault (1).
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-list (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-init.1

@@ -0,0 +1,27 @@
+.TH VAULT-INIT 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-init \- create an empty vault
+.SH SYNOPSIS
+.B vault init
+.RB [ \-\-vault
+.IR DIR ]
+.RB [ \-\-filler
+.IR SIZE ]
+.SH DESCRIPTION
+Creates the vault directory, with \fBrecords/\fR and \fBobjects/\fR each
+sharded into 256 subdirectories. It writes at least \fISIZE\fR bytes of
+unreadable filler so that real files cannot be told apart from it by anyone
+lacking the right passphrase. Filler is always written, even when \fISIZE\fR is 0.
+.SH OPTIONS
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.TP
+.BI \-\-filler " SIZE"
+Minimum filler, in bytes, with an optional K, M, G or T suffix (powers of
+1024), e.g. 500M or 2G. Default: 0.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-genkey (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-list.1

@@ -0,0 +1,34 @@
+.TH VAULT-LIST 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-list \- list files in the vault
+.SH SYNOPSIS
+.B vault list
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.RI [ PREFIX ]
+.br
+.B vault ls
+\&...
+.SH DESCRIPTION
+Prints one line per stored file, sorted by name: human-readable size,
+modification time (local, minute resolution) and name. With \fIPREFIX\fR only
+that file, or files under that folder, are shown. Records the passphrase
+cannot open are skipped silently. Takes a shared lock.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-get (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-migrate.1

@@ -0,0 +1,41 @@
+.TH VAULT-MIGRATE 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-migrate \- move files into the vault, deleting the originals
+.SH SYNOPSIS
+.B vault migrate
+.RB [ \-\-yes ]
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.IR FILE | DIR
+.RI [ FOLDER ]
+.SH DESCRIPTION
+Works like
+.BR vault\-add (1)
+but deletes each original once it is stored, then any directories left empty.
+Directories still holding skipped files stay. A URL cannot be migrated.
+.PP
+Before storing anything it lists the vault names that would be overwritten and
+asks for confirmation on the terminal. With no terminal the answer is no, and
+nothing changes.
+.SH OPTIONS
+.TP
+.B \-\-yes
+Overwrite without asking.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-add (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-mirror.1

@@ -0,0 +1,86 @@
+.TH VAULT-MIRROR 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-mirror \- mirror a website into an encrypted vault
+.SH SYNOPSIS
+.B vault mirror
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.I URL
+.RI [ FOLDER ]
+.SH DESCRIPTION
+.B vault mirror
+crawls the website at
+.I URL
+and stores every page and asset it finds in the vault, beneath
+.IR FOLDER / HOST
+(or just
+.I HOST
+if no folder is given).
+.PP
+The crawl is single threaded: one request at a time, in breadth-first order.
+It follows only links on the same host as
+.IR URL ,
+and requests are sent with a Chrome user agent string.
+.PP
+Links in HTML
+.RB ( href ,
+.BR src ,
+.BR action
+and
+.B poster
+attributes) and in CSS
+.RB ( url(...) )
+that point into the mirror are rewritten to relative paths, so the copy
+browses correctly once fetched out of the vault. Links to other hosts, and
+fragments, are left as they were.
+.PP
+Pages are named after their URL path. A path that is empty or ends in
+.B /
+becomes
+.BR index.html ;
+a query string is appended as
+.BI _ query
+(URL-escaped). Anything already stored under the same name is replaced.
+.PP
+A page that fails to download, or answers with anything but
+.BR "200 OK" ,
+is reported on standard error and skipped; the crawl carries on.
+The vault is locked exclusively for the whole run.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH EXAMPLES
+Mirror a site into the vault under
+.BR example.com/ :
+.PP
+.RS
+vault mirror https://example.com/
+.RE
+.PP
+Mirror it under a folder, then fetch the front page:
+.PP
+.RS
+vault mirror https://example.com/ archive/2026
+.br
+vault get archive/2026/example.com/index.html
+.RE
+.SH LIMITATIONS
+Links are found by pattern matching, not by parsing HTML: URLs assembled by
+scripts, and
+.B srcset
+lists, are neither followed nor rewritten. There is no depth limit, rate
+limit or robots.txt handling; only mirror sites you may copy.
+.SH SEE ALSO
+.BR vault (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-mv.1

@@ -0,0 +1,33 @@
+.TH VAULT-MV 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-mv \- rename a stored file
+.SH SYNOPSIS
+.B vault mv
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.I OLD NEW
+.br
+.B vault rename
+\&...
+.SH DESCRIPTION
+Renames \fIOLD\fR to \fINEW\fR, which may be in another folder. Fails if
+\fINEW\fR exists or is not a valid vault path. File contents are not
+re-encrypted.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-rm (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault-rm.1

@@ -0,0 +1,32 @@
+.TH VAULT-RM 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault-rm \- delete a stored file
+.SH SYNOPSIS
+.B vault rm
+.RB [ \-\-key
+.IR PATH ]
+.RB [ \-\-vault
+.IR DIR ]
+.I NAME
+.br
+.B vault delete
+\&...
+.SH DESCRIPTION
+Deletes the stored file \fINAME\fR from the vault. It does not ask for
+confirmation.
+.SH OPTIONS
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.SH SEE ALSO
+.BR vault (1),
+.BR vault\-list (1)
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

docs/vault.1

@@ -0,0 +1,109 @@
+.TH VAULT 1 "September 2026" "vault" "User Commands"
+.SH NAME
+vault \- post-quantum encrypted file vault
+.SH SYNOPSIS
+.B vault
+.I command
+.RI [ options ]
+.RI [ arguments ]
+.SH DESCRIPTION
+.B vault
+stores files in a directory of encrypted, fixed-size records and chunks.
+Encryption is a hybrid of FrodoKEM-640 and ML-KEM-768 with ChaCha20-Poly1305.
+The key file is protected by an Argon2id-derived passphrase key.
+File names, sizes and contents are only visible to a holder of the key and
+passphrase. Ciphertext sizes and timestamps are normalised to hide
+metadata from the storage host.
+.PP
+Options may appear before, between or after the arguments.
+.SH COMMANDS
+.TP
+.BR vault\-genkey (1)
+Create a new passphrase-protected key file.
+.TP
+.BR vault\-init (1)
+Create a new, empty vault.
+.TP
+.BR vault\-add (1)
+Encrypt a file, directory or downloaded URL into the vault.
+.TP
+.BR vault\-migrate (1)
+Like add, then delete each original.
+.TP
+.BR vault\-mirror (1)
+Crawl a website into the vault.
+.TP
+.BR vault\-absorb (1)
+Add every file in a tar archive read from standard input.
+.TP
+.BR vault\-list (1)
+List stored files (alias:
+.BR ls ).
+.TP
+.BR vault\-get (1)
+Decrypt a stored file.
+.TP
+.BR vault\-mv (1)
+Rename a stored file (alias:
+.BR rename ).
+.TP
+.BR vault\-rm (1)
+Delete a stored file (alias:
+.BR delete ).
+.TP
+.B help
+Print usage.
+.SH OPTIONS
+These apply to every command.
+.TP
+.BI \-\-key " PATH"
+Key file. Default: $VAULT_KEY, else ~/.vault/key.
+.TP
+.BI \-\-vault " DIR"
+Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
+.SH ENVIRONMENT
+.TP
+.B VAULT_KEY
+Default key file.
+.TP
+.B VAULT_DIR
+Default vault directory.
+.TP
+.B VAULT_PASSPHRASE
+Passphrase, instead of prompting on the terminal.
+.TP
+.B VAULT_DECOY_PASSPHRASE
+Decoy passphrase used by
+.BR "vault genkey" .
+.SH LOCKING
+Commands that change the vault hold an exclusive lock on
+.IR DIR/.lock ;
+readers hold a shared lock. The lock is taken after the passphrase prompt.
+.SH DECOY PASSPHRASE
+.B genkey
+can add a second passphrase. It unlocks a separate identity, so every
+command run with it sees and changes only the decoy files, and the real
+files stay invisible. The key file does not reveal whether a decoy exists,
+and
+.B init
+fills each vault with unreadable filler, so hidden files look like filler to
+anyone holding only the decoy passphrase.
+.SH EXIT STATUS
+0 on success; 1 on error (diagnostics on standard error); 2 when run with no command.
+.SH EXAMPLES
+.nf
+vault genkey
+vault init \-\-filler 500M
+vault add report.pdf work/2026
+vault list work
+vault get work/2026/report.pdf \-o /tmp/report.pdf
+.fi
+.SH FILES
+.TP
+.I ~/.vault/key
+Default key file.
+.TP
+.I ~/.vault/store
+Default vault directory.
+.SH AUTHOR
+Andy Dixon <vault@dixon.cx>, https://www.dixon.cx

main.go

@@ -5,6 +5,7 @@
 package main
 
 import (
+	"bufio"
 	"errors"
 	"flag"
 	"fmt"
@@ -32,6 +33,12 @@ const usage = `Usage:
     [--yes]                         it is stored; asks before overwriting
                                     anything already in the vault, unless
                                     --yes is given
+    vault mirror URL [FOLDER]       crawl a website (same host only, one request
+                                    at a time) into FOLDER/HOST, rewriting its
+                                    links to relative paths
+    vault absorb [FOLDER]           read a tar archive (optionally gzipped) from
+                                    standard input and add every file in it,
+                                    optionally under a folder inside the vault
     vault list [PREFIX]             list stored files
     vault get NAME [-o OUTPUT]      decrypt a stored file (default: its base name)
     vault mv OLD NEW                rename a stored file
@@ -130,6 +137,39 @@ func main() {
 			overwrite = func([]string) bool { return true }
 		}
 		check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
+	case "mirror":
+		want(1, 2)
+		if !isURL(pos[0]) {
+			fail("mirror needs an http:// or https:// URL")
+		}
+		dest := ""
+		if len(pos) == 2 {
+			dest = strings.Trim(pos[1], "/")
+		}
+		s := open(*keyPath, *vaultDir, true)
+		cat, err := s.catalogue()
+		check(err)
+		check(mirror(s, cat, pos[0], dest))
+	case "absorb":
+		want(0, 1)
+		if fi, err := os.Stdin.Stat(); err == nil && fi.Mode()&os.ModeCharDevice != 0 {
+			fail("absorb reads a tar archive from standard input; pipe one in")
+		}
+		dest := ""
+		if len(pos) == 1 {
+			dest = strings.Trim(pos[0], "/")
+		}
+		// Wait for the archive to start arriving before asking for the
+		// passphrase, so a program feeding the pipe (gpg, ssh) has finished
+		// with the terminal and cannot bury the prompt.
+		in := bufio.NewReader(os.Stdin)
+		if _, err := in.Peek(1); err != nil {
+			fail("nothing to absorb on standard input")
+		}
+		s := open(*keyPath, *vaultDir, true)
+		cat, err := s.catalogue()
+		check(err)
+		check(absorb(s, cat, in, dest))
 	case "list", "ls":
 		want(0, 1)
 		prefix := ""

mirror.go

@@ -0,0 +1,150 @@
+// Copyright 2026 The vault Authors.
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+package main
+
+import (
+	"fmt"
+	"html"
+	"io"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+)
+
+var (
+	// Attribute values in HTML, and url(...) in CSS: group 1 is everything
+	// up to the URL, group 2 is the URL.
+	htmlURLs = regexp.MustCompile(`(?i)(\b(?:href|src|action|poster)\s*=\s*["'])([^"']*)`)
+	cssURLs  = regexp.MustCompile(`(?i)(url\(\s*["']?)([^"')\s]+)`)
+)
+
+// mirror crawls the site at rawURL, one request at a time, staying on its
+// host, and adds every page and asset beneath dest/host. Links in HTML and
+// CSS that point into the mirror are rewritten to relative paths, so the
+// copy browses correctly once fetched out of the vault. A page that fails
+// is reported and skipped.
+// ponytail: regex rewriting, so links built by scripts or in srcset are left
+// as they were; swap in golang.org/x/net/html if that matters.
+func mirror(s *store, cat map[string][]*record, rawURL, dest string) error {
+	start, err := url.Parse(rawURL)
+	if err != nil {
+		return err
+	}
+	host := strings.ToLower(start.Host)
+	root := path.Join(dest, host)
+	seen := map[string]bool{}
+	queue := []*url.URL{start}
+	seen[key(start)] = true
+
+	for len(queue) > 0 {
+		u := queue[0]
+		queue = queue[1:]
+		err := mirrorOne(s, cat, u, host, root, func(l *url.URL) {
+			if k := key(l); !seen[k] {
+				seen[k] = true
+				queue = append(queue, l)
+			}
+		})
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "skipping %s: %v\n", u, err)
+		}
+	}
+	return nil
+}
+
+// key identifies a page independent of its fragment.
+func key(u *url.URL) string {
+	c := *u
+	c.Fragment = ""
+	return c.String()
+}
+
+// localPath is where u lives inside the mirror, relative to its root.
+func localPath(u *url.URL) string {
+	p := strings.TrimPrefix(u.Path, "/")
+	if p == "" || strings.HasSuffix(p, "/") {
+		p += "index.html"
+	}
+	if u.RawQuery != "" {
+		p += "_" + url.QueryEscape(u.RawQuery)
+	}
+	return p
+}
+
+// relative gives the link from the file "from" to the file "to", both
+// slash-separated paths relative to the same root.
+func relative(from, to string) string {
+	f := strings.Split(path.Dir(from), "/")
+	t := strings.Split(to, "/")
+	if f[0] == "." {
+		f = nil
+	}
+	for len(f) > 0 && len(t) > 1 && f[0] == t[0] {
+		f, t = f[1:], t[1:]
+	}
+	return strings.Repeat("../", len(f)) + strings.Join(t, "/")
+}
+
+func mirrorOne(s *store, cat map[string][]*record, u *url.URL, host, root string, enqueue func(*url.URL)) error {
+	req, err := http.NewRequest("GET", u.String(), nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", chromeUA)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s", resp.Status)
+	}
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+
+	self := localPath(u)
+	// rewrite maps a link found in the page to a relative one, queueing it.
+	rewrite := func(re *regexp.Regexp, text string) string {
+		return re.ReplaceAllStringFunc(text, func(m string) string {
+			sub := re.FindStringSubmatch(m)
+			l, err := u.Parse(html.UnescapeString(sub[2]))
+			if err != nil || (l.Scheme != "http" && l.Scheme != "https") || strings.ToLower(l.Host) != host {
+				return m
+			}
+			frag := l.Fragment
+			l.Fragment = ""
+			if l.Path == "" && l.RawQuery == "" {
+				l.Path = "/"
+			}
+			enqueue(l)
+			r := relative(self, localPath(l))
+			if frag != "" {
+				r += "#" + frag
+			}
+			return sub[1] + r
+		})
+	}
+	switch ct := strings.ToLower(resp.Header.Get("Content-Type")); {
+	case strings.HasPrefix(ct, "text/html"):
+		body = []byte(rewrite(cssURLs, rewrite(htmlURLs, string(body))))
+	case strings.HasPrefix(ct, "text/css"):
+		body = []byte(rewrite(cssURLs, string(body)))
+	}
+
+	tmp, err := os.CreateTemp("", "vault-mirror-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	defer tmp.Close()
+	if _, err := tmp.Write(body); err != nil {
+		return err
+	}
+	return addOne(s, cat, tmp.Name(), path.Join(root, self), false)
+}

store.go

@@ -14,6 +14,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"math"
 	mrand "math/rand/v2"
 	"os"
@@ -382,30 +383,43 @@ func (s *store) add(cat map[string][]*record, src, name string) (bool, error) {
 	if !info.Mode().IsRegular() || info.Size() > fileLimit {
 		return false, fmt.Errorf("%s: not a regular file of at most 1 TiB", src)
 	}
+	replaced, err := s.addReader(cat, f, info.Size(), info.Mode().Perm(), info.ModTime(), name)
+	if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
+		err = fmt.Errorf("%s changed while being added", src)
+	}
+	return replaced, err
+}
+
+// addReader stores size bytes from r as name, like add. Reading fewer than
+// size bytes stores nothing and fails with io.ErrUnexpectedEOF or io.EOF.
+func (s *store) addReader(cat map[string][]*record, r io.Reader, size int64, mode fs.FileMode, mtime time.Time, name string) (bool, error) {
+	if !validName(name) {
+		return false, fmt.Errorf("invalid vault path %q", name)
+	}
+	if size < 0 || size > fileLimit {
+		return false, fmt.Errorf("%s: larger than 1 TiB", name)
+	}
 	seed, err := randomHex(objectSeedSize)
 	if err != nil {
 		return false, err
 	}
-	if err := s.writeObjects(f, seed, info.Size()); err != nil {
+	if err := s.writeObjects(r, seed, size); err != nil {
-		s.dropObjects(seed, info.Size())
+		s.dropObjects(seed, size)
-		if errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
-			err = fmt.Errorf("%s changed while being added", src)
-		}
 		return false, err
 	}
-	r := &record{
+	rec := &record{
-		Name: name, Seed: seed, Size: info.Size(), Mode: uint32(info.Mode().Perm()),
+		Name: name, Seed: seed, Size: size, Mode: uint32(mode.Perm()),
-		Modified: info.ModTime().UTC(), Added: time.Now().UTC(),
+		Modified: mtime.UTC(), Added: time.Now().UTC(),
 	}
-	if err := s.writeRecord(r); err != nil {
+	if err := s.writeRecord(rec); err != nil {
-		s.dropObjects(seed, info.Size())
+		s.dropObjects(seed, size)
 		return false, err
 	}
 	replaced := len(cat[name]) > 0
 	if err := s.remove(cat, name, true); err != nil {
 		return replaced, err
 	}
-	cat[name] = []*record{r}
+	cat[name] = []*record{rec}
 	return replaced, nil
 }
 

vault_test.go

@@ -4,8 +4,10 @@
 package main
 
 import (
+	"archive/tar"
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"crypto/rand"
 	"io"
 	"io/fs"
@@ -373,3 +375,107 @@ func TestAddTree(t *testing.T) {
 		t.Fatalf("downloaded %q", b)
 	}
 }
+
+// TestMirror crawls a two-page site: links in HTML and CSS become relative,
+// off-host links stay, and everything is stored under host/.
+func TestMirror(t *testing.T) {
+	dir := t.TempDir()
+	keyPath := filepath.Join(dir, "key")
+	if err := writeKeyFile(keyPath, "pass", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
+		t.Fatal(err)
+	}
+	s := newTestStore(t, keyPath, "pass")
+	var srv *httptest.Server
+	srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.UserAgent() != chromeUA {
+			http.Error(w, "bad agent", http.StatusForbidden)
+			return
+		}
+		switch r.URL.Path {
+		case "/":
+			w.Header().Set("Content-Type", "text/html")
+			io.WriteString(w, `<a href="`+srv.URL+`/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>`)
+		case "/sub/p.html":
+			w.Header().Set("Content-Type", "text/html")
+			io.WriteString(w, `<a href="/">home</a> <img src="../img/a.png">`)
+		case "/s.css":
+			w.Header().Set("Content-Type", "text/css")
+			io.WriteString(w, `body{background:url(/img/a.png)}`)
+		case "/img/a.png":
+			io.WriteString(w, "png")
+		default:
+			http.NotFound(w, r)
+		}
+	}))
+	defer srv.Close()
+	cat, _ := s.catalogue()
+	if err := mirror(s, cat, srv.URL, "m"); err != nil {
+		t.Fatal(err)
+	}
+	host := srv.Listener.Addr().String()
+	for name, want := range map[string]string{
+		"m/" + host + "/index.html": `<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>`,
+		"m/" + host + "/sub/p.html": `<a href="../index.html">home</a> <img src="../img/a.png">`,
+		"m/" + host + "/s.css":      `body{background:url(img/a.png)}`,
+		"m/" + host + "/img/a.png":  "png",
+	} {
+		out := filepath.Join(dir, "out")
+		if err := s.get(name, out); err != nil {
+			t.Fatal(name, err)
+		}
+		if b, _ := os.ReadFile(out); string(b) != want {
+			t.Errorf("%s = %q, want %q", name, b, want)
+		}
+		os.Remove(out)
+	}
+}
+
+func TestAbsorb(t *testing.T) {
+	dir := t.TempDir()
+	keyPath := filepath.Join(dir, "key")
+	if err := writeKeyFile(keyPath, "pass", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
+		t.Fatal(err)
+	}
+	s := newTestStore(t, keyPath, "pass")
+	var buf bytes.Buffer
+	zw := gzip.NewWriter(&buf)
+	tw := tar.NewWriter(zw)
+	mtime := time.Date(2020, 1, 2, 3, 4, 5, 0, time.UTC)
+	for _, h := range []*tar.Header{
+		{Name: "./d/", Typeflag: tar.TypeDir, Mode: 0o755},
+		{Name: "./d/a.txt", Size: 5, Mode: 0o600, ModTime: mtime},
+		{Name: "../escape", Size: 5, Mode: 0o600},
+		{Name: "./d/link", Typeflag: tar.TypeSymlink, Linkname: "a.txt"},
+	} {
+		tw.WriteHeader(h)
+		if h.Size > 0 {
+			io.WriteString(tw, "hello")
+		}
+	}
+	tw.Close()
+	zw.Close()
+
+	cat, _ := s.catalogue()
+	if err := absorb(s, cat, &buf, "in"); err != nil {
+		t.Fatal(err)
+	}
+	if len(cat) != 1 || len(cat["in/d/a.txt"]) != 1 {
+		t.Fatalf("catalogue = %v, want only in/d/a.txt", cat)
+	}
+	if r := cat["in/d/a.txt"][0]; !r.Modified.Equal(mtime) || r.Mode != 0o600 {
+		t.Errorf("record = %v, %o", r.Modified, r.Mode)
+	}
+	out := filepath.Join(dir, "out")
+	if err := s.get("in/d/a.txt", out); err != nil {
+		t.Fatal(err)
+	}
+	if b, _ := os.ReadFile(out); string(b) != "hello" {
+		t.Errorf("got %q", b)
+	}
+}