Re-shoot the guide's screenshots and add one for migrate

shots.py gains a pseudo-terminal runner so it can answer migrate's
overwrite question, and the guide shows the resulting screenshot.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CLAUDE.md

@@ -7,7 +7,8 @@ This file provides guidance to Claude Code when working with this repository.
 vault is a post-quantum encrypted file vault: one `package main` at the repo
 root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
 
-- `main.go` - CLI: `genkey`, `init`, `add`, `list`, `get`, `mv`, `rm`.
+- `main.go` - CLI: `genkey`, `init`, `add` (file, directory
+  or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`.
 - `keyfile.go` - key file with two passphrase slots (Argon2id +
   XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second
   slot is either a decoy identity or random bytes; slot order is random.

docs/guide/shots.py

@@ -16,6 +16,7 @@ import glob
 import grp
 import html
 import os
+import pty
 import shutil
 import subprocess
 import sys
@@ -94,6 +95,28 @@ class Sandbox:
                            stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
         return self.tidy(p.stdout)
 
+    def run_tty(self, command, passphrase, answer):
+        """Run on a pseudo-terminal, typing answer at vault's [y/N] question."""
+        env = self.env({"VAULT_PASSPHRASE": passphrase})
+        pid, fd = pty.fork()
+        if pid == 0:
+            os.chdir(self.home)
+            os.execve("/bin/bash", ["bash", "-c", command], env)
+        out = b""
+        while b"[y/N]" not in out:
+            out += os.read(fd, 1024)
+        os.write(fd, answer.encode() + b"\n")
+        try:
+            while True:
+                chunk = os.read(fd, 1024)
+                if not chunk:
+                    break
+                out += chunk
+        except OSError:
+            pass
+        os.waitpid(pid, 0)
+        return self.tidy(out.decode().replace("\r\n", "\n"))
+
     def tidy(self, text):
         owner = " %s %s " % (getpass.getuser(), grp.getgrgid(os.getgid()).gr_name)
         return text.replace(self.home, SHOWN_HOME).replace(owner, " sam sam ").rstrip("\n")
@@ -278,6 +301,20 @@ def main():
     t.cmd("find ~/.vault/store/records -type f | wc -l")
     shoot("decoy-hidden", t)
 
+    # migrate: one file is already in the vault, so it asks first
+    box.write("Scans/receipts/march.pdf", os.urandom(48_113))
+    box.write("Scans/receipts/april.pdf", os.urandom(51_870))
+    box.run("vault add Scans/receipts/march.pdf receipts", P)
+    t = Term(box)
+    t.lines.append(prompt() + '<span class="c">vault migrate Scans/receipts</span>')
+    for line in box.run_tty("vault migrate Scans/receipts", P, "y").split("\n"):
+        esc = html.escape(line)
+        if "[y/N]" in line:
+            esc = '<span class="hl">%s</span>' % esc
+        t.lines.append(esc)
+    t.cmd("ls Scans")
+    shoot("migrate", t)
+
     # errors
     t = Term(box)
     t.cmd("vault list", "hunter2", highlight=("incorrect",))

docs/guide/vault-guide.tex

@@ -522,7 +522,7 @@ for your passphrase. That's the whole daily workflow.
 \section{Putting things in: \texttt{add}}
 
 \begin{shell}
-$ vault add FILE-OR-FOLDER [FOLDER-IN-VAULT]
+$ vault add FILE-OR-FOLDER-OR-URL [FOLDER-IN-VAULT]
 \end{shell}
 
 Give it a file, and it goes in under its own name. Give it a second argument,
@@ -540,6 +540,8 @@ Notice what happened there:
         \file{id/passport-scan.jpg}.
   \item Adding the folder \file{Documents/letters} kept its own name and
         structure: \file{letters/bank.txt}, \file{letters/landlord.txt}.
+  \item Adding \file{.} (or \file{..}) brings in that folder's
+        \emph{contents}, subfolders and all, without a folder name of its own.
 \end{itemize}
 
 \begin{headsup}
@@ -553,9 +555,30 @@ Adding a name that already exists \hl{replaces} it, and vault tells you so.
 The old version is deleted \emph{after} the new one is safely stored, so a
 crash halfway through never leaves you with neither.
 
+\subsection{Straight from the web}
+Give \cmd{add} an \file{http://} or \file{https://} address and vault
+downloads it (as Chrome, so fussy sites play along) and stores it under the
+last part of the address, such as \file{report.pdf}. The download sits in a
+private temporary file only until it's encrypted.
+
+\subsection{Moving in: \texttt{migrate}}
+\cmd{migrate} works exactly like \cmd{add}, but deletes each original once
+it's safely stored, then tidies away any folders left empty. It won't take a
+URL; there's no original to delete.
+
+Because the originals go, \cmd{migrate} checks first: if anything would
+\hl{replace} a file already in the vault, it lists those names and asks
+before touching anything. Answer anything but \texttt{y} and nothing changes.
+With no terminal to ask on, the answer is no. For scripts, \texttt{--yes}
+answers yes up front.
+
+\shot{migrate}{Moving a folder in. One file was already there, so vault asked first.}
+
 \subsection{What gets skipped}
 When adding a folder, anything that isn't a plain file, such as symbolic
-links, sockets or devices, is skipped with a note. Files larger than 1\,TiB are
+links, sockets or devices, is skipped with a note. So are the vault and key
+themselves, should they live inside the folder, so \cmd{migrate} can never
+eat them. Files larger than 1\,TiB are
 refused. (If you have a single 1\,TiB file, we have questions.)
 
 \section{Seeing what's inside: \texttt{list}}

internal/term/term.go

@@ -1,9 +1,11 @@
 package term
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 
 	"golang.org/x/term"
 )
@@ -72,6 +74,23 @@ func ReadSecret(prompt string) (s []byte, err error) {
 	return
 }
 
+// Confirm asks a yes/no question on the terminal. Only "y" or "yes" is yes.
+func Confirm(prompt string) (yes bool, err error) {
+	err = WithTerminal(func(in, out *os.File) error {
+		fmt.Fprintf(out, "%s ", prompt)
+		line, err := bufio.NewReader(in).ReadString('\n')
+		if err != nil && line == "" {
+			return err
+		}
+		switch strings.ToLower(strings.TrimSpace(line)) {
+		case "y", "yes":
+			yes = true
+		}
+		return nil
+	})
+	return
+}
+
 // ReadPassphrase returns the passphrase from the environment variable env if
 // it is set, otherwise prompts on the terminal.
 func ReadPassphrase(prompt, env string) (string, error) {

main.go

@@ -5,10 +5,12 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/fs"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -23,8 +25,13 @@ const usage = `Usage:
     vault genkey                    create a new passphrase-protected key
     vault init [--filler SIZE]      create a new, empty vault, with at least
                                     SIZE of filler (e.g. 500M, 2G)
-    vault add FILE|DIR [FOLDER]     encrypt a file or directory into the vault,
+    vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively)
+                                    or a downloaded URL into the vault,
                                     optionally under a folder inside the vault
+    vault migrate FILE|DIR [FOLDER] like add, then delete each original once
+    [--yes]                         it is stored; asks before overwriting
+                                    anything already in the vault, unless
+                                    --yes is given
     vault list [PREFIX]             list stored files
     vault get NAME [-o OUTPUT]      decrypt a stored file (default: its base name)
     vault mv OLD NEW                rename a stored file
@@ -65,6 +72,7 @@ func main() {
 	vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "")
 	output := fset.String("o", "", "")
 	filler := fset.String("filler", "0", "")
+	yes := fset.Bool("yes", false, "")
 	pos, err := parseInterleaved(fset, args)
 	if err != nil {
 		fail("%v\n\n%s", err, usage)
@@ -94,7 +102,7 @@ func main() {
 		check(err)
 		check(initStore(*vaultDir, size))
 		fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir)
-	case "add":
+	case "add", "migrate":
 		want(1, 2)
 		dest := ""
 		if len(pos) == 2 {
@@ -103,7 +111,25 @@ func main() {
 		s := open(*keyPath, *vaultDir, true)
 		cat, err := s.catalogue()
 		check(err)
-		check(addPath(s, cat, pos[0], dest))
+		if isURL(pos[0]) {
+			if cmd == "migrate" {
+				fail("cannot migrate a URL; use add")
+			}
+			check(addURL(s, cat, pos[0], dest))
+			break
+		}
+		// Never ingest, or delete, the vault or its key.
+		var skip []os.FileInfo
+		for _, p := range []string{*vaultDir, *keyPath} {
+			if fi, err := os.Stat(p); err == nil {
+				skip = append(skip, fi)
+			}
+		}
+		overwrite := confirmOverwrite
+		if *yes {
+			overwrite = func([]string) bool { return true }
+		}
+		check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
 	case "list", "ls":
 		want(0, 1)
 		prefix := ""
@@ -228,20 +254,52 @@ func decoyPassphrase() string {
 	return string(p)
 }
 
-// addPath adds a file, or every regular file under a directory, beneath dest.
-func addPath(s *store, cat map[string][]*record, src, dest string) error {
+// addPath adds a file, or every regular file under a directory, beneath dest,
+// keeping the directory structure. A directory named "." or ".." adds its
+// contents directly beneath dest. With migrate, each original is deleted
+// once stored, then any directories left empty, and nothing is stored
+// unless overwrite approves the vault names it would replace, if any.
+func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool, skip []os.FileInfo, overwrite func([]string) bool) error {
 	info, err := os.Stat(src)
 	if err != nil {
 		return err
 	}
 	base := filepath.Base(filepath.Clean(src))
-	if !info.IsDir() {
-		return addOne(s, cat, src, path.Join(dest, base))
+	if base == "." || base == ".." || base == string(filepath.Separator) {
+		base = ""
 	}
-	return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
-		if err != nil || d.IsDir() {
+	skipped := func(fi os.FileInfo) bool {
+		for _, s := range skip {
+			if os.SameFile(fi, s) {
+				return true
+			}
+		}
+		return false
+	}
+	if skipped(info) {
+		return fmt.Errorf("%s is the vault or its key", src)
+	}
+	type item struct{ src, name string }
+	var items []item
+	var dirs []string
+	if !info.IsDir() {
+		items = append(items, item{src, path.Join(dest, filepath.Base(src))})
+	} else {
+		err = filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
+			if err != nil {
 				return err
 			}
+			if fi, err := d.Info(); err == nil && skipped(fi) {
+				fmt.Fprintf(os.Stderr, "skipping %s: the vault or its key\n", p)
+				if d.IsDir() {
+					return filepath.SkipDir
+				}
+				return nil
+			}
+			if d.IsDir() {
+				dirs = append(dirs, p)
+				return nil
+			}
 			if !d.Type().IsRegular() {
 				fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p)
 				return nil
@@ -250,11 +308,95 @@ func addPath(s *store, cat map[string][]*record, src, dest string) error {
 			if err != nil {
 				return err
 			}
-		return addOne(s, cat, p, path.Join(dest, base, filepath.ToSlash(rel)))
+			items = append(items, item{p, path.Join(dest, base, filepath.ToSlash(rel))})
+			return nil
 		})
+		if err != nil {
+			return err
+		}
+	}
+	if migrate {
+		var existing []string
+		for _, it := range items {
+			if len(cat[it.name]) > 0 {
+				existing = append(existing, it.name)
+			}
+		}
+		if len(existing) > 0 && !overwrite(existing) {
+			return errors.New("migrate cancelled; nothing changed")
+		}
+	}
+	for _, it := range items {
+		if err := addOne(s, cat, it.src, it.name, migrate); err != nil {
+			return err
+		}
+	}
+	if migrate {
+		// Children come after their parents in walk order, so go backwards.
+		// Directories still holding skipped files stay.
+		for i := len(dirs) - 1; i >= 0; i-- {
+			os.Remove(dirs[i])
+		}
+	}
+	return nil
 }
 
-func addOne(s *store, cat map[string][]*record, src, name string) error {
+// confirmOverwrite lists the vault names a migrate would replace and asks
+// on the terminal whether to go ahead. No terminal means no.
+func confirmOverwrite(names []string) bool {
+	fmt.Fprintln(os.Stderr, "these files are already in the vault and would be overwritten:")
+	for _, name := range names {
+		fmt.Fprintf(os.Stderr, "    %s\n", name)
+	}
+	ok, err := term.Confirm(fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names)))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err)
+	}
+	return ok
+}
+
+func isURL(s string) bool {
+	return strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://")
+}
+
+const chromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
+
+// addURL downloads rawURL to a private temporary file and adds it beneath
+// dest, named after the last element of the final (post-redirect) URL path.
+func addURL(s *store, cat map[string][]*record, rawURL, dest string) error {
+	req, err := http.NewRequest("GET", rawURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", chromeUA)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s", rawURL, resp.Status)
+	}
+	name := path.Base(resp.Request.URL.Path)
+	if name == "/" || name == "." {
+		name = resp.Request.URL.Hostname()
+	}
+	tmp, err := os.CreateTemp("", "vault-download-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	defer tmp.Close()
+	if _, err := io.Copy(tmp, resp.Body); err != nil {
+		return fmt.Errorf("%s: %w", rawURL, err)
+	}
+	if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
+		os.Chtimes(tmp.Name(), t, t)
+	}
+	return addOne(s, cat, tmp.Name(), path.Join(dest, name), false)
+}
+
+func addOne(s *store, cat map[string][]*record, src, name string, migrate bool) error {
 	replaced, err := s.add(cat, src, name)
 	if err != nil {
 		return err
@@ -263,6 +405,12 @@ func addOne(s *store, cat map[string][]*record, src, name string) error {
 	if replaced {
 		verb = "replaced"
 	}
+	if migrate {
+		if err := os.Remove(src); err != nil {
+			return err
+		}
+		verb = "migrated"
+	}
 	fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
 	return nil
 }

vault_test.go

@@ -9,9 +9,12 @@ import (
 	"crypto/rand"
 	"io"
 	"io/fs"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 )
@@ -295,3 +298,78 @@ func TestFiller(t *testing.T) {
 		t.Fatalf("filler objects total %d, largest %d", total, largest)
 	}
 }
+
+// TestAddTree migrates "." holding the vault and key: structure is kept,
+// originals and emptied directories go, the vault and key are left alone.
+// A URL download is then added under a folder.
+func TestAddTree(t *testing.T) {
+	dir := t.TempDir()
+	keyPath := filepath.Join(dir, "key")
+	if err := writeKeyFile(keyPath, "pass", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
+		t.Fatal(err)
+	}
+	s := newTestStore(t, keyPath, "pass")
+	os.MkdirAll(filepath.Join(dir, "a", "b"), 0o755)
+	os.WriteFile(filepath.Join(dir, "x"), []byte("x"), 0o600)
+	os.WriteFile(filepath.Join(dir, "a", "b", "y"), []byte("y"), 0o600)
+	t.Chdir(dir)
+
+	var skip []os.FileInfo
+	for _, p := range []string{"store", "key"} {
+		fi, _ := os.Stat(p)
+		skip = append(skip, fi)
+	}
+	cat, _ := s.catalogue()
+	if err := addPath(s, cat, "x", "d", false, skip, nil); err != nil {
+		t.Fatal(err)
+	}
+	// Migrating over d/x must ask first, and a no must leave everything.
+	var asked []string
+	no := func(names []string) bool { asked = names; return false }
+	if err := addPath(s, cat, ".", "d", true, skip, no); err == nil || !slices.Equal(asked, []string{"d/x"}) {
+		t.Fatalf("migrate over d/x: err %v, asked %v", err, asked)
+	}
+	if _, err := os.Stat("x"); err != nil || len(cat) != 1 {
+		t.Fatalf("declined migrate changed things: %v, %d stored", err, len(cat))
+	}
+	if err := addPath(s, cat, ".", "d", true, skip, func([]string) bool { return true }); err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for name := range cat {
+		got = append(got, name)
+	}
+	slices.Sort(got)
+	if !slices.Equal(got, []string{"d/a/b/y", "d/x"}) {
+		t.Fatalf("stored %v", got)
+	}
+	var left []string
+	entries, _ := os.ReadDir(".")
+	for _, e := range entries {
+		left = append(left, e.Name())
+	}
+	if !slices.Equal(left, []string{"key", "store"}) {
+		t.Fatalf("left behind %v", left)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.UserAgent() != chromeUA {
+			http.Error(w, "bad agent", http.StatusForbidden)
+			return
+		}
+		io.WriteString(w, "body")
+	}))
+	defer srv.Close()
+	if err := addURL(s, cat, srv.URL+"/files/page.html", "web"); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.get("web/page.html", "out"); err != nil {
+		t.Fatal(err)
+	}
+	if b, _ := os.ReadFile("out"); string(b) != "body" {
+		t.Fatalf("downloaded %q", b)
+	}
+}