Add URLs to add, and a migrate command that deletes originals

add now downloads http(s) URLs (with a Chrome user agent) through a
private temporary file, and treats ".", ".." and "/" as "this folder's
contents" so the destination folder is no longer dropped for "..".

migrate is add followed by deleting each original and any directories
left empty. It asks on the terminal before overwriting anything already
in the vault, refusing when there is no terminal, unless --yes is given.

Adding or migrating a directory skips the vault and key file if they
live inside it, so migrate can never ingest or delete them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CLAUDE.md

@@ -7,7 +7,8 @@ This file provides guidance to Claude Code when working with this repository.
 vault is a post-quantum encrypted file vault: one `package main` at the repo
 root that builds a single `vault` binary. Module `github.com/andydixon/vault`.
 
-- `main.go` - CLI: `genkey`, `init`, `add`, `list`, `get`, `mv`, `rm`.
+- `main.go` - CLI: `genkey`, `init`, `add` (file, directory
+  or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`.
 - `keyfile.go` - key file with two passphrase slots (Argon2id +
   XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second
   slot is either a decoy identity or random bytes; slot order is random.

docs/guide/vault-guide.tex

@@ -522,7 +522,7 @@ for your passphrase. That's the whole daily workflow.
 \section{Putting things in: \texttt{add}}
 
 \begin{shell}
-$ vault add FILE-OR-FOLDER [FOLDER-IN-VAULT]
+$ vault add FILE-OR-FOLDER-OR-URL [FOLDER-IN-VAULT]
 \end{shell}
 
 Give it a file, and it goes in under its own name. Give it a second argument,
@@ -540,6 +540,8 @@ Notice what happened there:
         \file{id/passport-scan.jpg}.
   \item Adding the folder \file{Documents/letters} kept its own name and
         structure: \file{letters/bank.txt}, \file{letters/landlord.txt}.
+  \item Adding \file{.} (or \file{..}) brings in that folder's
+        \emph{contents}, subfolders and all, without a folder name of its own.
 \end{itemize}
 
 \begin{headsup}
@@ -553,9 +555,28 @@ Adding a name that already exists \hl{replaces} it, and vault tells you so.
 The old version is deleted \emph{after} the new one is safely stored, so a
 crash halfway through never leaves you with neither.
 
+\subsection{Straight from the web}
+Give \cmd{add} an \file{http://} or \file{https://} address and vault
+downloads it (as Chrome, so fussy sites play along) and stores it under the
+last part of the address, such as \file{report.pdf}. The download sits in a
+private temporary file only until it's encrypted.
+
+\subsection{Moving in: \texttt{migrate}}
+\cmd{migrate} works exactly like \cmd{add}, but deletes each original once
+it's safely stored, then tidies away any folders left empty. It won't take a
+URL; there's no original to delete.
+
+Because the originals go, \cmd{migrate} checks first: if anything would
+\hl{replace} a file already in the vault, it lists those names and asks
+before touching anything. Answer anything but \texttt{y} and nothing changes.
+With no terminal to ask on, the answer is no. For scripts, \texttt{--yes}
+answers yes up front.
+
 \subsection{What gets skipped}
 When adding a folder, anything that isn't a plain file, such as symbolic
-links, sockets or devices, is skipped with a note. Files larger than 1\,TiB are
+links, sockets or devices, is skipped with a note. So are the vault and key
+themselves, should they live inside the folder, so \cmd{migrate} can never
+eat them. Files larger than 1\,TiB are
 refused. (If you have a single 1\,TiB file, we have questions.)
 
 \section{Seeing what's inside: \texttt{list}}

internal/term/term.go

@@ -1,9 +1,11 @@
 package term
 
 import (
+	"bufio"
 	"fmt"
 	"os"
 	"runtime"
+	"strings"
 
 	"golang.org/x/term"
 )
@@ -72,6 +74,23 @@ func ReadSecret(prompt string) (s []byte, err error) {
 	return
 }
 
+// Confirm asks a yes/no question on the terminal. Only "y" or "yes" is yes.
+func Confirm(prompt string) (yes bool, err error) {
+	err = WithTerminal(func(in, out *os.File) error {
+		fmt.Fprintf(out, "%s ", prompt)
+		line, err := bufio.NewReader(in).ReadString('\n')
+		if err != nil && line == "" {
+			return err
+		}
+		switch strings.ToLower(strings.TrimSpace(line)) {
+		case "y", "yes":
+			yes = true
+		}
+		return nil
+	})
+	return
+}
+
 // ReadPassphrase returns the passphrase from the environment variable env if
 // it is set, otherwise prompts on the terminal.
 func ReadPassphrase(prompt, env string) (string, error) {

main.go

@@ -5,10 +5,12 @@
 package main
 
 import (
+	"errors"
 	"flag"
 	"fmt"
 	"io"
 	"io/fs"
+	"net/http"
 	"os"
 	"path"
 	"path/filepath"
@@ -23,8 +25,13 @@ const usage = `Usage:
     vault genkey                    create a new passphrase-protected key
     vault init [--filler SIZE]      create a new, empty vault, with at least
                                     SIZE of filler (e.g. 500M, 2G)
-    vault add FILE|DIR [FOLDER]     encrypt a file or directory into the vault,
+    vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively)
+                                    or a downloaded URL into the vault,
                                     optionally under a folder inside the vault
+    vault migrate FILE|DIR [FOLDER] like add, then delete each original once
+    [--yes]                         it is stored; asks before overwriting
+                                    anything already in the vault, unless
+                                    --yes is given
     vault list [PREFIX]             list stored files
     vault get NAME [-o OUTPUT]      decrypt a stored file (default: its base name)
     vault mv OLD NEW                rename a stored file
@@ -65,6 +72,7 @@ func main() {
 	vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "")
 	output := fset.String("o", "", "")
 	filler := fset.String("filler", "0", "")
+	yes := fset.Bool("yes", false, "")
 	pos, err := parseInterleaved(fset, args)
 	if err != nil {
 		fail("%v\n\n%s", err, usage)
@@ -94,7 +102,7 @@ func main() {
 		check(err)
 		check(initStore(*vaultDir, size))
 		fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir)
-	case "add":
+	case "add", "migrate":
 		want(1, 2)
 		dest := ""
 		if len(pos) == 2 {
@@ -103,7 +111,25 @@ func main() {
 		s := open(*keyPath, *vaultDir, true)
 		cat, err := s.catalogue()
 		check(err)
-		check(addPath(s, cat, pos[0], dest))
+		if isURL(pos[0]) {
+			if cmd == "migrate" {
+				fail("cannot migrate a URL; use add")
+			}
+			check(addURL(s, cat, pos[0], dest))
+			break
+		}
+		// Never ingest, or delete, the vault or its key.
+		var skip []os.FileInfo
+		for _, p := range []string{*vaultDir, *keyPath} {
+			if fi, err := os.Stat(p); err == nil {
+				skip = append(skip, fi)
+			}
+		}
+		overwrite := confirmOverwrite
+		if *yes {
+			overwrite = func([]string) bool { return true }
+		}
+		check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
 	case "list", "ls":
 		want(0, 1)
 		prefix := ""
@@ -228,33 +254,149 @@ func decoyPassphrase() string {
 	return string(p)
 }
 
-// addPath adds a file, or every regular file under a directory, beneath dest.
-func addPath(s *store, cat map[string][]*record, src, dest string) error {
+// addPath adds a file, or every regular file under a directory, beneath dest,
+// keeping the directory structure. A directory named "." or ".." adds its
+// contents directly beneath dest. With migrate, each original is deleted
+// once stored, then any directories left empty, and nothing is stored
+// unless overwrite approves the vault names it would replace, if any.
+func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool, skip []os.FileInfo, overwrite func([]string) bool) error {
 	info, err := os.Stat(src)
 	if err != nil {
 		return err
 	}
 	base := filepath.Base(filepath.Clean(src))
-	if !info.IsDir() {
-		return addOne(s, cat, src, path.Join(dest, base))
+	if base == "." || base == ".." || base == string(filepath.Separator) {
+		base = ""
 	}
-	return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
-		if err != nil || d.IsDir() {
-			return err
+	skipped := func(fi os.FileInfo) bool {
+		for _, s := range skip {
+			if os.SameFile(fi, s) {
+				return true
+			}
 		}
-		if !d.Type().IsRegular() {
-			fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p)
+		return false
+	}
+	if skipped(info) {
+		return fmt.Errorf("%s is the vault or its key", src)
+	}
+	type item struct{ src, name string }
+	var items []item
+	var dirs []string
+	if !info.IsDir() {
+		items = append(items, item{src, path.Join(dest, filepath.Base(src))})
+	} else {
+		err = filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			if fi, err := d.Info(); err == nil && skipped(fi) {
+				fmt.Fprintf(os.Stderr, "skipping %s: the vault or its key\n", p)
+				if d.IsDir() {
+					return filepath.SkipDir
+				}
+				return nil
+			}
+			if d.IsDir() {
+				dirs = append(dirs, p)
+				return nil
+			}
+			if !d.Type().IsRegular() {
+				fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p)
+				return nil
+			}
+			rel, err := filepath.Rel(src, p)
+			if err != nil {
+				return err
+			}
+			items = append(items, item{p, path.Join(dest, base, filepath.ToSlash(rel))})
 			return nil
-		}
-		rel, err := filepath.Rel(src, p)
+		})
 		if err != nil {
 			return err
 		}
-		return addOne(s, cat, p, path.Join(dest, base, filepath.ToSlash(rel)))
-	})
+	}
+	if migrate {
+		var existing []string
+		for _, it := range items {
+			if len(cat[it.name]) > 0 {
+				existing = append(existing, it.name)
+			}
+		}
+		if len(existing) > 0 && !overwrite(existing) {
+			return errors.New("migrate cancelled; nothing changed")
+		}
+	}
+	for _, it := range items {
+		if err := addOne(s, cat, it.src, it.name, migrate); err != nil {
+			return err
+		}
+	}
+	if migrate {
+		// Children come after their parents in walk order, so go backwards.
+		// Directories still holding skipped files stay.
+		for i := len(dirs) - 1; i >= 0; i-- {
+			os.Remove(dirs[i])
+		}
+	}
+	return nil
 }
 
-func addOne(s *store, cat map[string][]*record, src, name string) error {
+// confirmOverwrite lists the vault names a migrate would replace and asks
+// on the terminal whether to go ahead. No terminal means no.
+func confirmOverwrite(names []string) bool {
+	fmt.Fprintln(os.Stderr, "these files are already in the vault and would be overwritten:")
+	for _, name := range names {
+		fmt.Fprintf(os.Stderr, "    %s\n", name)
+	}
+	ok, err := term.Confirm(fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names)))
+	if err != nil {
+		fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err)
+	}
+	return ok
+}
+
+func isURL(s string) bool {
+	return strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://")
+}
+
+const chromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"
+
+// addURL downloads rawURL to a private temporary file and adds it beneath
+// dest, named after the last element of the final (post-redirect) URL path.
+func addURL(s *store, cat map[string][]*record, rawURL, dest string) error {
+	req, err := http.NewRequest("GET", rawURL, nil)
+	if err != nil {
+		return err
+	}
+	req.Header.Set("User-Agent", chromeUA)
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("%s: %s", rawURL, resp.Status)
+	}
+	name := path.Base(resp.Request.URL.Path)
+	if name == "/" || name == "." {
+		name = resp.Request.URL.Hostname()
+	}
+	tmp, err := os.CreateTemp("", "vault-download-")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name())
+	defer tmp.Close()
+	if _, err := io.Copy(tmp, resp.Body); err != nil {
+		return fmt.Errorf("%s: %w", rawURL, err)
+	}
+	if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
+		os.Chtimes(tmp.Name(), t, t)
+	}
+	return addOne(s, cat, tmp.Name(), path.Join(dest, name), false)
+}
+
+func addOne(s *store, cat map[string][]*record, src, name string, migrate bool) error {
 	replaced, err := s.add(cat, src, name)
 	if err != nil {
 		return err
@@ -263,6 +405,12 @@ func addOne(s *store, cat map[string][]*record, src, name string) error {
 	if replaced {
 		verb = "replaced"
 	}
+	if migrate {
+		if err := os.Remove(src); err != nil {
+			return err
+		}
+		verb = "migrated"
+	}
 	fmt.Fprintf(os.Stderr, "%s %s\n", verb, name)
 	return nil
 }

vault_test.go

@@ -9,9 +9,12 @@ import (
 	"crypto/rand"
 	"io"
 	"io/fs"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"slices"
 	"testing"
 	"time"
 )
@@ -295,3 +298,78 @@ func TestFiller(t *testing.T) {
 		t.Fatalf("filler objects total %d, largest %d", total, largest)
 	}
 }
+
+// TestAddTree migrates "." holding the vault and key: structure is kept,
+// originals and emptied directories go, the vault and key are left alone.
+// A URL download is then added under a folder.
+func TestAddTree(t *testing.T) {
+	dir := t.TempDir()
+	keyPath := filepath.Join(dir, "key")
+	if err := writeKeyFile(keyPath, "pass", ""); err != nil {
+		t.Fatal(err)
+	}
+	if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
+		t.Fatal(err)
+	}
+	s := newTestStore(t, keyPath, "pass")
+	os.MkdirAll(filepath.Join(dir, "a", "b"), 0o755)
+	os.WriteFile(filepath.Join(dir, "x"), []byte("x"), 0o600)
+	os.WriteFile(filepath.Join(dir, "a", "b", "y"), []byte("y"), 0o600)
+	t.Chdir(dir)
+
+	var skip []os.FileInfo
+	for _, p := range []string{"store", "key"} {
+		fi, _ := os.Stat(p)
+		skip = append(skip, fi)
+	}
+	cat, _ := s.catalogue()
+	if err := addPath(s, cat, "x", "d", false, skip, nil); err != nil {
+		t.Fatal(err)
+	}
+	// Migrating over d/x must ask first, and a no must leave everything.
+	var asked []string
+	no := func(names []string) bool { asked = names; return false }
+	if err := addPath(s, cat, ".", "d", true, skip, no); err == nil || !slices.Equal(asked, []string{"d/x"}) {
+		t.Fatalf("migrate over d/x: err %v, asked %v", err, asked)
+	}
+	if _, err := os.Stat("x"); err != nil || len(cat) != 1 {
+		t.Fatalf("declined migrate changed things: %v, %d stored", err, len(cat))
+	}
+	if err := addPath(s, cat, ".", "d", true, skip, func([]string) bool { return true }); err != nil {
+		t.Fatal(err)
+	}
+	var got []string
+	for name := range cat {
+		got = append(got, name)
+	}
+	slices.Sort(got)
+	if !slices.Equal(got, []string{"d/a/b/y", "d/x"}) {
+		t.Fatalf("stored %v", got)
+	}
+	var left []string
+	entries, _ := os.ReadDir(".")
+	for _, e := range entries {
+		left = append(left, e.Name())
+	}
+	if !slices.Equal(left, []string{"key", "store"}) {
+		t.Fatalf("left behind %v", left)
+	}
+
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.UserAgent() != chromeUA {
+			http.Error(w, "bad agent", http.StatusForbidden)
+			return
+		}
+		io.WriteString(w, "body")
+	}))
+	defer srv.Close()
+	if err := addURL(s, cat, srv.URL+"/files/page.html", "web"); err != nil {
+		t.Fatal(err)
+	}
+	if err := s.get("web/page.html", "out"); err != nil {
+		t.Fatal(err)
+	}
+	if b, _ := os.ReadFile("out"); string(b) != "body" {
+		t.Fatalf("downloaded %q", b)
+	}
+}