From eb138b55240cfbfb48646bb9c915c629d7868574 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Mon, 28 Sep 2026 15:59:09 +0100 Subject: [PATCH] Add URLs to add, and a migrate command that deletes originals add now downloads http(s) URLs (with a Chrome user agent) through a private temporary file, and treats ".", ".." and "/" as "this folder's contents" so the destination folder is no longer dropped for "..". migrate is add followed by deleting each original and any directories left empty. It asks on the terminal before overwriting anything already in the vault, refusing when there is no terminal, unless --yes is given. Adding or migrating a directory skips the vault and key file if they live inside it, so migrate can never ingest or delete them. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 3 +- docs/guide/vault-guide.tex | 25 ++++- internal/term/term.go | 19 ++++ main.go | 182 +++++++++++++++++++++++++++++++++---- vault_test.go | 78 ++++++++++++++++ 5 files changed, 287 insertions(+), 20 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 588a772..759f6d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,8 @@ This file provides guidance to Claude Code when working with this repository. vault is a post-quantum encrypted file vault: one `package main` at the repo root that builds a single `vault` binary. Module `github.com/andydixon/vault`. -- `main.go` - CLI: `genkey`, `init`, `add`, `list`, `get`, `mv`, `rm`. +- `main.go` - CLI: `genkey`, `init`, `add` (file, directory + or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`. - `keyfile.go` - key file with two passphrase slots (Argon2id + XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second slot is either a decoy identity or random bytes; slot order is random. diff --git a/docs/guide/vault-guide.tex b/docs/guide/vault-guide.tex index e0a238d..b8de223 100644 --- a/docs/guide/vault-guide.tex +++ b/docs/guide/vault-guide.tex @@ -522,7 +522,7 @@ for your passphrase. That's the whole daily workflow. \section{Putting things in: \texttt{add}} \begin{shell} -$ vault add FILE-OR-FOLDER [FOLDER-IN-VAULT] +$ vault add FILE-OR-FOLDER-OR-URL [FOLDER-IN-VAULT] \end{shell} Give it a file, and it goes in under its own name. Give it a second argument, @@ -540,6 +540,8 @@ Notice what happened there: \file{id/passport-scan.jpg}. \item Adding the folder \file{Documents/letters} kept its own name and structure: \file{letters/bank.txt}, \file{letters/landlord.txt}. + \item Adding \file{.} (or \file{..}) brings in that folder's + \emph{contents}, subfolders and all, without a folder name of its own. \end{itemize} \begin{headsup} @@ -553,9 +555,28 @@ Adding a name that already exists \hl{replaces} it, and vault tells you so. The old version is deleted \emph{after} the new one is safely stored, so a crash halfway through never leaves you with neither. +\subsection{Straight from the web} +Give \cmd{add} an \file{http://} or \file{https://} address and vault +downloads it (as Chrome, so fussy sites play along) and stores it under the +last part of the address, such as \file{report.pdf}. The download sits in a +private temporary file only until it's encrypted. + +\subsection{Moving in: \texttt{migrate}} +\cmd{migrate} works exactly like \cmd{add}, but deletes each original once +it's safely stored, then tidies away any folders left empty. It won't take a +URL; there's no original to delete. + +Because the originals go, \cmd{migrate} checks first: if anything would +\hl{replace} a file already in the vault, it lists those names and asks +before touching anything. Answer anything but \texttt{y} and nothing changes. +With no terminal to ask on, the answer is no. For scripts, \texttt{--yes} +answers yes up front. + \subsection{What gets skipped} When adding a folder, anything that isn't a plain file, such as symbolic -links, sockets or devices, is skipped with a note. Files larger than 1\,TiB are +links, sockets or devices, is skipped with a note. So are the vault and key +themselves, should they live inside the folder, so \cmd{migrate} can never +eat them. Files larger than 1\,TiB are refused. (If you have a single 1\,TiB file, we have questions.) \section{Seeing what's inside: \texttt{list}} diff --git a/internal/term/term.go b/internal/term/term.go index e45779b..2f833a0 100644 --- a/internal/term/term.go +++ b/internal/term/term.go @@ -1,9 +1,11 @@ package term import ( + "bufio" "fmt" "os" "runtime" + "strings" "golang.org/x/term" ) @@ -72,6 +74,23 @@ func ReadSecret(prompt string) (s []byte, err error) { return } +// Confirm asks a yes/no question on the terminal. Only "y" or "yes" is yes. +func Confirm(prompt string) (yes bool, err error) { + err = WithTerminal(func(in, out *os.File) error { + fmt.Fprintf(out, "%s ", prompt) + line, err := bufio.NewReader(in).ReadString('\n') + if err != nil && line == "" { + return err + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + yes = true + } + return nil + }) + return +} + // ReadPassphrase returns the passphrase from the environment variable env if // it is set, otherwise prompts on the terminal. func ReadPassphrase(prompt, env string) (string, error) { diff --git a/main.go b/main.go index e359c2d..8031293 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,12 @@ package main import ( + "errors" "flag" "fmt" "io" "io/fs" + "net/http" "os" "path" "path/filepath" @@ -23,8 +25,13 @@ const usage = `Usage: vault genkey create a new passphrase-protected key vault init [--filler SIZE] create a new, empty vault, with at least SIZE of filler (e.g. 500M, 2G) - vault add FILE|DIR [FOLDER] encrypt a file or directory into the vault, + vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively) + or a downloaded URL into the vault, optionally under a folder inside the vault + vault migrate FILE|DIR [FOLDER] like add, then delete each original once + [--yes] it is stored; asks before overwriting + anything already in the vault, unless + --yes is given vault list [PREFIX] list stored files vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name) vault mv OLD NEW rename a stored file @@ -65,6 +72,7 @@ func main() { vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "") output := fset.String("o", "", "") filler := fset.String("filler", "0", "") + yes := fset.Bool("yes", false, "") pos, err := parseInterleaved(fset, args) if err != nil { fail("%v\n\n%s", err, usage) @@ -94,7 +102,7 @@ func main() { check(err) check(initStore(*vaultDir, size)) fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir) - case "add": + case "add", "migrate": want(1, 2) dest := "" if len(pos) == 2 { @@ -103,7 +111,25 @@ func main() { s := open(*keyPath, *vaultDir, true) cat, err := s.catalogue() check(err) - check(addPath(s, cat, pos[0], dest)) + if isURL(pos[0]) { + if cmd == "migrate" { + fail("cannot migrate a URL; use add") + } + check(addURL(s, cat, pos[0], dest)) + break + } + // Never ingest, or delete, the vault or its key. + var skip []os.FileInfo + for _, p := range []string{*vaultDir, *keyPath} { + if fi, err := os.Stat(p); err == nil { + skip = append(skip, fi) + } + } + overwrite := confirmOverwrite + if *yes { + overwrite = func([]string) bool { return true } + } + check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite)) case "list", "ls": want(0, 1) prefix := "" @@ -228,33 +254,149 @@ func decoyPassphrase() string { return string(p) } -// addPath adds a file, or every regular file under a directory, beneath dest. -func addPath(s *store, cat map[string][]*record, src, dest string) error { +// addPath adds a file, or every regular file under a directory, beneath dest, +// keeping the directory structure. A directory named "." or ".." adds its +// contents directly beneath dest. With migrate, each original is deleted +// once stored, then any directories left empty, and nothing is stored +// unless overwrite approves the vault names it would replace, if any. +func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool, skip []os.FileInfo, overwrite func([]string) bool) error { info, err := os.Stat(src) if err != nil { return err } base := filepath.Base(filepath.Clean(src)) - if !info.IsDir() { - return addOne(s, cat, src, path.Join(dest, base)) + if base == "." || base == ".." || base == string(filepath.Separator) { + base = "" } - return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err + skipped := func(fi os.FileInfo) bool { + for _, s := range skip { + if os.SameFile(fi, s) { + return true + } } - if !d.Type().IsRegular() { - fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p) + return false + } + if skipped(info) { + return fmt.Errorf("%s is the vault or its key", src) + } + type item struct{ src, name string } + var items []item + var dirs []string + if !info.IsDir() { + items = append(items, item{src, path.Join(dest, filepath.Base(src))}) + } else { + err = filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if fi, err := d.Info(); err == nil && skipped(fi) { + fmt.Fprintf(os.Stderr, "skipping %s: the vault or its key\n", p) + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + dirs = append(dirs, p) + return nil + } + if !d.Type().IsRegular() { + fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p) + return nil + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + items = append(items, item{p, path.Join(dest, base, filepath.ToSlash(rel))}) return nil - } - rel, err := filepath.Rel(src, p) + }) if err != nil { return err } - return addOne(s, cat, p, path.Join(dest, base, filepath.ToSlash(rel))) - }) + } + if migrate { + var existing []string + for _, it := range items { + if len(cat[it.name]) > 0 { + existing = append(existing, it.name) + } + } + if len(existing) > 0 && !overwrite(existing) { + return errors.New("migrate cancelled; nothing changed") + } + } + for _, it := range items { + if err := addOne(s, cat, it.src, it.name, migrate); err != nil { + return err + } + } + if migrate { + // Children come after their parents in walk order, so go backwards. + // Directories still holding skipped files stay. + for i := len(dirs) - 1; i >= 0; i-- { + os.Remove(dirs[i]) + } + } + return nil } -func addOne(s *store, cat map[string][]*record, src, name string) error { +// confirmOverwrite lists the vault names a migrate would replace and asks +// on the terminal whether to go ahead. No terminal means no. +func confirmOverwrite(names []string) bool { + fmt.Fprintln(os.Stderr, "these files are already in the vault and would be overwritten:") + for _, name := range names { + fmt.Fprintf(os.Stderr, " %s\n", name) + } + ok, err := term.Confirm(fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names))) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err) + } + return ok +} + +func isURL(s string) bool { + return strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://") +} + +const chromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" + +// addURL downloads rawURL to a private temporary file and adds it beneath +// dest, named after the last element of the final (post-redirect) URL path. +func addURL(s *store, cat map[string][]*record, rawURL, dest string) error { + req, err := http.NewRequest("GET", rawURL, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", chromeUA) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: %s", rawURL, resp.Status) + } + name := path.Base(resp.Request.URL.Path) + if name == "/" || name == "." { + name = resp.Request.URL.Hostname() + } + tmp, err := os.CreateTemp("", "vault-download-") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, resp.Body); err != nil { + return fmt.Errorf("%s: %w", rawURL, err) + } + if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil { + os.Chtimes(tmp.Name(), t, t) + } + return addOne(s, cat, tmp.Name(), path.Join(dest, name), false) +} + +func addOne(s *store, cat map[string][]*record, src, name string, migrate bool) error { replaced, err := s.add(cat, src, name) if err != nil { return err @@ -263,6 +405,12 @@ func addOne(s *store, cat map[string][]*record, src, name string) error { if replaced { verb = "replaced" } + if migrate { + if err := os.Remove(src); err != nil { + return err + } + verb = "migrated" + } fmt.Fprintf(os.Stderr, "%s %s\n", verb, name) return nil } diff --git a/vault_test.go b/vault_test.go index 036ae18..4befb66 100644 --- a/vault_test.go +++ b/vault_test.go @@ -9,9 +9,12 @@ import ( "crypto/rand" "io" "io/fs" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" + "slices" "testing" "time" ) @@ -295,3 +298,78 @@ func TestFiller(t *testing.T) { t.Fatalf("filler objects total %d, largest %d", total, largest) } } + +// TestAddTree migrates "." holding the vault and key: structure is kept, +// originals and emptied directories go, the vault and key are left alone. +// A URL download is then added under a folder. +func TestAddTree(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "key") + if err := writeKeyFile(keyPath, "pass", ""); err != nil { + t.Fatal(err) + } + if err := initStore(filepath.Join(dir, "store"), 0); err != nil { + t.Fatal(err) + } + s := newTestStore(t, keyPath, "pass") + os.MkdirAll(filepath.Join(dir, "a", "b"), 0o755) + os.WriteFile(filepath.Join(dir, "x"), []byte("x"), 0o600) + os.WriteFile(filepath.Join(dir, "a", "b", "y"), []byte("y"), 0o600) + t.Chdir(dir) + + var skip []os.FileInfo + for _, p := range []string{"store", "key"} { + fi, _ := os.Stat(p) + skip = append(skip, fi) + } + cat, _ := s.catalogue() + if err := addPath(s, cat, "x", "d", false, skip, nil); err != nil { + t.Fatal(err) + } + // Migrating over d/x must ask first, and a no must leave everything. + var asked []string + no := func(names []string) bool { asked = names; return false } + if err := addPath(s, cat, ".", "d", true, skip, no); err == nil || !slices.Equal(asked, []string{"d/x"}) { + t.Fatalf("migrate over d/x: err %v, asked %v", err, asked) + } + if _, err := os.Stat("x"); err != nil || len(cat) != 1 { + t.Fatalf("declined migrate changed things: %v, %d stored", err, len(cat)) + } + if err := addPath(s, cat, ".", "d", true, skip, func([]string) bool { return true }); err != nil { + t.Fatal(err) + } + var got []string + for name := range cat { + got = append(got, name) + } + slices.Sort(got) + if !slices.Equal(got, []string{"d/a/b/y", "d/x"}) { + t.Fatalf("stored %v", got) + } + var left []string + entries, _ := os.ReadDir(".") + for _, e := range entries { + left = append(left, e.Name()) + } + if !slices.Equal(left, []string{"key", "store"}) { + t.Fatalf("left behind %v", left) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() != chromeUA { + http.Error(w, "bad agent", http.StatusForbidden) + return + } + io.WriteString(w, "body") + })) + defer srv.Close() + if err := addURL(s, cat, srv.URL+"/files/page.html", "web"); err != nil { + t.Fatal(err) + } + if err := s.get("web/page.html", "out"); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile("out"); string(b) != "body" { + t.Fatalf("downloaded %q", b) + } +}