diff --git a/CLAUDE.md b/CLAUDE.md index 588a772..759f6d1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,8 @@ This file provides guidance to Claude Code when working with this repository. vault is a post-quantum encrypted file vault: one `package main` at the repo root that builds a single `vault` binary. Module `github.com/andydixon/vault`. -- `main.go` - CLI: `genkey`, `init`, `add`, `list`, `get`, `mv`, `rm`. +- `main.go` - CLI: `genkey`, `init`, `add` (file, directory + or URL), `migrate` (add, then delete originals), `list`, `get`, `mv`, `rm`. - `keyfile.go` - key file with two passphrase slots (Argon2id + XChaCha20-Poly1305 sealing a suite ID and 32-byte master seed). The second slot is either a decoy identity or random bytes; slot order is random. diff --git a/docs/guide/vault-guide.tex b/docs/guide/vault-guide.tex index e0a238d..b8de223 100644 --- a/docs/guide/vault-guide.tex +++ b/docs/guide/vault-guide.tex @@ -522,7 +522,7 @@ for your passphrase. That's the whole daily workflow. \section{Putting things in: \texttt{add}} \begin{shell} -$ vault add FILE-OR-FOLDER [FOLDER-IN-VAULT] +$ vault add FILE-OR-FOLDER-OR-URL [FOLDER-IN-VAULT] \end{shell} Give it a file, and it goes in under its own name. Give it a second argument, @@ -540,6 +540,8 @@ Notice what happened there: \file{id/passport-scan.jpg}. \item Adding the folder \file{Documents/letters} kept its own name and structure: \file{letters/bank.txt}, \file{letters/landlord.txt}. + \item Adding \file{.} (or \file{..}) brings in that folder's + \emph{contents}, subfolders and all, without a folder name of its own. \end{itemize} \begin{headsup} @@ -553,9 +555,28 @@ Adding a name that already exists \hl{replaces} it, and vault tells you so. The old version is deleted \emph{after} the new one is safely stored, so a crash halfway through never leaves you with neither. +\subsection{Straight from the web} +Give \cmd{add} an \file{http://} or \file{https://} address and vault +downloads it (as Chrome, so fussy sites play along) and stores it under the +last part of the address, such as \file{report.pdf}. The download sits in a +private temporary file only until it's encrypted. + +\subsection{Moving in: \texttt{migrate}} +\cmd{migrate} works exactly like \cmd{add}, but deletes each original once +it's safely stored, then tidies away any folders left empty. It won't take a +URL; there's no original to delete. + +Because the originals go, \cmd{migrate} checks first: if anything would +\hl{replace} a file already in the vault, it lists those names and asks +before touching anything. Answer anything but \texttt{y} and nothing changes. +With no terminal to ask on, the answer is no. For scripts, \texttt{--yes} +answers yes up front. + \subsection{What gets skipped} When adding a folder, anything that isn't a plain file, such as symbolic -links, sockets or devices, is skipped with a note. Files larger than 1\,TiB are +links, sockets or devices, is skipped with a note. So are the vault and key +themselves, should they live inside the folder, so \cmd{migrate} can never +eat them. Files larger than 1\,TiB are refused. (If you have a single 1\,TiB file, we have questions.) \section{Seeing what's inside: \texttt{list}} diff --git a/internal/term/term.go b/internal/term/term.go index e45779b..2f833a0 100644 --- a/internal/term/term.go +++ b/internal/term/term.go @@ -1,9 +1,11 @@ package term import ( + "bufio" "fmt" "os" "runtime" + "strings" "golang.org/x/term" ) @@ -72,6 +74,23 @@ func ReadSecret(prompt string) (s []byte, err error) { return } +// Confirm asks a yes/no question on the terminal. Only "y" or "yes" is yes. +func Confirm(prompt string) (yes bool, err error) { + err = WithTerminal(func(in, out *os.File) error { + fmt.Fprintf(out, "%s ", prompt) + line, err := bufio.NewReader(in).ReadString('\n') + if err != nil && line == "" { + return err + } + switch strings.ToLower(strings.TrimSpace(line)) { + case "y", "yes": + yes = true + } + return nil + }) + return +} + // ReadPassphrase returns the passphrase from the environment variable env if // it is set, otherwise prompts on the terminal. func ReadPassphrase(prompt, env string) (string, error) { diff --git a/main.go b/main.go index e359c2d..8031293 100644 --- a/main.go +++ b/main.go @@ -5,10 +5,12 @@ package main import ( + "errors" "flag" "fmt" "io" "io/fs" + "net/http" "os" "path" "path/filepath" @@ -23,8 +25,13 @@ const usage = `Usage: vault genkey create a new passphrase-protected key vault init [--filler SIZE] create a new, empty vault, with at least SIZE of filler (e.g. 500M, 2G) - vault add FILE|DIR [FOLDER] encrypt a file or directory into the vault, + vault add FILE|DIR|URL [FOLDER] encrypt a file, a directory (recursively) + or a downloaded URL into the vault, optionally under a folder inside the vault + vault migrate FILE|DIR [FOLDER] like add, then delete each original once + [--yes] it is stored; asks before overwriting + anything already in the vault, unless + --yes is given vault list [PREFIX] list stored files vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name) vault mv OLD NEW rename a stored file @@ -65,6 +72,7 @@ func main() { vaultDir := fset.String("vault", defaultPath("VAULT_DIR", "store"), "") output := fset.String("o", "", "") filler := fset.String("filler", "0", "") + yes := fset.Bool("yes", false, "") pos, err := parseInterleaved(fset, args) if err != nil { fail("%v\n\n%s", err, usage) @@ -94,7 +102,7 @@ func main() { check(err) check(initStore(*vaultDir, size)) fmt.Fprintf(os.Stderr, "vault created at %s\n", *vaultDir) - case "add": + case "add", "migrate": want(1, 2) dest := "" if len(pos) == 2 { @@ -103,7 +111,25 @@ func main() { s := open(*keyPath, *vaultDir, true) cat, err := s.catalogue() check(err) - check(addPath(s, cat, pos[0], dest)) + if isURL(pos[0]) { + if cmd == "migrate" { + fail("cannot migrate a URL; use add") + } + check(addURL(s, cat, pos[0], dest)) + break + } + // Never ingest, or delete, the vault or its key. + var skip []os.FileInfo + for _, p := range []string{*vaultDir, *keyPath} { + if fi, err := os.Stat(p); err == nil { + skip = append(skip, fi) + } + } + overwrite := confirmOverwrite + if *yes { + overwrite = func([]string) bool { return true } + } + check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite)) case "list", "ls": want(0, 1) prefix := "" @@ -228,33 +254,149 @@ func decoyPassphrase() string { return string(p) } -// addPath adds a file, or every regular file under a directory, beneath dest. -func addPath(s *store, cat map[string][]*record, src, dest string) error { +// addPath adds a file, or every regular file under a directory, beneath dest, +// keeping the directory structure. A directory named "." or ".." adds its +// contents directly beneath dest. With migrate, each original is deleted +// once stored, then any directories left empty, and nothing is stored +// unless overwrite approves the vault names it would replace, if any. +func addPath(s *store, cat map[string][]*record, src, dest string, migrate bool, skip []os.FileInfo, overwrite func([]string) bool) error { info, err := os.Stat(src) if err != nil { return err } base := filepath.Base(filepath.Clean(src)) - if !info.IsDir() { - return addOne(s, cat, src, path.Join(dest, base)) + if base == "." || base == ".." || base == string(filepath.Separator) { + base = "" } - return filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { - if err != nil || d.IsDir() { - return err + skipped := func(fi os.FileInfo) bool { + for _, s := range skip { + if os.SameFile(fi, s) { + return true + } } - if !d.Type().IsRegular() { - fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p) + return false + } + if skipped(info) { + return fmt.Errorf("%s is the vault or its key", src) + } + type item struct{ src, name string } + var items []item + var dirs []string + if !info.IsDir() { + items = append(items, item{src, path.Join(dest, filepath.Base(src))}) + } else { + err = filepath.WalkDir(src, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if fi, err := d.Info(); err == nil && skipped(fi) { + fmt.Fprintf(os.Stderr, "skipping %s: the vault or its key\n", p) + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() { + dirs = append(dirs, p) + return nil + } + if !d.Type().IsRegular() { + fmt.Fprintf(os.Stderr, "skipping %s: not a regular file\n", p) + return nil + } + rel, err := filepath.Rel(src, p) + if err != nil { + return err + } + items = append(items, item{p, path.Join(dest, base, filepath.ToSlash(rel))}) return nil - } - rel, err := filepath.Rel(src, p) + }) if err != nil { return err } - return addOne(s, cat, p, path.Join(dest, base, filepath.ToSlash(rel))) - }) + } + if migrate { + var existing []string + for _, it := range items { + if len(cat[it.name]) > 0 { + existing = append(existing, it.name) + } + } + if len(existing) > 0 && !overwrite(existing) { + return errors.New("migrate cancelled; nothing changed") + } + } + for _, it := range items { + if err := addOne(s, cat, it.src, it.name, migrate); err != nil { + return err + } + } + if migrate { + // Children come after their parents in walk order, so go backwards. + // Directories still holding skipped files stay. + for i := len(dirs) - 1; i >= 0; i-- { + os.Remove(dirs[i]) + } + } + return nil } -func addOne(s *store, cat map[string][]*record, src, name string) error { +// confirmOverwrite lists the vault names a migrate would replace and asks +// on the terminal whether to go ahead. No terminal means no. +func confirmOverwrite(names []string) bool { + fmt.Fprintln(os.Stderr, "these files are already in the vault and would be overwritten:") + for _, name := range names { + fmt.Fprintf(os.Stderr, " %s\n", name) + } + ok, err := term.Confirm(fmt.Sprintf("overwrite %d file(s)? [y/N]", len(names))) + if err != nil { + fmt.Fprintf(os.Stderr, "cannot ask for confirmation: %v\n", err) + } + return ok +} + +func isURL(s string) bool { + return strings.HasPrefix(s, "https://") || strings.HasPrefix(s, "http://") +} + +const chromeUA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36" + +// addURL downloads rawURL to a private temporary file and adds it beneath +// dest, named after the last element of the final (post-redirect) URL path. +func addURL(s *store, cat map[string][]*record, rawURL, dest string) error { + req, err := http.NewRequest("GET", rawURL, nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", chromeUA) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s: %s", rawURL, resp.Status) + } + name := path.Base(resp.Request.URL.Path) + if name == "/" || name == "." { + name = resp.Request.URL.Hostname() + } + tmp, err := os.CreateTemp("", "vault-download-") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := io.Copy(tmp, resp.Body); err != nil { + return fmt.Errorf("%s: %w", rawURL, err) + } + if t, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil { + os.Chtimes(tmp.Name(), t, t) + } + return addOne(s, cat, tmp.Name(), path.Join(dest, name), false) +} + +func addOne(s *store, cat map[string][]*record, src, name string, migrate bool) error { replaced, err := s.add(cat, src, name) if err != nil { return err @@ -263,6 +405,12 @@ func addOne(s *store, cat map[string][]*record, src, name string) error { if replaced { verb = "replaced" } + if migrate { + if err := os.Remove(src); err != nil { + return err + } + verb = "migrated" + } fmt.Fprintf(os.Stderr, "%s %s\n", verb, name) return nil } diff --git a/vault_test.go b/vault_test.go index 036ae18..4befb66 100644 --- a/vault_test.go +++ b/vault_test.go @@ -9,9 +9,12 @@ import ( "crypto/rand" "io" "io/fs" + "net/http" + "net/http/httptest" "os" "os/exec" "path/filepath" + "slices" "testing" "time" ) @@ -295,3 +298,78 @@ func TestFiller(t *testing.T) { t.Fatalf("filler objects total %d, largest %d", total, largest) } } + +// TestAddTree migrates "." holding the vault and key: structure is kept, +// originals and emptied directories go, the vault and key are left alone. +// A URL download is then added under a folder. +func TestAddTree(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "key") + if err := writeKeyFile(keyPath, "pass", ""); err != nil { + t.Fatal(err) + } + if err := initStore(filepath.Join(dir, "store"), 0); err != nil { + t.Fatal(err) + } + s := newTestStore(t, keyPath, "pass") + os.MkdirAll(filepath.Join(dir, "a", "b"), 0o755) + os.WriteFile(filepath.Join(dir, "x"), []byte("x"), 0o600) + os.WriteFile(filepath.Join(dir, "a", "b", "y"), []byte("y"), 0o600) + t.Chdir(dir) + + var skip []os.FileInfo + for _, p := range []string{"store", "key"} { + fi, _ := os.Stat(p) + skip = append(skip, fi) + } + cat, _ := s.catalogue() + if err := addPath(s, cat, "x", "d", false, skip, nil); err != nil { + t.Fatal(err) + } + // Migrating over d/x must ask first, and a no must leave everything. + var asked []string + no := func(names []string) bool { asked = names; return false } + if err := addPath(s, cat, ".", "d", true, skip, no); err == nil || !slices.Equal(asked, []string{"d/x"}) { + t.Fatalf("migrate over d/x: err %v, asked %v", err, asked) + } + if _, err := os.Stat("x"); err != nil || len(cat) != 1 { + t.Fatalf("declined migrate changed things: %v, %d stored", err, len(cat)) + } + if err := addPath(s, cat, ".", "d", true, skip, func([]string) bool { return true }); err != nil { + t.Fatal(err) + } + var got []string + for name := range cat { + got = append(got, name) + } + slices.Sort(got) + if !slices.Equal(got, []string{"d/a/b/y", "d/x"}) { + t.Fatalf("stored %v", got) + } + var left []string + entries, _ := os.ReadDir(".") + for _, e := range entries { + left = append(left, e.Name()) + } + if !slices.Equal(left, []string{"key", "store"}) { + t.Fatalf("left behind %v", left) + } + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() != chromeUA { + http.Error(w, "bad agent", http.StatusForbidden) + return + } + io.WriteString(w, "body") + })) + defer srv.Close() + if err := addURL(s, cat, srv.URL+"/files/page.html", "web"); err != nil { + t.Fatal(err) + } + if err := s.get("web/page.html", "out"); err != nil { + t.Fatal(err) + } + if b, _ := os.ReadFile("out"); string(b) != "body" { + t.Fatalf("downloaded %q", b) + } +}