Add a mirror command that crawls a site into the vault

Single-threaded, same host only, Chrome user agent. Links in HTML and
CSS are rewritten to relative paths. Includes a man page and a guide note.

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Andy Dixon
2026-09-29 15:02:12 +01:00
co-authored by Claude Sonnet 5.5
parent e40a803367
commit 10f2a88493
5 changed files with 315 additions and 0 deletions
+16
View File
@@ -32,6 +32,9 @@ const usage = `Usage:
[--yes] it is stored; asks before overwriting
anything already in the vault, unless
--yes is given
vault mirror URL [FOLDER] crawl a website (same host only, one request
at a time) into FOLDER/HOST, rewriting its
links to relative paths
vault list [PREFIX] list stored files
vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name)
vault mv OLD NEW rename a stored file
@@ -130,6 +133,19 @@ func main() {
overwrite = func([]string) bool { return true }
}
check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
case "mirror":
want(1, 2)
if !isURL(pos[0]) {
fail("mirror needs an http:// or https:// URL")
}
dest := ""
if len(pos) == 2 {
dest = strings.Trim(pos[1], "/")
}
s := open(*keyPath, *vaultDir, true)
cat, err := s.catalogue()
check(err)
check(mirror(s, cat, pos[0], dest))
case "list", "ls":
want(0, 1)
prefix := ""