From 10f2a884935a0da4ff0b0aa0b5d77a14924fad42 Mon Sep 17 00:00:00 2001 From: Andy Dixon Date: Tue, 29 Sep 2026 15:02:12 +0100 Subject: [PATCH] Add a mirror command that crawls a site into the vault Single-threaded, same host only, Chrome user agent. Links in HTML and CSS are rewritten to relative paths. Includes a man page and a guide note. Co-Authored-By: Claude Sonnet 5.5 --- docs/guide/vault-guide.tex | 6 ++ docs/vault-mirror.1 | 86 +++++++++++++++++++++ main.go | 16 ++++ mirror.go | 150 +++++++++++++++++++++++++++++++++++++ vault_test.go | 57 ++++++++++++++ 5 files changed, 315 insertions(+) create mode 100644 docs/vault-mirror.1 create mode 100644 mirror.go diff --git a/docs/guide/vault-guide.tex b/docs/guide/vault-guide.tex index dcef4be..011a192 100644 --- a/docs/guide/vault-guide.tex +++ b/docs/guide/vault-guide.tex @@ -561,6 +561,12 @@ downloads it (as Chrome, so fussy sites play along) and stores it under the last part of the address, such as \file{report.pdf}. The download sits in a private temporary file only until it's encrypted. +\subsection{Copying a website: \texttt{mirror}} +\cmd{mirror} takes an \file{http://} or \file{https://} address and copies +the whole site, one request at a time, into a folder named after its host. +Links inside pages and stylesheets are rewritten to relative paths, so the +copy still browses once you get it out. Only the same host is followed. + \subsection{Moving in: \texttt{migrate}} \cmd{migrate} works exactly like \cmd{add}, but deletes each original once it's safely stored, then tidies away any folders left empty. It won't take a diff --git a/docs/vault-mirror.1 b/docs/vault-mirror.1 new file mode 100644 index 0000000..5ea3bce --- /dev/null +++ b/docs/vault-mirror.1 @@ -0,0 +1,86 @@ +.TH VAULT-MIRROR 1 "September 2026" "vault" "User Commands" +.SH NAME +vault-mirror \- mirror a website into an encrypted vault +.SH SYNOPSIS +.B vault mirror +.RB [ \-\-key +.IR PATH ] +.RB [ \-\-vault +.IR DIR ] +.I URL +.RI [ FOLDER ] +.SH DESCRIPTION +.B vault mirror +crawls the website at +.I URL +and stores every page and asset it finds in the vault, beneath +.IR FOLDER / HOST +(or just +.I HOST +if no folder is given). +.PP +The crawl is single threaded: one request at a time, in breadth-first order. +It follows only links on the same host as +.IR URL , +and requests are sent with a Chrome user agent string. +.PP +Links in HTML +.RB ( href , +.BR src , +.BR action +and +.B poster +attributes) and in CSS +.RB ( url(...) ) +that point into the mirror are rewritten to relative paths, so the copy +browses correctly once fetched out of the vault. Links to other hosts, and +fragments, are left as they were. +.PP +Pages are named after their URL path. A path that is empty or ends in +.B / +becomes +.BR index.html ; +a query string is appended as +.BI _ query +(URL-escaped). Anything already stored under the same name is replaced. +.PP +A page that fails to download, or answers with anything but +.BR "200 OK" , +is reported on standard error and skipped; the crawl carries on. +The vault is locked exclusively for the whole run. +.SH OPTIONS +.TP +.BI \-\-key " PATH" +Key file. Default: $VAULT_KEY, else ~/.vault/key. +.TP +.BI \-\-vault " DIR" +Vault directory. Default: $VAULT_DIR, else ~/.vault/store. +.SH ENVIRONMENT +.TP +.B VAULT_PASSPHRASE +Passphrase, instead of prompting on the terminal. +.SH EXAMPLES +Mirror a site into the vault under +.BR example.com/ : +.PP +.RS +vault mirror https://example.com/ +.RE +.PP +Mirror it under a folder, then fetch the front page: +.PP +.RS +vault mirror https://example.com/ archive/2026 +.br +vault get archive/2026/example.com/index.html +.RE +.SH LIMITATIONS +Links are found by pattern matching, not by parsing HTML: URLs assembled by +scripts, and +.B srcset +lists, are neither followed nor rewritten. There is no depth limit, rate +limit or robots.txt handling; only mirror sites you may copy. +.SH SEE ALSO +.BR vault (1) +.SH AUTHOR +Andy Dixon , https://www.dixon.cx diff --git a/main.go b/main.go index 8031293..ceda30f 100644 --- a/main.go +++ b/main.go @@ -32,6 +32,9 @@ const usage = `Usage: [--yes] it is stored; asks before overwriting anything already in the vault, unless --yes is given + vault mirror URL [FOLDER] crawl a website (same host only, one request + at a time) into FOLDER/HOST, rewriting its + links to relative paths vault list [PREFIX] list stored files vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name) vault mv OLD NEW rename a stored file @@ -130,6 +133,19 @@ func main() { overwrite = func([]string) bool { return true } } check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite)) + case "mirror": + want(1, 2) + if !isURL(pos[0]) { + fail("mirror needs an http:// or https:// URL") + } + dest := "" + if len(pos) == 2 { + dest = strings.Trim(pos[1], "/") + } + s := open(*keyPath, *vaultDir, true) + cat, err := s.catalogue() + check(err) + check(mirror(s, cat, pos[0], dest)) case "list", "ls": want(0, 1) prefix := "" diff --git a/mirror.go b/mirror.go new file mode 100644 index 0000000..9699fb1 --- /dev/null +++ b/mirror.go @@ -0,0 +1,150 @@ +// Copyright 2026 The vault Authors. +// SPDX-License-Identifier: GPL-3.0-or-later + +package main + +import ( + "fmt" + "html" + "io" + "net/http" + "net/url" + "os" + "path" + "regexp" + "strings" +) + +var ( + // Attribute values in HTML, and url(...) in CSS: group 1 is everything + // up to the URL, group 2 is the URL. + htmlURLs = regexp.MustCompile(`(?i)(\b(?:href|src|action|poster)\s*=\s*["'])([^"']*)`) + cssURLs = regexp.MustCompile(`(?i)(url\(\s*["']?)([^"')\s]+)`) +) + +// mirror crawls the site at rawURL, one request at a time, staying on its +// host, and adds every page and asset beneath dest/host. Links in HTML and +// CSS that point into the mirror are rewritten to relative paths, so the +// copy browses correctly once fetched out of the vault. A page that fails +// is reported and skipped. +// ponytail: regex rewriting, so links built by scripts or in srcset are left +// as they were; swap in golang.org/x/net/html if that matters. +func mirror(s *store, cat map[string][]*record, rawURL, dest string) error { + start, err := url.Parse(rawURL) + if err != nil { + return err + } + host := strings.ToLower(start.Host) + root := path.Join(dest, host) + seen := map[string]bool{} + queue := []*url.URL{start} + seen[key(start)] = true + + for len(queue) > 0 { + u := queue[0] + queue = queue[1:] + err := mirrorOne(s, cat, u, host, root, func(l *url.URL) { + if k := key(l); !seen[k] { + seen[k] = true + queue = append(queue, l) + } + }) + if err != nil { + fmt.Fprintf(os.Stderr, "skipping %s: %v\n", u, err) + } + } + return nil +} + +// key identifies a page independent of its fragment. +func key(u *url.URL) string { + c := *u + c.Fragment = "" + return c.String() +} + +// localPath is where u lives inside the mirror, relative to its root. +func localPath(u *url.URL) string { + p := strings.TrimPrefix(u.Path, "/") + if p == "" || strings.HasSuffix(p, "/") { + p += "index.html" + } + if u.RawQuery != "" { + p += "_" + url.QueryEscape(u.RawQuery) + } + return p +} + +// relative gives the link from the file "from" to the file "to", both +// slash-separated paths relative to the same root. +func relative(from, to string) string { + f := strings.Split(path.Dir(from), "/") + t := strings.Split(to, "/") + if f[0] == "." { + f = nil + } + for len(f) > 0 && len(t) > 1 && f[0] == t[0] { + f, t = f[1:], t[1:] + } + return strings.Repeat("../", len(f)) + strings.Join(t, "/") +} + +func mirrorOne(s *store, cat map[string][]*record, u *url.URL, host, root string, enqueue func(*url.URL)) error { + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return err + } + req.Header.Set("User-Agent", chromeUA) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s", resp.Status) + } + body, err := io.ReadAll(resp.Body) + if err != nil { + return err + } + + self := localPath(u) + // rewrite maps a link found in the page to a relative one, queueing it. + rewrite := func(re *regexp.Regexp, text string) string { + return re.ReplaceAllStringFunc(text, func(m string) string { + sub := re.FindStringSubmatch(m) + l, err := u.Parse(html.UnescapeString(sub[2])) + if err != nil || (l.Scheme != "http" && l.Scheme != "https") || strings.ToLower(l.Host) != host { + return m + } + frag := l.Fragment + l.Fragment = "" + if l.Path == "" && l.RawQuery == "" { + l.Path = "/" + } + enqueue(l) + r := relative(self, localPath(l)) + if frag != "" { + r += "#" + frag + } + return sub[1] + r + }) + } + switch ct := strings.ToLower(resp.Header.Get("Content-Type")); { + case strings.HasPrefix(ct, "text/html"): + body = []byte(rewrite(cssURLs, rewrite(htmlURLs, string(body)))) + case strings.HasPrefix(ct, "text/css"): + body = []byte(rewrite(cssURLs, string(body))) + } + + tmp, err := os.CreateTemp("", "vault-mirror-") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + defer tmp.Close() + if _, err := tmp.Write(body); err != nil { + return err + } + return addOne(s, cat, tmp.Name(), path.Join(root, self), false) +} diff --git a/vault_test.go b/vault_test.go index 4befb66..05ae53c 100644 --- a/vault_test.go +++ b/vault_test.go @@ -373,3 +373,60 @@ func TestAddTree(t *testing.T) { t.Fatalf("downloaded %q", b) } } + +// TestMirror crawls a two-page site: links in HTML and CSS become relative, +// off-host links stay, and everything is stored under host/. +func TestMirror(t *testing.T) { + dir := t.TempDir() + keyPath := filepath.Join(dir, "key") + if err := writeKeyFile(keyPath, "pass", ""); err != nil { + t.Fatal(err) + } + if err := initStore(filepath.Join(dir, "store"), 0); err != nil { + t.Fatal(err) + } + s := newTestStore(t, keyPath, "pass") + var srv *httptest.Server + srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.UserAgent() != chromeUA { + http.Error(w, "bad agent", http.StatusForbidden) + return + } + switch r.URL.Path { + case "/": + w.Header().Set("Content-Type", "text/html") + io.WriteString(w, `p e `) + case "/sub/p.html": + w.Header().Set("Content-Type", "text/html") + io.WriteString(w, `home `) + case "/s.css": + w.Header().Set("Content-Type", "text/css") + io.WriteString(w, `body{background:url(/img/a.png)}`) + case "/img/a.png": + io.WriteString(w, "png") + default: + http.NotFound(w, r) + } + })) + defer srv.Close() + cat, _ := s.catalogue() + if err := mirror(s, cat, srv.URL, "m"); err != nil { + t.Fatal(err) + } + host := srv.Listener.Addr().String() + for name, want := range map[string]string{ + "m/" + host + "/index.html": `p e `, + "m/" + host + "/sub/p.html": `home `, + "m/" + host + "/s.css": `body{background:url(img/a.png)}`, + "m/" + host + "/img/a.png": "png", + } { + out := filepath.Join(dir, "out") + if err := s.get(name, out); err != nil { + t.Fatal(name, err) + } + if b, _ := os.ReadFile(out); string(b) != want { + t.Errorf("%s = %q, want %q", name, b, want) + } + os.Remove(out) + } +}