Add a mirror command that crawls a site into the vault
Single-threaded, same host only, Chrome user agent. Links in HTML and CSS are rewritten to relative paths. Includes a man page and a guide note. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
Andy Dixon
co-authored by
Claude Sonnet 5.5
Claude Sonnet 5.5
parent
e40a803367
commit
10f2a88493
@@ -561,6 +561,12 @@ downloads it (as Chrome, so fussy sites play along) and stores it under the
|
||||
last part of the address, such as \file{report.pdf}. The download sits in a
|
||||
private temporary file only until it's encrypted.
|
||||
|
||||
\subsection{Copying a website: \texttt{mirror}}
|
||||
\cmd{mirror} takes an \file{http://} or \file{https://} address and copies
|
||||
the whole site, one request at a time, into a folder named after its host.
|
||||
Links inside pages and stylesheets are rewritten to relative paths, so the
|
||||
copy still browses once you get it out. Only the same host is followed.
|
||||
|
||||
\subsection{Moving in: \texttt{migrate}}
|
||||
\cmd{migrate} works exactly like \cmd{add}, but deletes each original once
|
||||
it's safely stored, then tidies away any folders left empty. It won't take a
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
.TH VAULT-MIRROR 1 "September 2026" "vault" "User Commands"
|
||||
.SH NAME
|
||||
vault-mirror \- mirror a website into an encrypted vault
|
||||
.SH SYNOPSIS
|
||||
.B vault mirror
|
||||
.RB [ \-\-key
|
||||
.IR PATH ]
|
||||
.RB [ \-\-vault
|
||||
.IR DIR ]
|
||||
.I URL
|
||||
.RI [ FOLDER ]
|
||||
.SH DESCRIPTION
|
||||
.B vault mirror
|
||||
crawls the website at
|
||||
.I URL
|
||||
and stores every page and asset it finds in the vault, beneath
|
||||
.IR FOLDER / HOST
|
||||
(or just
|
||||
.I HOST
|
||||
if no folder is given).
|
||||
.PP
|
||||
The crawl is single threaded: one request at a time, in breadth-first order.
|
||||
It follows only links on the same host as
|
||||
.IR URL ,
|
||||
and requests are sent with a Chrome user agent string.
|
||||
.PP
|
||||
Links in HTML
|
||||
.RB ( href ,
|
||||
.BR src ,
|
||||
.BR action
|
||||
and
|
||||
.B poster
|
||||
attributes) and in CSS
|
||||
.RB ( url(...) )
|
||||
that point into the mirror are rewritten to relative paths, so the copy
|
||||
browses correctly once fetched out of the vault. Links to other hosts, and
|
||||
fragments, are left as they were.
|
||||
.PP
|
||||
Pages are named after their URL path. A path that is empty or ends in
|
||||
.B /
|
||||
becomes
|
||||
.BR index.html ;
|
||||
a query string is appended as
|
||||
.BI _ query
|
||||
(URL-escaped). Anything already stored under the same name is replaced.
|
||||
.PP
|
||||
A page that fails to download, or answers with anything but
|
||||
.BR "200 OK" ,
|
||||
is reported on standard error and skipped; the crawl carries on.
|
||||
The vault is locked exclusively for the whole run.
|
||||
.SH OPTIONS
|
||||
.TP
|
||||
.BI \-\-key " PATH"
|
||||
Key file. Default: $VAULT_KEY, else ~/.vault/key.
|
||||
.TP
|
||||
.BI \-\-vault " DIR"
|
||||
Vault directory. Default: $VAULT_DIR, else ~/.vault/store.
|
||||
.SH ENVIRONMENT
|
||||
.TP
|
||||
.B VAULT_PASSPHRASE
|
||||
Passphrase, instead of prompting on the terminal.
|
||||
.SH EXAMPLES
|
||||
Mirror a site into the vault under
|
||||
.BR example.com/ :
|
||||
.PP
|
||||
.RS
|
||||
vault mirror https://example.com/
|
||||
.RE
|
||||
.PP
|
||||
Mirror it under a folder, then fetch the front page:
|
||||
.PP
|
||||
.RS
|
||||
vault mirror https://example.com/ archive/2026
|
||||
.br
|
||||
vault get archive/2026/example.com/index.html
|
||||
.RE
|
||||
.SH LIMITATIONS
|
||||
Links are found by pattern matching, not by parsing HTML: URLs assembled by
|
||||
scripts, and
|
||||
.B srcset
|
||||
lists, are neither followed nor rewritten. There is no depth limit, rate
|
||||
limit or robots.txt handling; only mirror sites you may copy.
|
||||
.SH SEE ALSO
|
||||
.BR vault (1)
|
||||
.SH AUTHOR
|
||||
Andy Dixon <vault@dixon.cx>, https://www.dixon.cx
|
||||
@@ -32,6 +32,9 @@ const usage = `Usage:
|
||||
[--yes] it is stored; asks before overwriting
|
||||
anything already in the vault, unless
|
||||
--yes is given
|
||||
vault mirror URL [FOLDER] crawl a website (same host only, one request
|
||||
at a time) into FOLDER/HOST, rewriting its
|
||||
links to relative paths
|
||||
vault list [PREFIX] list stored files
|
||||
vault get NAME [-o OUTPUT] decrypt a stored file (default: its base name)
|
||||
vault mv OLD NEW rename a stored file
|
||||
@@ -130,6 +133,19 @@ func main() {
|
||||
overwrite = func([]string) bool { return true }
|
||||
}
|
||||
check(addPath(s, cat, pos[0], dest, cmd == "migrate", skip, overwrite))
|
||||
case "mirror":
|
||||
want(1, 2)
|
||||
if !isURL(pos[0]) {
|
||||
fail("mirror needs an http:// or https:// URL")
|
||||
}
|
||||
dest := ""
|
||||
if len(pos) == 2 {
|
||||
dest = strings.Trim(pos[1], "/")
|
||||
}
|
||||
s := open(*keyPath, *vaultDir, true)
|
||||
cat, err := s.catalogue()
|
||||
check(err)
|
||||
check(mirror(s, cat, pos[0], dest))
|
||||
case "list", "ls":
|
||||
want(0, 1)
|
||||
prefix := ""
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
// Copyright 2026 The vault Authors.
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"html"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
// Attribute values in HTML, and url(...) in CSS: group 1 is everything
|
||||
// up to the URL, group 2 is the URL.
|
||||
htmlURLs = regexp.MustCompile(`(?i)(\b(?:href|src|action|poster)\s*=\s*["'])([^"']*)`)
|
||||
cssURLs = regexp.MustCompile(`(?i)(url\(\s*["']?)([^"')\s]+)`)
|
||||
)
|
||||
|
||||
// mirror crawls the site at rawURL, one request at a time, staying on its
|
||||
// host, and adds every page and asset beneath dest/host. Links in HTML and
|
||||
// CSS that point into the mirror are rewritten to relative paths, so the
|
||||
// copy browses correctly once fetched out of the vault. A page that fails
|
||||
// is reported and skipped.
|
||||
// ponytail: regex rewriting, so links built by scripts or in srcset are left
|
||||
// as they were; swap in golang.org/x/net/html if that matters.
|
||||
func mirror(s *store, cat map[string][]*record, rawURL, dest string) error {
|
||||
start, err := url.Parse(rawURL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
host := strings.ToLower(start.Host)
|
||||
root := path.Join(dest, host)
|
||||
seen := map[string]bool{}
|
||||
queue := []*url.URL{start}
|
||||
seen[key(start)] = true
|
||||
|
||||
for len(queue) > 0 {
|
||||
u := queue[0]
|
||||
queue = queue[1:]
|
||||
err := mirrorOne(s, cat, u, host, root, func(l *url.URL) {
|
||||
if k := key(l); !seen[k] {
|
||||
seen[k] = true
|
||||
queue = append(queue, l)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "skipping %s: %v\n", u, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// key identifies a page independent of its fragment.
|
||||
func key(u *url.URL) string {
|
||||
c := *u
|
||||
c.Fragment = ""
|
||||
return c.String()
|
||||
}
|
||||
|
||||
// localPath is where u lives inside the mirror, relative to its root.
|
||||
func localPath(u *url.URL) string {
|
||||
p := strings.TrimPrefix(u.Path, "/")
|
||||
if p == "" || strings.HasSuffix(p, "/") {
|
||||
p += "index.html"
|
||||
}
|
||||
if u.RawQuery != "" {
|
||||
p += "_" + url.QueryEscape(u.RawQuery)
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
// relative gives the link from the file "from" to the file "to", both
|
||||
// slash-separated paths relative to the same root.
|
||||
func relative(from, to string) string {
|
||||
f := strings.Split(path.Dir(from), "/")
|
||||
t := strings.Split(to, "/")
|
||||
if f[0] == "." {
|
||||
f = nil
|
||||
}
|
||||
for len(f) > 0 && len(t) > 1 && f[0] == t[0] {
|
||||
f, t = f[1:], t[1:]
|
||||
}
|
||||
return strings.Repeat("../", len(f)) + strings.Join(t, "/")
|
||||
}
|
||||
|
||||
func mirrorOne(s *store, cat map[string][]*record, u *url.URL, host, root string, enqueue func(*url.URL)) error {
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("User-Agent", chromeUA)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("%s", resp.Status)
|
||||
}
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
self := localPath(u)
|
||||
// rewrite maps a link found in the page to a relative one, queueing it.
|
||||
rewrite := func(re *regexp.Regexp, text string) string {
|
||||
return re.ReplaceAllStringFunc(text, func(m string) string {
|
||||
sub := re.FindStringSubmatch(m)
|
||||
l, err := u.Parse(html.UnescapeString(sub[2]))
|
||||
if err != nil || (l.Scheme != "http" && l.Scheme != "https") || strings.ToLower(l.Host) != host {
|
||||
return m
|
||||
}
|
||||
frag := l.Fragment
|
||||
l.Fragment = ""
|
||||
if l.Path == "" && l.RawQuery == "" {
|
||||
l.Path = "/"
|
||||
}
|
||||
enqueue(l)
|
||||
r := relative(self, localPath(l))
|
||||
if frag != "" {
|
||||
r += "#" + frag
|
||||
}
|
||||
return sub[1] + r
|
||||
})
|
||||
}
|
||||
switch ct := strings.ToLower(resp.Header.Get("Content-Type")); {
|
||||
case strings.HasPrefix(ct, "text/html"):
|
||||
body = []byte(rewrite(cssURLs, rewrite(htmlURLs, string(body))))
|
||||
case strings.HasPrefix(ct, "text/css"):
|
||||
body = []byte(rewrite(cssURLs, string(body)))
|
||||
}
|
||||
|
||||
tmp, err := os.CreateTemp("", "vault-mirror-")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
defer tmp.Close()
|
||||
if _, err := tmp.Write(body); err != nil {
|
||||
return err
|
||||
}
|
||||
return addOne(s, cat, tmp.Name(), path.Join(root, self), false)
|
||||
}
|
||||
@@ -373,3 +373,60 @@ func TestAddTree(t *testing.T) {
|
||||
t.Fatalf("downloaded %q", b)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMirror crawls a two-page site: links in HTML and CSS become relative,
|
||||
// off-host links stay, and everything is stored under host/.
|
||||
func TestMirror(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keyPath := filepath.Join(dir, "key")
|
||||
if err := writeKeyFile(keyPath, "pass", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := initStore(filepath.Join(dir, "store"), 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := newTestStore(t, keyPath, "pass")
|
||||
var srv *httptest.Server
|
||||
srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.UserAgent() != chromeUA {
|
||||
http.Error(w, "bad agent", http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
switch r.URL.Path {
|
||||
case "/":
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
io.WriteString(w, `<a href="`+srv.URL+`/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>`)
|
||||
case "/sub/p.html":
|
||||
w.Header().Set("Content-Type", "text/html")
|
||||
io.WriteString(w, `<a href="/">home</a> <img src="../img/a.png">`)
|
||||
case "/s.css":
|
||||
w.Header().Set("Content-Type", "text/css")
|
||||
io.WriteString(w, `body{background:url(/img/a.png)}`)
|
||||
case "/img/a.png":
|
||||
io.WriteString(w, "png")
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
cat, _ := s.catalogue()
|
||||
if err := mirror(s, cat, srv.URL, "m"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
host := srv.Listener.Addr().String()
|
||||
for name, want := range map[string]string{
|
||||
"m/" + host + "/index.html": `<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>`,
|
||||
"m/" + host + "/sub/p.html": `<a href="../index.html">home</a> <img src="../img/a.png">`,
|
||||
"m/" + host + "/s.css": `body{background:url(img/a.png)}`,
|
||||
"m/" + host + "/img/a.png": "png",
|
||||
} {
|
||||
out := filepath.Join(dir, "out")
|
||||
if err := s.get(name, out); err != nil {
|
||||
t.Fatal(name, err)
|
||||
}
|
||||
if b, _ := os.ReadFile(out); string(b) != want {
|
||||
t.Errorf("%s = %q, want %q", name, b, want)
|
||||
}
|
||||
os.Remove(out)
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user