diff --git a/oracle/gen-vectors.sh b/oracle/gen-vectors.sh index 44cd62b..c2e18cb 100755 --- a/oracle/gen-vectors.sh +++ b/oracle/gen-vectors.sh @@ -10,4 +10,5 @@ cd "$(dirname "$0")/.." out=$PWD/tests/vectors mkdir -p "$out" (cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream) +(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./) echo "vectors written to $out" diff --git a/oracle/go/net_oracle_vectors_test.go b/oracle/go/net_oracle_vectors_test.go new file mode 100644 index 0000000..ef06739 --- /dev/null +++ b/oracle/go/net_oracle_vectors_test.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Test-only vector generator for the Rust port's URL handling and mirror +// link mapping; see oracle/gen-vectors.sh. + +package main + +import ( + "encoding/json" + "net/http" + "net/url" + "os" + "path/filepath" + "testing" +) + +func TestOracleNetVectors(t *testing.T) { + dir := os.Getenv("VAULT_ORACLE_OUT") + if dir == "" { + t.Skip("set VAULT_ORACLE_OUT to write vectors") + } + type parsed struct { + Input string `json:"input"` + Err bool `json:"err"` + String string `json:"string,omitempty"` + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Hostname string `json:"hostname,omitempty"` + Path string `json:"path,omitempty"` + RawQuery string `json:"raw_query,omitempty"` + Fragment string `json:"fragment,omitempty"` + Local string `json:"local_path,omitempty"` + Key string `json:"key,omitempty"` + } + describe := func(in string, u *url.URL, err error) parsed { + if err != nil { + return parsed{Input: in, Err: true} + } + return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path, + RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)} + } + refs := []string{ + "", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080", + "?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A", + "https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..", + "a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f", + "http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only", + } + bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"} + var parses []parsed + for _, r := range refs { + u, err := url.Parse(r) + parses = append(parses, describe(r, u, err)) + } + type resolved struct { + Base string `json:"base"` + Ref string `json:"ref"` + Result parsed `json:"result"` + } + var resolves []resolved + for _, b := range bases { + bu, err := url.Parse(b) + if err != nil { + t.Fatal(err) + } + for _, r := range refs { + u, err := bu.Parse(r) + resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)}) + } + } + rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"}, + {"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}} + var relOut [][3]string + for _, p := range rels { + relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])}) + } + escapes := map[string]string{} + for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} { + escapes[s] = url.QueryEscape(s) + } + times := map[string]any{} + for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} { + if tm, err := http.ParseTime(s); err == nil { + times[s] = tm.Unix() + } else { + times[s] = nil + } + } + b, _ := json.MarshalIndent(map[string]any{ + "provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go", + "parse": parses, + "resolve": resolves, + "relative": relOut, + "query_escape": escapes, + "http_time": times, + }, "", " ") + if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/src/net/html.rs b/src/net/html.rs new file mode 100644 index 0000000..8362c97 --- /dev/null +++ b/src/net/html.rs @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! Link rewriting for mirrored HTML and CSS. +//! +//! HTML is scanned with a small tokenizer following the HTML5 tokenizer's +//! rules for what matters here: tag and attribute syntax (quoted, unquoted +//! and valueless attributes), comments, doctype and other `` elements and `style` +//! attributes. CSS files get the `url(...)` rewrite only, with the same +//! matching as the Go version's regular expression +//! `(?i)(url\(\s*["']?)([^"')\s]+)`. Character references in attribute +//! values are decoded before a URL is parsed: numeric ones and the named +//! `amp`, `lt`, `gt`, `quot`, `apos` and `nbsp`; other named references +//! are left as written. + +/// Something the tokenizer found. +#[derive(Debug, PartialEq, Eq)] +pub enum Item { + /// An attribute value on a start tag: lower-case tag and attribute + /// names, the value's byte span, and whether it was quoted. + Attr { + tag: String, + name: String, + start: usize, + end: usize, + quoted: bool, + }, + /// The raw text inside a `"), + "" + ); + assert_eq!( + rw(r#"
caf\xe9
"; + assert_eq!(rewrite_html(latin1, &mut |_| None, &mut |_| None), latin1); + assert_eq!(unescape("a&b//<&unknown;'&apos"), "a&b//<&unknown;'&apos"); + } + + #[test] + fn unterminated_input() { + for s in ["", "