Rewrite vault in Rust (format-compatible with 0.5) #1
@@ -10,4 +10,5 @@ cd "$(dirname "$0")/.."
|
||||
out=$PWD/tests/vectors
|
||||
mkdir -p "$out"
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream)
|
||||
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./)
|
||||
echo "vectors written to $out"
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
// Test-only vector generator for the Rust port's URL handling and mirror
|
||||
// link mapping; see oracle/gen-vectors.sh.
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestOracleNetVectors(t *testing.T) {
|
||||
dir := os.Getenv("VAULT_ORACLE_OUT")
|
||||
if dir == "" {
|
||||
t.Skip("set VAULT_ORACLE_OUT to write vectors")
|
||||
}
|
||||
type parsed struct {
|
||||
Input string `json:"input"`
|
||||
Err bool `json:"err"`
|
||||
String string `json:"string,omitempty"`
|
||||
Scheme string `json:"scheme,omitempty"`
|
||||
Host string `json:"host,omitempty"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
RawQuery string `json:"raw_query,omitempty"`
|
||||
Fragment string `json:"fragment,omitempty"`
|
||||
Local string `json:"local_path,omitempty"`
|
||||
Key string `json:"key,omitempty"`
|
||||
}
|
||||
describe := func(in string, u *url.URL, err error) parsed {
|
||||
if err != nil {
|
||||
return parsed{Input: in, Err: true}
|
||||
}
|
||||
return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path,
|
||||
RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)}
|
||||
}
|
||||
refs := []string{
|
||||
"", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080",
|
||||
"?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A",
|
||||
"https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..",
|
||||
"a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f",
|
||||
"http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only",
|
||||
}
|
||||
bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"}
|
||||
var parses []parsed
|
||||
for _, r := range refs {
|
||||
u, err := url.Parse(r)
|
||||
parses = append(parses, describe(r, u, err))
|
||||
}
|
||||
type resolved struct {
|
||||
Base string `json:"base"`
|
||||
Ref string `json:"ref"`
|
||||
Result parsed `json:"result"`
|
||||
}
|
||||
var resolves []resolved
|
||||
for _, b := range bases {
|
||||
bu, err := url.Parse(b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, r := range refs {
|
||||
u, err := bu.Parse(r)
|
||||
resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)})
|
||||
}
|
||||
}
|
||||
rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"},
|
||||
{"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}}
|
||||
var relOut [][3]string
|
||||
for _, p := range rels {
|
||||
relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])})
|
||||
}
|
||||
escapes := map[string]string{}
|
||||
for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} {
|
||||
escapes[s] = url.QueryEscape(s)
|
||||
}
|
||||
times := map[string]any{}
|
||||
for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} {
|
||||
if tm, err := http.ParseTime(s); err == nil {
|
||||
times[s] = tm.Unix()
|
||||
} else {
|
||||
times[s] = nil
|
||||
}
|
||||
}
|
||||
b, _ := json.MarshalIndent(map[string]any{
|
||||
"provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go",
|
||||
"parse": parses,
|
||||
"resolve": resolves,
|
||||
"relative": relOut,
|
||||
"query_escape": escapes,
|
||||
"http_time": times,
|
||||
}, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
+503
@@ -0,0 +1,503 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Link rewriting for mirrored HTML and CSS.
|
||||
//!
|
||||
//! HTML is scanned with a small tokenizer following the HTML5 tokenizer's
|
||||
//! rules for what matters here: tag and attribute syntax (quoted, unquoted
|
||||
//! and valueless attributes), comments, doctype and other `<!`/`<?`
|
||||
//! markup, raw-text elements (script, style, xmp, iframe, noembed,
|
||||
//! noframes), RCDATA (textarea, title) and plaintext. It records byte
|
||||
//! spans, so rewriting splices new values in and leaves every other byte —
|
||||
//! quote style included — untouched.
|
||||
//!
|
||||
//! Rewritten: the values of `href`, `src`, `action` and `poster` on start
|
||||
//! tags, and CSS `url(...)` inside `<style>` elements and `style`
|
||||
//! attributes. CSS files get the `url(...)` rewrite only, with the same
|
||||
//! matching as the Go version's regular expression
|
||||
//! `(?i)(url\(\s*["']?)([^"')\s]+)`. Character references in attribute
|
||||
//! values are decoded before a URL is parsed: numeric ones and the named
|
||||
//! `amp`, `lt`, `gt`, `quot`, `apos` and `nbsp`; other named references
|
||||
//! are left as written.
|
||||
|
||||
/// Something the tokenizer found.
|
||||
#[derive(Debug, PartialEq, Eq)]
|
||||
pub enum Item {
|
||||
/// An attribute value on a start tag: lower-case tag and attribute
|
||||
/// names, the value's byte span, and whether it was quoted.
|
||||
Attr {
|
||||
tag: String,
|
||||
name: String,
|
||||
start: usize,
|
||||
end: usize,
|
||||
quoted: bool,
|
||||
},
|
||||
/// The raw text inside a `<style>` element.
|
||||
Style { start: usize, end: usize },
|
||||
}
|
||||
|
||||
fn is_ws(c: u8) -> bool {
|
||||
matches!(c, b'\t' | b'\n' | b'\x0c' | b'\r' | b' ')
|
||||
}
|
||||
|
||||
fn find(h: &[u8], from: usize, n: &[u8]) -> Option<usize> {
|
||||
if from > h.len() {
|
||||
return None;
|
||||
}
|
||||
h[from..].windows(n.len()).position(|w| w == n).map(|i| i + from)
|
||||
}
|
||||
|
||||
/// Scans `b` and returns every attribute value and style element, in
|
||||
/// document order.
|
||||
pub fn tokenize(b: &[u8]) -> Vec<Item> {
|
||||
let mut items = Vec::new();
|
||||
let mut pos = 0;
|
||||
while let Some(off) = b[pos..].iter().position(|&c| c == b'<') {
|
||||
let i = pos + off;
|
||||
let next = b.get(i + 1).copied();
|
||||
pos = match next {
|
||||
Some(b'!') => {
|
||||
if b[i..].starts_with(b"<!--") {
|
||||
let body = i + 4;
|
||||
if b[body..].starts_with(b">") {
|
||||
body + 1
|
||||
} else if b[body..].starts_with(b"->") {
|
||||
body + 2
|
||||
} else {
|
||||
find(b, body, b"-->").map_or(b.len(), |e| e + 3)
|
||||
}
|
||||
} else {
|
||||
to_gt(b, i + 2)
|
||||
}
|
||||
}
|
||||
Some(b'?') => to_gt(b, i + 2),
|
||||
Some(b'/') => match b.get(i + 2) {
|
||||
Some(c) if c.is_ascii_alphabetic() => {
|
||||
let (_, after) = tag(b, i + 2, &mut Vec::new(), "");
|
||||
after
|
||||
}
|
||||
Some(b'>') => i + 3,
|
||||
Some(_) => to_gt(b, i + 2),
|
||||
None => b.len(),
|
||||
},
|
||||
Some(c) if c.is_ascii_alphabetic() => {
|
||||
let (name, after) = tag(b, i + 1, &mut items, "start");
|
||||
match name.as_str() {
|
||||
"script" | "style" | "xmp" | "iframe" | "noembed" | "noframes" | "textarea" | "title" => {
|
||||
let end = raw_text_end(b, after, &name);
|
||||
if name == "style" {
|
||||
items.push(Item::Style { start: after, end });
|
||||
}
|
||||
end
|
||||
}
|
||||
"plaintext" => b.len(),
|
||||
_ => after,
|
||||
}
|
||||
}
|
||||
_ => i + 1,
|
||||
};
|
||||
if pos >= b.len() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
items
|
||||
}
|
||||
|
||||
fn to_gt(b: &[u8], from: usize) -> usize {
|
||||
b[from.min(b.len())..]
|
||||
.iter()
|
||||
.position(|&c| c == b'>')
|
||||
.map_or(b.len(), |e| from + e + 1)
|
||||
}
|
||||
|
||||
/// Where raw text started at `from` ends: at `</name` followed by
|
||||
/// whitespace, `/` or `>` (ASCII case-insensitive), or at end of input.
|
||||
fn raw_text_end(b: &[u8], from: usize, name: &str) -> usize {
|
||||
let mut p = from;
|
||||
while let Some(i) = find(b, p, b"</") {
|
||||
let n = i + 2;
|
||||
let candidate = b.get(n..n + name.len());
|
||||
if candidate.is_some_and(|c| c.eq_ignore_ascii_case(name.as_bytes())) {
|
||||
match b.get(n + name.len()) {
|
||||
Some(&c) if is_ws(c) || c == b'/' || c == b'>' => return i,
|
||||
None => return i,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
p = i + 2;
|
||||
}
|
||||
b.len()
|
||||
}
|
||||
|
||||
/// Parses a tag whose name starts at `from`; records start-tag attributes
|
||||
/// (when `kind` is "start") and returns the lower-case name and the index
|
||||
/// just past the closing `>` (or the end of input).
|
||||
fn tag(b: &[u8], from: usize, items: &mut Vec<Item>, kind: &str) -> (String, usize) {
|
||||
let mut attrs = Vec::new();
|
||||
let (name, end, closed) = tag_inner(b, from, &mut attrs);
|
||||
// A tag cut off by the end of input is not a tag at all.
|
||||
if closed && kind == "start" {
|
||||
items.extend(attrs);
|
||||
}
|
||||
(name, end)
|
||||
}
|
||||
|
||||
fn tag_inner(b: &[u8], from: usize, items: &mut Vec<Item>) -> (String, usize, bool) {
|
||||
let mut p = from;
|
||||
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' {
|
||||
p += 1;
|
||||
}
|
||||
let name = String::from_utf8_lossy(&b[from..p]).to_ascii_lowercase();
|
||||
loop {
|
||||
while p < b.len() && (is_ws(b[p]) || b[p] == b'/') {
|
||||
p += 1;
|
||||
}
|
||||
if p >= b.len() {
|
||||
return (name, b.len(), false);
|
||||
}
|
||||
if b[p] == b'>' {
|
||||
return (name, p + 1, true);
|
||||
}
|
||||
// Attribute name: the first character may be anything (even '='),
|
||||
// then up to whitespace, '/', '>' or '='.
|
||||
let ns = p;
|
||||
p += 1;
|
||||
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' && b[p] != b'=' {
|
||||
p += 1;
|
||||
}
|
||||
let attr = String::from_utf8_lossy(&b[ns..p]).to_ascii_lowercase();
|
||||
let mut q = p;
|
||||
while q < b.len() && is_ws(b[q]) {
|
||||
q += 1;
|
||||
}
|
||||
if q >= b.len() || b[q] != b'=' {
|
||||
p = q;
|
||||
continue;
|
||||
}
|
||||
q += 1;
|
||||
while q < b.len() && is_ws(b[q]) {
|
||||
q += 1;
|
||||
}
|
||||
if q >= b.len() {
|
||||
return (name, b.len(), false);
|
||||
}
|
||||
let (start, end, quoted, next) = match b[q] {
|
||||
quote @ (b'"' | b'\'') => {
|
||||
let s = q + 1;
|
||||
match b[s..].iter().position(|&c| c == quote) {
|
||||
Some(e) => (s, s + e, true, s + e + 1),
|
||||
// An unterminated value runs to the end; the tag is
|
||||
// incomplete, so nothing is recorded.
|
||||
None => return (name, b.len(), false),
|
||||
}
|
||||
}
|
||||
b'>' => (q, q, false, q),
|
||||
_ => {
|
||||
let mut e = q;
|
||||
while e < b.len() && !is_ws(b[e]) && b[e] != b'>' {
|
||||
e += 1;
|
||||
}
|
||||
(q, e, false, e)
|
||||
}
|
||||
};
|
||||
items.push(Item::Attr {
|
||||
tag: name.clone(),
|
||||
name: attr,
|
||||
start,
|
||||
end,
|
||||
quoted,
|
||||
});
|
||||
p = next;
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes character references (see the module documentation).
|
||||
pub fn unescape(s: &str) -> String {
|
||||
let mut out = String::with_capacity(s.len());
|
||||
let mut rest = s;
|
||||
while let Some(i) = rest.find('&') {
|
||||
out.push_str(&rest[..i]);
|
||||
rest = &rest[i..];
|
||||
let (text, used) = reference(rest);
|
||||
out.push_str(&text);
|
||||
rest = &rest[used..];
|
||||
}
|
||||
out.push_str(rest);
|
||||
out
|
||||
}
|
||||
|
||||
/// One reference at the start of `s` (which begins with '&'): its text and
|
||||
/// how many bytes it used; an unknown one is passed through as "&".
|
||||
fn reference(s: &str) -> (String, usize) {
|
||||
let b = s.as_bytes();
|
||||
if b.get(1) == Some(&b'#') {
|
||||
let (hex, start) = if matches!(b.get(2), Some(b'x' | b'X')) {
|
||||
(true, 3)
|
||||
} else {
|
||||
(false, 2)
|
||||
};
|
||||
let mut e = start;
|
||||
while e < b.len() && (if hex { b[e].is_ascii_hexdigit() } else { b[e].is_ascii_digit() }) {
|
||||
e += 1;
|
||||
}
|
||||
if e == start {
|
||||
return ("&".into(), 1);
|
||||
}
|
||||
let v = u32::from_str_radix(&s[start..e], if hex { 16 } else { 10 }).unwrap_or(0xfffd);
|
||||
let c = char::from_u32(v).filter(|&c| c != '\0').unwrap_or('\u{fffd}');
|
||||
let used = if b.get(e) == Some(&b';') { e + 1 } else { e };
|
||||
return (c.to_string(), used);
|
||||
}
|
||||
for (name, text, legacy) in [
|
||||
("amp", "&", true),
|
||||
("lt", "<", true),
|
||||
("gt", ">", true),
|
||||
("quot", "\"", true),
|
||||
("apos", "'", false),
|
||||
("nbsp", "\u{a0}", true),
|
||||
] {
|
||||
if let Some(r) = s[1..].strip_prefix(name) {
|
||||
if r.starts_with(';') {
|
||||
return (text.into(), name.len() + 2);
|
||||
}
|
||||
if legacy {
|
||||
return (text.into(), name.len() + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
("&".into(), 1)
|
||||
}
|
||||
|
||||
/// Escapes a value for an attribute, as Go's `html.EscapeString`.
|
||||
pub fn escape_attr(s: &str) -> String {
|
||||
let mut o = String::with_capacity(s.len());
|
||||
for c in s.chars() {
|
||||
match c {
|
||||
'&' => o.push_str("&"),
|
||||
'"' => o.push_str("""),
|
||||
'\'' => o.push_str("'"),
|
||||
'<' => o.push_str("<"),
|
||||
'>' => o.push_str(">"),
|
||||
c => o.push(c),
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
/// Rewrites `url(...)` references in CSS. `map` gets each URL (HTML-
|
||||
/// unescaped, as the Go version did) and returns its replacement, or None
|
||||
/// to leave it; replacements are attribute-escaped when `in_attr`. Bytes
|
||||
/// that are not UTF-8 pass through unchanged.
|
||||
pub fn rewrite_css(b: &[u8], in_attr: bool, map: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(b.len());
|
||||
let mut last = 0;
|
||||
let mut i = 0;
|
||||
while i + 4 <= b.len() {
|
||||
if !b[i..i + 4].eq_ignore_ascii_case(b"url(") {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
let mut p = i + 4;
|
||||
while p < b.len() && is_ws(b[p]) {
|
||||
p += 1;
|
||||
}
|
||||
if p < b.len() && (b[p] == b'"' || b[p] == b'\'') {
|
||||
p += 1;
|
||||
} else if in_attr {
|
||||
p += entity_quote(&b[p..]);
|
||||
}
|
||||
let start = p;
|
||||
while p < b.len() && !is_ws(b[p]) && !matches!(b[p], b'"' | b'\'' | b')') && !(in_attr && entity_quote(&b[p..]) > 0) {
|
||||
p += 1;
|
||||
}
|
||||
if p == start {
|
||||
i += 1;
|
||||
continue;
|
||||
}
|
||||
if let Some(r) = map(&unescape(&String::from_utf8_lossy(&b[start..p]))) {
|
||||
out.extend_from_slice(&b[last..start]);
|
||||
out.extend_from_slice(if in_attr { escape_attr(&r) } else { r }.as_bytes());
|
||||
last = p;
|
||||
}
|
||||
i = p;
|
||||
}
|
||||
out.extend_from_slice(&b[last..]);
|
||||
out
|
||||
}
|
||||
|
||||
/// The length of a quote written as a character reference at the start of
|
||||
/// `b` (inside an attribute value, CSS quotes arrive that way), or 0.
|
||||
fn entity_quote(b: &[u8]) -> usize {
|
||||
[""", """, "'", "'", """, "'"]
|
||||
.iter()
|
||||
.find(|q| b.starts_with(q.as_bytes()))
|
||||
.map_or(0, |q| q.len())
|
||||
}
|
||||
|
||||
/// Rewrites an HTML page. `base` is called with the first `<base href>`
|
||||
/// value (decoded) before any link after it, and returns what that value
|
||||
/// becomes; `link` maps a decoded link to its replacement, or None to
|
||||
/// leave it. Every byte outside a replaced value is kept.
|
||||
pub fn rewrite_html(page: &[u8], base: &mut dyn FnMut(&str) -> Option<String>, link: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(page.len());
|
||||
let mut last = 0;
|
||||
let mut seen_base = false;
|
||||
for item in tokenize(page) {
|
||||
let (start, end, replacement) = match item {
|
||||
Item::Attr {
|
||||
tag,
|
||||
name,
|
||||
start,
|
||||
end,
|
||||
quoted,
|
||||
} => {
|
||||
let raw = &page[start..end];
|
||||
let value = || unescape(&String::from_utf8_lossy(raw));
|
||||
let r = match name.as_str() {
|
||||
"href" if tag == "base" && !seen_base => {
|
||||
seen_base = true;
|
||||
base(&value())
|
||||
}
|
||||
"href" | "src" | "action" | "poster" => link(&value()),
|
||||
"style" => {
|
||||
let r = rewrite_css(raw, true, link);
|
||||
if r != raw {
|
||||
out.extend_from_slice(&page[last..start]);
|
||||
out.extend_from_slice(&r);
|
||||
last = end;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let Some(r) = r else { continue };
|
||||
let mut v = escape_attr(&r);
|
||||
if !quoted && (v.is_empty() || v.bytes().any(|c| is_ws(c) || b"\"'=<>`".contains(&c))) {
|
||||
v = format!("\"{v}\"");
|
||||
}
|
||||
(start, end, v.into_bytes())
|
||||
}
|
||||
Item::Style { start, end } => (start, end, rewrite_css(&page[start..end], false, link)),
|
||||
};
|
||||
out.extend_from_slice(&page[last..start]);
|
||||
out.extend_from_slice(&replacement);
|
||||
last = end;
|
||||
}
|
||||
out.extend_from_slice(&page[last..]);
|
||||
out
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn upper(s: &str) -> Option<String> {
|
||||
Some(s.to_uppercase())
|
||||
}
|
||||
|
||||
fn rw(page: &str) -> String {
|
||||
String::from_utf8(rewrite_html(page.as_bytes(), &mut |_| None, &mut upper)).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attributes_and_quotes_preserved() {
|
||||
assert_eq!(
|
||||
rw(r#"<a href="x">x</a> <link href='y'> <img src=z alt=q>"#),
|
||||
r#"<a href="X">x</a> <link href='Y'> <img src=Z alt=q>"#
|
||||
);
|
||||
assert_eq!(rw(r#"<A HREF = "x" >"#), r#"<A HREF = "X" >"#);
|
||||
assert_eq!(rw(r#"<form action="/p?a=1&b=2">"#), r#"<form action="/P?A=1&B=2">"#);
|
||||
assert_eq!(
|
||||
rw(r#"<video poster="p.jpg" data-src="d">"#),
|
||||
r#"<video poster="P.JPG" data-src="d">"#
|
||||
);
|
||||
assert_eq!(rw("<p>href=\"x\"</p>"), "<p>href=\"x\"</p>", "text is not markup");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn comments_scripts_rcdata_untouched() {
|
||||
let s = r#"<!-- <a href="c"> --><script>var a = '<a href="s">';</script><textarea><a href="t"></textarea><title><img src=u></title><a href="ok">"#;
|
||||
assert_eq!(rw(s), s.replace("\"ok\"", "\"OK\""));
|
||||
assert_eq!(rw("<!--><a href=a>"), "<!--><a href=A>");
|
||||
assert_eq!(rw("<!DOCTYPE html><a href=a>"), "<!DOCTYPE html><a href=A>");
|
||||
assert_eq!(rw("<script>x</SCRIPT ><a href=a>"), "<script>x</SCRIPT ><a href=A>");
|
||||
assert_eq!(rw("<script>x</scripty><a href=a>"), "<script>x</scripty><a href=a>");
|
||||
assert_eq!(rw("<plaintext><a href=a>"), "<plaintext><a href=a>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn css_in_html_and_files() {
|
||||
assert_eq!(
|
||||
rw("<style>body{background:url( 'a.png' )}</style>"),
|
||||
"<style>body{background:url( 'A.PNG' )}</style>"
|
||||
);
|
||||
assert_eq!(
|
||||
rw(r#"<div style="background: URL("b.png")">"#),
|
||||
r#"<div style="background: URL("B.PNG")">"#
|
||||
);
|
||||
let mut m = |s: &str| Some(format!("<{s}>"));
|
||||
assert_eq!(
|
||||
rewrite_css(b"a{b:url(x)} c{d:url()} url(\"y\")", false, &mut m),
|
||||
b"a{b:url(<x>)} c{d:url()} url(\"<y>\")"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base_href_and_escaping() {
|
||||
let mut seen = Vec::new();
|
||||
let out = rewrite_html(
|
||||
br#"<base href="http://h/d/"><base href="x"><a href=a>"#,
|
||||
&mut |b| {
|
||||
seen.push(b.to_string());
|
||||
Some("p.html".into())
|
||||
},
|
||||
&mut upper,
|
||||
);
|
||||
assert_eq!(seen, ["http://h/d/"]);
|
||||
assert_eq!(out, br#"<base href="p.html"><base href="X"><a href=A>"#);
|
||||
let out = rewrite_html(b"<a href=a>", &mut |_| None, &mut |_| Some("a b\"&".into()));
|
||||
assert_eq!(out, br#"<a href="a b"&">"#);
|
||||
// Non-UTF-8 bytes around and inside values survive.
|
||||
let latin1 = b"<p>caf\xe9</p><a href=\"x\xe9\">";
|
||||
assert_eq!(rewrite_html(latin1, &mut |_| None, &mut |_| None), latin1);
|
||||
assert_eq!(unescape("a&b//<&unknown;'&apos"), "a&b//<&unknown;'&apos");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unterminated_input() {
|
||||
for s in ["<a href=\"x", "<a href=", "<a", "<", "<!--", "<script>", "</a", "<a href='x' "] {
|
||||
assert_eq!(rw(s), s, "{s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
/// Random and mutated pages never panic, and an identity rewrite
|
||||
/// returns the input byte for byte.
|
||||
#[test]
|
||||
fn fuzz_identity() {
|
||||
let seeds = [
|
||||
r#"<html><head><base href="/x/"><style>a{b:url(c)}</style></head><body><a href='d'>e</a><img src=f style="g:url(h)"><!-- i --><script>j</script></body></html>"#,
|
||||
"<a href=\"x\"><b <c/> </d e='f'> <!DOCTYPE> <? pi ?> <textarea></textarea>",
|
||||
];
|
||||
let mut rng = crate::rng::os();
|
||||
let alphabet = b"<>/!-='\" \tabcdefhrsxyzlptu()&;#";
|
||||
for i in 0..20000 {
|
||||
let mut page: Vec<u8> = seeds[i % 2].as_bytes().to_vec();
|
||||
for _ in 0..rng.below(8) {
|
||||
let at = rng.below(page.len() as u64 + 1) as usize;
|
||||
match rng.below(3) {
|
||||
0 => page.insert(at, alphabet[rng.below(alphabet.len() as u64) as usize]),
|
||||
1 if at < page.len() => {
|
||||
page.remove(at);
|
||||
}
|
||||
_ => page.truncate(at),
|
||||
}
|
||||
}
|
||||
if i % 3 == 0 {
|
||||
page.push(0xff);
|
||||
}
|
||||
assert_eq!(rewrite_html(&page, &mut |_| None, &mut |_| None), page);
|
||||
// Rewriting everything must not panic either.
|
||||
let _ = rewrite_html(&page, &mut |b| Some(b.into()), &mut |v| Some(format!("{v}&x")));
|
||||
let _ = tokenize(&page);
|
||||
}
|
||||
}
|
||||
}
|
||||
+657
-1
@@ -1,3 +1,659 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Network import: `add URL` and `mirror`. (Being written.)
|
||||
//! Network import: `add URL` and `mirror`.
|
||||
//!
|
||||
//! Every request goes through one explicit client: TLS certificates are
|
||||
//! always verified against the platform's roots, redirects are followed by
|
||||
//! hand (http and https only, at most [`Limits::max_redirects`], and for
|
||||
//! `mirror` only within the starting host), and every body is counted
|
||||
//! after decompression against a byte budget — `Content-Length` is never
|
||||
//! trusted. Proxies follow Go's `ProxyFromEnvironment`: `HTTPS_PROXY` for
|
||||
//! https, `HTTP_PROXY` for http (ignored under CGI, when `REQUEST_METHOD`
|
||||
//! is set), `NO_PROXY` exclusions, and never for localhost or loopback.
|
||||
//!
|
||||
//! Defaults (all overridable from the command line, the byte budget never
|
||||
//! above the 1 TiB file limit):
|
||||
//!
|
||||
//! | limit | `add URL` | `mirror` |
|
||||
//! |------------------------|-----------|-----------------|
|
||||
//! | connect | 30 s | 30 s |
|
||||
//! | response headers | 60 s | 60 s |
|
||||
//! | whole request | 2 h | 5 min each |
|
||||
//! | bytes | 16 GiB | 4 GiB in total |
|
||||
//! | redirects per request | 10 | 10 |
|
||||
//! | pages | — | 10 000 |
|
||||
//! | HTML/CSS held to rewrite | — | 16 MiB per page |
|
||||
//!
|
||||
//! Downloads that are not rewritten are streamed to a private (0600)
|
||||
//! temporary file, because the vault needs a file's size before storing
|
||||
//! it; that file holds plaintext until it is removed, on every path, once
|
||||
//! stored. HTML and CSS for rewriting are held in memory instead.
|
||||
|
||||
mod html;
|
||||
mod url;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store::{Catalogue, FILE_LIMIT, Store, VaultName};
|
||||
use crate::util::Instant;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
use std::io::{Read, Write};
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
pub use url::Url;
|
||||
|
||||
pub const CHROME_UA: &str =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct Limits {
|
||||
pub connect: Duration,
|
||||
/// Until the response headers have arrived.
|
||||
pub response: Duration,
|
||||
/// End to end, body included, for each request.
|
||||
pub request: Duration,
|
||||
/// Body bytes: per download for `add`, in total for `mirror`.
|
||||
pub max_bytes: u64,
|
||||
pub max_redirects: u32,
|
||||
pub max_pages: u64,
|
||||
/// Largest HTML or CSS document held in memory for rewriting.
|
||||
pub max_text: u64,
|
||||
}
|
||||
|
||||
impl Limits {
|
||||
pub fn add_defaults() -> Limits {
|
||||
Limits {
|
||||
connect: Duration::from_secs(30),
|
||||
response: Duration::from_secs(60),
|
||||
request: Duration::from_secs(2 * 3600),
|
||||
max_bytes: 16 << 30,
|
||||
max_redirects: 10,
|
||||
max_pages: 1,
|
||||
max_text: 16 << 20,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn mirror_defaults() -> Limits {
|
||||
Limits {
|
||||
request: Duration::from_secs(300),
|
||||
max_bytes: 4 << 30,
|
||||
max_pages: 10_000,
|
||||
..Limits::add_defaults()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_url(s: &str) -> bool {
|
||||
s.starts_with("https://") || s.starts_with("http://")
|
||||
}
|
||||
|
||||
/// Why a fetch failed: the byte budget ends a mirror, anything else only
|
||||
/// skips one resource.
|
||||
enum FetchError {
|
||||
Budget(u64),
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for FetchError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
FetchError::Budget(n) => write!(f, "download exceeds {n} bytes"),
|
||||
FetchError::Other(m) => f.write_str(m),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<String> for FetchError {
|
||||
fn from(s: String) -> Self {
|
||||
FetchError::Other(s)
|
||||
}
|
||||
}
|
||||
|
||||
type Response = ureq::http::Response<ureq::Body>;
|
||||
|
||||
struct Client {
|
||||
limits: Limits,
|
||||
agents: HashMap<Option<String>, ureq::Agent>,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
fn new(limits: &Limits) -> Client {
|
||||
Client {
|
||||
limits: limits.clone(),
|
||||
agents: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn agent(&mut self, proxy: Option<String>) -> std::result::Result<&ureq::Agent, String> {
|
||||
if !self.agents.contains_key(&proxy) {
|
||||
let p = match &proxy {
|
||||
Some(p) => Some(ureq::Proxy::new(p).map_err(|e| format!("invalid proxy {p:?}: {e}"))?),
|
||||
None => None,
|
||||
};
|
||||
let tls = ureq::tls::TlsConfig::builder()
|
||||
.root_certs(ureq::tls::RootCerts::PlatformVerifier)
|
||||
.build();
|
||||
let l = &self.limits;
|
||||
let cfg = ureq::Agent::config_builder()
|
||||
.tls_config(tls)
|
||||
.proxy(p)
|
||||
.max_redirects(0)
|
||||
.http_status_as_error(false)
|
||||
.user_agent(CHROME_UA)
|
||||
.timeout_connect(Some(l.connect))
|
||||
.timeout_recv_response(Some(l.response))
|
||||
.timeout_global(Some(l.request))
|
||||
.build();
|
||||
self.agents.insert(proxy.clone(), cfg.into());
|
||||
}
|
||||
Ok(&self.agents[&proxy])
|
||||
}
|
||||
|
||||
/// GETs `u`, following redirects; with `host`, every hop must stay on
|
||||
/// it. Returns the final URL and its response, whatever the status.
|
||||
fn get(&mut self, u: &Url, host: Option<&str>) -> std::result::Result<(Url, Response), String> {
|
||||
let mut cur = u.clone();
|
||||
for hop in 0..=self.limits.max_redirects {
|
||||
if cur.scheme != "http" && cur.scheme != "https" {
|
||||
return Err(format!("unsupported protocol scheme {:?}", cur.scheme));
|
||||
}
|
||||
let agent = self.agent(proxy_for(&cur))?;
|
||||
let resp = agent.get(cur.to_string()).call().map_err(|e| e.to_string())?;
|
||||
let code = resp.status().as_u16();
|
||||
let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_string);
|
||||
match (code, location) {
|
||||
(301 | 302 | 303 | 307 | 308, Some(loc)) => {
|
||||
if hop == self.limits.max_redirects {
|
||||
return Err(format!("stopped after {} redirects", self.limits.max_redirects));
|
||||
}
|
||||
cur = cur.join(&loc).map_err(|e| format!("bad redirect location {loc:?}: {e}"))?;
|
||||
if host.is_some_and(|h| cur.host.to_lowercase() != h) {
|
||||
return Err(format!("redirected to another host: {cur}"));
|
||||
}
|
||||
}
|
||||
_ => return Ok((cur, resp)),
|
||||
}
|
||||
}
|
||||
unreachable!("the loop returns on its last hop")
|
||||
}
|
||||
}
|
||||
|
||||
fn status_error(resp: &Response) -> Option<String> {
|
||||
let s = resp.status();
|
||||
(s.as_u16() != 200).then(|| {
|
||||
format!("{} {}", s.as_u16(), s.canonical_reason().unwrap_or(""))
|
||||
.trim_end()
|
||||
.to_string()
|
||||
})
|
||||
}
|
||||
|
||||
/// Copies at most `budget` decoded body bytes to `dst`; returns the count.
|
||||
fn copy_body(resp: &mut Response, dst: &mut dyn Write, budget: u64) -> std::result::Result<u64, FetchError> {
|
||||
let mut r = resp.body_mut().with_config().limit(u64::MAX).reader();
|
||||
let mut buf = vec![0u8; 64 << 10];
|
||||
let mut n = 0u64;
|
||||
loop {
|
||||
let k = match r.read(&mut buf) {
|
||||
Ok(0) => return Ok(n),
|
||||
Ok(k) => k,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(FetchError::Other(crate::util::io_reason(&e))),
|
||||
};
|
||||
n += k as u64;
|
||||
if n > budget {
|
||||
return Err(FetchError::Budget(budget));
|
||||
}
|
||||
dst.write_all(&buf[..k])
|
||||
.map_err(|e| FetchError::Other(crate::util::io_reason(&e)))?;
|
||||
}
|
||||
}
|
||||
|
||||
/// A private temporary file, removed when dropped.
|
||||
struct TempFile {
|
||||
file: Option<std::fs::File>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl TempFile {
|
||||
fn new(prefix: &str) -> Result<TempFile> {
|
||||
let dir = std::env::temp_dir();
|
||||
let (file, path) = crate::platform::fs::create_temp(&dir, prefix).map_err(|e| Error::io("create temporary file in", &dir, &e))?;
|
||||
Ok(TempFile { file: Some(file), path })
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for TempFile {
|
||||
fn drop(&mut self) {
|
||||
drop(self.file.take());
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn vault_name(dest: &str, name: &str) -> Result<VaultName> {
|
||||
let joined = crate::util::join(&[dest, name]);
|
||||
VaultName::new(&joined).ok_or_else(|| Error::new(format!("invalid vault path {joined:?}")))
|
||||
}
|
||||
|
||||
/// Downloads `url` and stores it beneath `dest`, named after the last
|
||||
/// element of the final (post-redirect) URL's path, or its host name.
|
||||
pub fn add_url(
|
||||
store: &Store,
|
||||
cat: &mut Catalogue,
|
||||
url: &str,
|
||||
dest: &str,
|
||||
limits: &Limits,
|
||||
report: &mut dyn FnMut(&VaultName, bool),
|
||||
) -> Result<()> {
|
||||
let u = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
let mut client = Client::new(limits);
|
||||
let (fin, mut resp) = client.get(&u, None).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
if let Some(e) = status_error(&resp) {
|
||||
crate::bail!("{url}: {e}");
|
||||
}
|
||||
let mut name = crate::util::base(&fin.path);
|
||||
if name == "/" || name == "." {
|
||||
name = fin.hostname().to_string();
|
||||
}
|
||||
let vn = vault_name(dest, &name)?;
|
||||
let mut tmp = TempFile::new("vault-download-")?;
|
||||
let f = tmp.file.as_mut().expect("open");
|
||||
copy_body(&mut resp, f, limits.max_bytes.min(FILE_LIMIT)).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
if let Some(t) = resp
|
||||
.headers()
|
||||
.get("last-modified")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.and_then(parse_http_time)
|
||||
{
|
||||
let t = Instant { secs: t, nanos: 0 }.to_system();
|
||||
let _ = f.set_times(std::fs::FileTimes::new().set_accessed(t).set_modified(t));
|
||||
}
|
||||
drop(tmp.file.take());
|
||||
let (added, _) = store.add_file(cat, &tmp.path, &vn)?;
|
||||
report(&vn, added.replaced);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub enum MirrorOutcome {
|
||||
Complete { stored: u64 },
|
||||
Partial { stored: u64, skipped: u64 },
|
||||
}
|
||||
|
||||
/// Where `u` lives inside a mirror, relative to its root (Go's localPath).
|
||||
pub(crate) fn local_path(u: &Url) -> String {
|
||||
let mut p = u.path.strip_prefix('/').unwrap_or(&u.path).to_string();
|
||||
if p.is_empty() || p.ends_with('/') {
|
||||
p.push_str("index.html");
|
||||
}
|
||||
if !u.raw_query.is_empty() {
|
||||
p.push('_');
|
||||
p.push_str(&url::query_escape(&u.raw_query));
|
||||
}
|
||||
p
|
||||
}
|
||||
|
||||
/// A page's identity, independent of its fragment (Go's key).
|
||||
pub(crate) fn key(u: &Url) -> String {
|
||||
let mut c = u.clone();
|
||||
c.clear_fragment();
|
||||
c.to_string()
|
||||
}
|
||||
|
||||
/// The link from the file `from` to the file `to`, both relative to the
|
||||
/// same root (Go's relative).
|
||||
pub(crate) fn relative(from: &str, to: &str) -> String {
|
||||
let dir = crate::util::clean(from.rsplit_once('/').map_or(".", |(d, _)| if d.is_empty() { "/" } else { d }));
|
||||
let mut f: Vec<&str> = dir.split('/').collect();
|
||||
let mut t: Vec<&str> = to.split('/').collect();
|
||||
if f[0] == "." {
|
||||
f.clear();
|
||||
}
|
||||
while !f.is_empty() && t.len() > 1 && f[0] == t[0] {
|
||||
f.remove(0);
|
||||
t.remove(0);
|
||||
}
|
||||
format!("{}{}", "../".repeat(f.len()), t.join("/"))
|
||||
}
|
||||
|
||||
struct Crawl<'a> {
|
||||
store: &'a Store,
|
||||
client: Client,
|
||||
host: String,
|
||||
root: String,
|
||||
seen: HashSet<String>,
|
||||
queue: VecDeque<Url>,
|
||||
owners: HashMap<String, String>,
|
||||
budget: u64,
|
||||
max_pages: u64,
|
||||
limit_hit: bool,
|
||||
max_text: u64,
|
||||
}
|
||||
|
||||
/// Crawls the site at `url`, one request at a time, staying on its host,
|
||||
/// and stores every page and asset beneath `dest/HOST`. Links in HTML and
|
||||
/// CSS that point into the mirror are rewritten to relative paths. A
|
||||
/// resource that fails is reported and skipped.
|
||||
pub fn mirror(
|
||||
store: &Store,
|
||||
cat: &mut Catalogue,
|
||||
url: &str,
|
||||
dest: &str,
|
||||
limits: &Limits,
|
||||
report: &mut dyn FnMut(&VaultName, bool),
|
||||
) -> Result<MirrorOutcome> {
|
||||
let mut start = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
|
||||
if start.scheme != "http" && start.scheme != "https" {
|
||||
crate::bail!("mirror needs an http:// or https:// URL");
|
||||
}
|
||||
if start.path.is_empty() && start.raw_query.is_empty() {
|
||||
// The same page as a link to "/" would name.
|
||||
start = start.join("/").expect("valid reference");
|
||||
}
|
||||
let host = start.host.to_lowercase();
|
||||
let mut c = Crawl {
|
||||
store,
|
||||
client: Client::new(limits),
|
||||
root: crate::util::join(&[dest, &host]),
|
||||
host,
|
||||
seen: HashSet::new(),
|
||||
queue: VecDeque::new(),
|
||||
owners: HashMap::new(),
|
||||
budget: limits.max_bytes.min(FILE_LIMIT),
|
||||
max_pages: limits.max_pages,
|
||||
limit_hit: false,
|
||||
max_text: limits.max_text,
|
||||
};
|
||||
c.enqueue(start);
|
||||
let (mut stored, mut skipped) = (0u64, 0u64);
|
||||
while let Some(u) = c.queue.pop_front() {
|
||||
match c.one(cat, &u, report) {
|
||||
Ok(()) => stored += 1,
|
||||
Err(FetchError::Budget(n)) => {
|
||||
eprintln!("skipping {u}: the mirror's {n}-byte budget is used up");
|
||||
skipped += 1 + c.queue.len() as u64;
|
||||
c.queue.clear();
|
||||
}
|
||||
Err(FetchError::Other(e)) => {
|
||||
eprintln!("skipping {u}: {e}");
|
||||
skipped += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
if c.limit_hit {
|
||||
eprintln!("mirror: page limit of {} reached; further links were not followed", c.max_pages);
|
||||
skipped += 1;
|
||||
}
|
||||
Ok(if skipped == 0 {
|
||||
MirrorOutcome::Complete { stored }
|
||||
} else {
|
||||
MirrorOutcome::Partial { stored, skipped }
|
||||
})
|
||||
}
|
||||
|
||||
impl Crawl<'_> {
|
||||
fn enqueue(&mut self, u: Url) {
|
||||
let k = key(&u);
|
||||
if self.seen.contains(&k) {
|
||||
return;
|
||||
}
|
||||
if self.seen.len() as u64 >= self.max_pages {
|
||||
self.limit_hit = true;
|
||||
return;
|
||||
}
|
||||
self.seen.insert(k);
|
||||
self.queue.push_back(u);
|
||||
}
|
||||
|
||||
fn one(&mut self, cat: &mut Catalogue, u: &Url, report: &mut dyn FnMut(&VaultName, bool)) -> std::result::Result<(), FetchError> {
|
||||
let local = local_path(u);
|
||||
let k = key(u);
|
||||
match self.owners.get(&local) {
|
||||
Some(owner) if *owner != k => return Err(format!("its local name {local} is already used by {owner}").into()),
|
||||
_ => {
|
||||
self.owners.insert(local.clone(), k);
|
||||
}
|
||||
}
|
||||
let name = vault_name(&self.root, &local).map_err(|e| e.msg)?;
|
||||
let (fin, mut resp) = self.client.get(u, Some(&self.host))?;
|
||||
if let Some(e) = status_error(&resp) {
|
||||
return Err(e.into());
|
||||
}
|
||||
let ct = resp
|
||||
.headers()
|
||||
.get("content-type")
|
||||
.and_then(|v| v.to_str().ok())
|
||||
.unwrap_or("")
|
||||
.to_lowercase();
|
||||
let (is_html, is_css) = (ct.starts_with("text/html"), ct.starts_with("text/css"));
|
||||
let added = if is_html || is_css {
|
||||
let mut body = Vec::new();
|
||||
let cap = self.budget.min(self.max_text);
|
||||
let n = copy_body(&mut resp, &mut body, cap).map_err(|e| match e {
|
||||
FetchError::Budget(_) if cap < self.budget => {
|
||||
FetchError::Other(format!("larger than the {}-byte limit for pages to rewrite", self.max_text))
|
||||
}
|
||||
e => e,
|
||||
})?;
|
||||
self.budget -= n;
|
||||
let body = self.rewrite(&body, is_html, &fin, &local);
|
||||
self.store
|
||||
.add_reader(cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name)
|
||||
.map_err(|e| FetchError::Other(e.msg))?
|
||||
} else {
|
||||
let mut tmp = TempFile::new("vault-mirror-").map_err(|e| FetchError::Other(e.msg))?;
|
||||
let n = copy_body(&mut resp, tmp.file.as_mut().expect("open"), self.budget)?;
|
||||
self.budget -= n;
|
||||
drop(tmp.file.take());
|
||||
self.store.add_file(cat, &tmp.path, &name).map_err(|e| FetchError::Other(e.msg))?.0
|
||||
};
|
||||
report(&name, added.replaced);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Rewrites links that point into the mirror, queueing their targets.
|
||||
fn rewrite(&mut self, body: &[u8], is_html: bool, page: &Url, local: &str) -> Vec<u8> {
|
||||
let base = std::cell::RefCell::new(page.clone());
|
||||
let found = std::cell::RefCell::new(Vec::new());
|
||||
let host = self.host.clone();
|
||||
let mut link = |v: &str| -> Option<String> {
|
||||
let mut l = base.borrow().join(v).ok()?;
|
||||
if (l.scheme != "http" && l.scheme != "https") || l.host.to_lowercase() != host {
|
||||
return None;
|
||||
}
|
||||
let frag = l.fragment.clone();
|
||||
l.clear_fragment();
|
||||
if l.path.is_empty() && l.raw_query.is_empty() {
|
||||
l = l.join("/").ok()?;
|
||||
}
|
||||
let mut r = relative(local, &local_path(&l));
|
||||
found.borrow_mut().push(l);
|
||||
if !frag.is_empty() {
|
||||
r.push('#');
|
||||
r.push_str(&frag);
|
||||
}
|
||||
Some(r)
|
||||
};
|
||||
let out = if is_html {
|
||||
let mut on_base = |v: &str| -> Option<String> {
|
||||
let b = base.borrow().join(v).ok()?;
|
||||
let same = (b.scheme == "http" || b.scheme == "https") && b.host.to_lowercase() == host;
|
||||
*base.borrow_mut() = b;
|
||||
// Point the copy's base at the page itself, so the relative
|
||||
// links written here resolve as intended.
|
||||
same.then(|| relative(local, local))
|
||||
};
|
||||
html::rewrite_html(body, &mut on_base, &mut link)
|
||||
} else {
|
||||
html::rewrite_css(body, false, &mut link)
|
||||
};
|
||||
for l in found.into_inner() {
|
||||
self.enqueue(l);
|
||||
}
|
||||
out
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `ProxyFromEnvironment`: the proxy URL for `u`, if any.
|
||||
fn proxy_for(u: &Url) -> Option<String> {
|
||||
let env = |names: &[&str]| names.iter().find_map(|n| std::env::var(n).ok().filter(|v| !v.is_empty()));
|
||||
let proxy = if u.scheme == "https" {
|
||||
env(&["HTTPS_PROXY", "https_proxy"])
|
||||
} else if std::env::var_os("REQUEST_METHOD").is_some() {
|
||||
None
|
||||
} else {
|
||||
env(&["HTTP_PROXY", "http_proxy"])
|
||||
}?;
|
||||
if !use_proxy(u, env(&["NO_PROXY", "no_proxy"]).as_deref().unwrap_or("")) {
|
||||
return None;
|
||||
}
|
||||
Some(if proxy.contains("://") { proxy } else { format!("http://{proxy}") })
|
||||
}
|
||||
|
||||
/// Whether `u` should go through a proxy given `no_proxy` (Go's
|
||||
/// httpproxy semantics).
|
||||
fn use_proxy(u: &Url, no_proxy: &str) -> bool {
|
||||
let host = u.hostname().to_lowercase();
|
||||
let port = u.port_or_default();
|
||||
if host == "localhost" {
|
||||
return false;
|
||||
}
|
||||
let ip: Option<std::net::IpAddr> = host.parse().ok();
|
||||
if ip.is_some_and(|ip| ip.is_loopback()) {
|
||||
return false;
|
||||
}
|
||||
for entry in no_proxy.split(',').map(|e| e.trim().to_lowercase()).filter(|e| !e.is_empty()) {
|
||||
if entry == "*" {
|
||||
return false;
|
||||
}
|
||||
if let Some((net, bits)) = entry.split_once('/') {
|
||||
if let (Ok(net), Ok(bits), Some(ip)) = (net.parse::<std::net::IpAddr>(), bits.parse::<u32>(), ip)
|
||||
&& in_cidr(ip, net, bits)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if let Ok(eip) = entry.trim_start_matches('[').trim_end_matches(']').parse::<std::net::IpAddr>() {
|
||||
if ip == Some(eip) {
|
||||
return false;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let (mut phost, pport) = match entry.rsplit_once(':') {
|
||||
Some((h, p)) if !h.is_empty() && p.bytes().all(|c| c.is_ascii_digit()) => (h.to_string(), Some(p)),
|
||||
_ => (entry.clone(), None),
|
||||
};
|
||||
if let Some(rest) = phost.strip_prefix("*.") {
|
||||
phost = format!(".{rest}");
|
||||
}
|
||||
let match_host = !phost.starts_with('.');
|
||||
if match_host {
|
||||
phost.insert(0, '.');
|
||||
}
|
||||
if (host.ends_with(&phost) || (match_host && host == phost[1..])) && pport.is_none_or(|p| p == port) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn in_cidr(ip: std::net::IpAddr, net: std::net::IpAddr, bits: u32) -> bool {
|
||||
let (a, b, width): (u128, u128, u32) = match (ip, net) {
|
||||
(std::net::IpAddr::V4(a), std::net::IpAddr::V4(b)) => (u32::from(a) as u128, u32::from(b) as u128, 32),
|
||||
(std::net::IpAddr::V6(a), std::net::IpAddr::V6(b)) => (u128::from(a), u128::from(b), 128),
|
||||
_ => return false,
|
||||
};
|
||||
if bits > width {
|
||||
return false;
|
||||
}
|
||||
let shift = width - bits;
|
||||
shift >= width || (a >> shift) == (b >> shift)
|
||||
}
|
||||
|
||||
/// Go's `http.ParseTime`: RFC 1123 with "GMT", RFC 850, or ANSI C, as Unix
|
||||
/// seconds.
|
||||
fn parse_http_time(s: &str) -> Option<i64> {
|
||||
const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
|
||||
const DAYS: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
|
||||
const LONG: [&str; 7] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
|
||||
let month = |m: &str| MONTHS.iter().position(|x| *x == m).map(|i| i as i64 + 1);
|
||||
let num = |t: &str, n: usize| {
|
||||
(t.len() == n && t.bytes().all(|c| c.is_ascii_digit()))
|
||||
.then(|| t.parse::<i64>().ok())
|
||||
.flatten()
|
||||
};
|
||||
let clock = |t: &str| -> Option<i64> {
|
||||
let p: Vec<&str> = t.split(':').collect();
|
||||
let (h, m, s) = (num(p.first()?, 2)?, num(p.get(1)?, 2)?, num(p.get(2)?, 2)?);
|
||||
(p.len() == 3 && h < 24 && m < 60 && s < 60).then_some(h * 3600 + m * 60 + s)
|
||||
};
|
||||
let date = |y: i64, mo: i64, d: i64| -> Option<i64> {
|
||||
let dim = [
|
||||
31,
|
||||
if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 { 29 } else { 28 },
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
30,
|
||||
31,
|
||||
];
|
||||
(d >= 1 && d <= dim[(mo - 1) as usize]).then(|| days_from_civil(y, mo, d) * 86400)
|
||||
};
|
||||
// Mon, 02 Jan 2006 15:04:05 GMT
|
||||
if let Some(rest) = s.get(..5).filter(|d| DAYS.contains(&&d[..3]) && &d[3..] == ", ").map(|_| &s[5..]) {
|
||||
let p: Vec<&str> = rest.split(' ').collect();
|
||||
if p.len() == 5 && p[4] == "GMT" {
|
||||
return Some(date(num(p[2], 4)?, month(p[1])?, num(p[0], 2)?)? + clock(p[3])?);
|
||||
}
|
||||
return None;
|
||||
}
|
||||
// Monday, 02-Jan-06 15:04:05 MST
|
||||
if let Some((day, rest)) = s.split_once(", ") {
|
||||
if !LONG.contains(&day) {
|
||||
return None;
|
||||
}
|
||||
let p: Vec<&str> = rest.split(' ').collect();
|
||||
if p.len() != 3 || !(3..=5).contains(&p[2].len()) || !p[2].bytes().all(|c| c.is_ascii_uppercase()) {
|
||||
return None;
|
||||
}
|
||||
let d: Vec<&str> = p[0].split('-').collect();
|
||||
if d.len() != 3 {
|
||||
return None;
|
||||
}
|
||||
let yy = num(d[2], 2)?;
|
||||
return Some(date(if yy >= 69 { 1900 + yy } else { 2000 + yy }, month(d[1])?, num(d[0], 2)?)? + clock(p[1])?);
|
||||
}
|
||||
// Mon Jan _2 15:04:05 2006
|
||||
let (head, rest) = (s.get(..8)?, s.get(8..)?);
|
||||
if !DAYS.contains(&&head[..3]) || &head[3..4] != " " || &head[7..8] != " " {
|
||||
return None;
|
||||
}
|
||||
let mo = month(&head[4..7])?;
|
||||
let day = if let Some(r) = rest.strip_prefix(' ') {
|
||||
(r, num(r.get(..1)?, 1)?)
|
||||
} else {
|
||||
(rest, num(rest.get(..2)?, 2)?)
|
||||
};
|
||||
let (r, d) = (day.0, day.1);
|
||||
let after = &r[if rest.starts_with(' ') { 1 } else { 2 }..];
|
||||
let p: Vec<&str> = after.strip_prefix(' ')?.split(' ').collect();
|
||||
if p.len() != 2 {
|
||||
return None;
|
||||
}
|
||||
Some(date(num(p[1], 4)?, mo, d)? + clock(p[0])?)
|
||||
}
|
||||
|
||||
/// Days since 1970-01-01 of a proleptic Gregorian date.
|
||||
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
|
||||
let y = if m <= 2 { y - 1 } else { y };
|
||||
let era = y.div_euclid(400);
|
||||
let yoe = y - era * 400;
|
||||
let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1;
|
||||
era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468
|
||||
}
|
||||
|
||||
@@ -0,0 +1,412 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Network tests against local HTTP fixtures; nothing leaves 127.0.0.1.
|
||||
|
||||
use super::*;
|
||||
use crate::crypto::kem::{Identity, SUITE_ID};
|
||||
use crate::pipeline::Pool;
|
||||
use std::io::BufRead;
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// What a fixture handler sends back for a request.
|
||||
enum Reply {
|
||||
/// Status line text, headers, body; Content-Length is added.
|
||||
Full(&'static str, Vec<(String, String)>, Vec<u8>),
|
||||
/// Raw bytes, then the connection closes.
|
||||
Raw(Vec<u8>),
|
||||
/// Headers then a stall.
|
||||
Stall,
|
||||
}
|
||||
|
||||
struct Server {
|
||||
addr: String,
|
||||
}
|
||||
|
||||
impl Server {
|
||||
/// Serves `handler(path, user_agent)` on 127.0.0.1, one thread per
|
||||
/// connection, for the life of the test process.
|
||||
fn start(handler: impl Fn(&str, &str) -> Reply + Send + Sync + 'static) -> Server {
|
||||
let l = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let addr = l.local_addr().unwrap().to_string();
|
||||
let handler = Arc::new(handler);
|
||||
std::thread::spawn(move || {
|
||||
for c in l.incoming().flatten() {
|
||||
let h = Arc::clone(&handler);
|
||||
std::thread::spawn(move || serve(c, &*h));
|
||||
}
|
||||
});
|
||||
Server { addr }
|
||||
}
|
||||
|
||||
fn url(&self, p: &str) -> String {
|
||||
format!("http://{}{p}", self.addr)
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(mut c: TcpStream, h: &dyn Fn(&str, &str) -> Reply) {
|
||||
let mut r = std::io::BufReader::new(c.try_clone().unwrap());
|
||||
let mut line = String::new();
|
||||
if r.read_line(&mut line).is_err() {
|
||||
return;
|
||||
}
|
||||
let path = line.split(' ').nth(1).unwrap_or("/").to_string();
|
||||
let mut ua = String::new();
|
||||
loop {
|
||||
let mut l = String::new();
|
||||
if r.read_line(&mut l).unwrap_or(0) == 0 || l == "\r\n" {
|
||||
break;
|
||||
}
|
||||
if let Some(v) = l.strip_prefix("user-agent: ").or_else(|| l.strip_prefix("User-Agent: ")) {
|
||||
ua = v.trim().to_string();
|
||||
}
|
||||
}
|
||||
let out = match h(&path, &ua) {
|
||||
Reply::Full(status, headers, body) => {
|
||||
let mut o = format!("HTTP/1.1 {status}\r\nConnection: close\r\nContent-Length: {}\r\n", body.len());
|
||||
for (k, v) in headers {
|
||||
o.push_str(&format!("{k}: {v}\r\n"));
|
||||
}
|
||||
o.push_str("\r\n");
|
||||
[o.into_bytes(), body].concat()
|
||||
}
|
||||
Reply::Raw(b) => b,
|
||||
Reply::Stall => {
|
||||
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc");
|
||||
std::thread::sleep(Duration::from_secs(5));
|
||||
return;
|
||||
}
|
||||
};
|
||||
let _ = c.write_all(&out);
|
||||
}
|
||||
|
||||
fn ok(ct: &str, body: &str) -> Reply {
|
||||
let headers = if ct.is_empty() {
|
||||
vec![]
|
||||
} else {
|
||||
vec![("Content-Type".to_string(), ct.to_string())]
|
||||
};
|
||||
Reply::Full("200 OK", headers, body.as_bytes().to_vec())
|
||||
}
|
||||
|
||||
fn not_found() -> Reply {
|
||||
Reply::Full("404 Not Found", vec![], b"no".to_vec())
|
||||
}
|
||||
|
||||
fn redirect(to: &str) -> Reply {
|
||||
Reply::Full("302 Found", vec![("Location".into(), to.into())], vec![])
|
||||
}
|
||||
|
||||
struct Vault {
|
||||
dir: PathBuf,
|
||||
store: Store,
|
||||
}
|
||||
|
||||
impl Drop for Vault {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_dir_all(&self.dir);
|
||||
}
|
||||
}
|
||||
|
||||
fn vault() -> Vault {
|
||||
let r: [u8; 8] = crate::rng::os().array();
|
||||
let dir = std::env::temp_dir().join(format!("vault-net-{}-{}", std::process::id(), crate::store::hex(&r)));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let root = dir.join("store");
|
||||
let pool = Arc::new(Pool::new(2));
|
||||
Store::init(&root, 0, Arc::clone(&pool)).unwrap();
|
||||
let store = Store::open(&root, Identity::from_suite_seed(SUITE_ID, &[1; 32]).unwrap(), pool).unwrap();
|
||||
Vault { dir, store }
|
||||
}
|
||||
|
||||
impl Vault {
|
||||
fn get(&self, name: &str) -> Vec<u8> {
|
||||
let cat = self.store.catalogue().unwrap();
|
||||
let rec = cat
|
||||
.current(name)
|
||||
.unwrap_or_else(|| panic!("{name} not stored; have {:?}", cat.names().collect::<Vec<_>>()));
|
||||
let out = format!("out-{}", rec.id.hex());
|
||||
self.store
|
||||
.extract(rec, &crate::platform::fs::Dir::open(&self.dir).unwrap(), std::ffi::OsStr::new(&out))
|
||||
.unwrap();
|
||||
std::fs::read(self.dir.join(out)).unwrap()
|
||||
}
|
||||
|
||||
fn names(&self) -> Vec<String> {
|
||||
self.store.catalogue().unwrap().names().map(String::from).collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn mirror_with(v: &Vault, url: &str, limits: &Limits) -> MirrorOutcome {
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
mirror(&v.store, &mut cat, url, "m", limits, &mut |_, _| {}).unwrap()
|
||||
}
|
||||
|
||||
fn temp_leftovers(prefix: &str) -> usize {
|
||||
std::fs::read_dir(std::env::temp_dir())
|
||||
.unwrap()
|
||||
.flatten()
|
||||
.filter(|e| e.file_name().to_string_lossy().starts_with(prefix))
|
||||
.count()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_url_names_and_user_agent() {
|
||||
let srv = Server::start(|p, ua| {
|
||||
if ua != CHROME_UA {
|
||||
return Reply::Full("403 Forbidden", vec![], b"bad agent".to_vec());
|
||||
}
|
||||
match p {
|
||||
"/files/page.html" => Reply::Full(
|
||||
"200 OK",
|
||||
vec![("Last-Modified".into(), "Tue, 15 Nov 1994 08:12:31 GMT".into())],
|
||||
b"body".to_vec(),
|
||||
),
|
||||
"/old" => redirect("/files/new.txt"),
|
||||
"/files/new.txt" => ok("", "moved"),
|
||||
"/" => ok("", "root"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
let v = vault();
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
let mut reported = Vec::new();
|
||||
let lim = Limits::add_defaults();
|
||||
add_url(&v.store, &mut cat, &srv.url("/files/page.html"), "web", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
add_url(&v.store, &mut cat, &srv.url("/old"), "web", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
add_url(&v.store, &mut cat, &srv.url(""), "", &lim, &mut |n, r| {
|
||||
reported.push((n.to_string(), r))
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
reported,
|
||||
[
|
||||
("web/page.html".into(), false),
|
||||
("web/new.txt".into(), false),
|
||||
("127.0.0.1".into(), false)
|
||||
]
|
||||
);
|
||||
assert_eq!(v.get("web/page.html"), b"body");
|
||||
assert_eq!(v.get("web/new.txt"), b"moved", "named after the final URL");
|
||||
assert_eq!(
|
||||
v.store.catalogue().unwrap().current("web/page.html").unwrap().modified.secs,
|
||||
784887151
|
||||
);
|
||||
let e = add_url(&v.store, &mut cat, &srv.url("/missing"), "", &lim, &mut |_, _| {}).unwrap_err();
|
||||
assert!(e.msg.ends_with("404 Not Found"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn add_url_limits() {
|
||||
let gz = {
|
||||
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
|
||||
e.write_all(&vec![b'a'; 1 << 20]).unwrap();
|
||||
e.finish().unwrap()
|
||||
};
|
||||
let srv = Server::start(move |p, _| match p {
|
||||
"/stall" => Reply::Stall,
|
||||
"/long" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabcdefgh".to_vec()),
|
||||
"/short" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nabc".to_vec()),
|
||||
"/bomb" => Reply::Full("200 OK", vec![("Content-Encoding".into(), "gzip".into())], gz.clone()),
|
||||
"/loop" => redirect("/loop"),
|
||||
"/big" => ok("", &"x".repeat(5000)),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let mut cat = v.store.catalogue().unwrap();
|
||||
let before = temp_leftovers("vault-download-");
|
||||
let small = Limits {
|
||||
request: Duration::from_millis(500),
|
||||
max_bytes: 4096,
|
||||
..Limits::add_defaults()
|
||||
};
|
||||
let mut try_add = |p: &str| add_url(&v.store, &mut cat, &srv.url(p), "", &small, &mut |_, _| {});
|
||||
let e = try_add("/stall").unwrap_err();
|
||||
assert!(e.msg.contains("timeout") || e.msg.contains("timed out"), "{e}");
|
||||
// A server sending more than it declared is cut at its Content-Length.
|
||||
try_add("/long").unwrap();
|
||||
assert!(try_add("/short").is_err(), "a body shorter than declared is an error");
|
||||
let e = try_add("/bomb").unwrap_err();
|
||||
assert!(e.msg.contains("exceeds 4096 bytes"), "decoded bytes count: {e}");
|
||||
let e = try_add("/loop").unwrap_err();
|
||||
assert!(e.msg.contains("stopped after 10 redirects"), "{e}");
|
||||
assert!(try_add("/big").unwrap_err().msg.contains("exceeds"));
|
||||
assert_eq!(v.get("long"), b"abc");
|
||||
assert_eq!(v.names(), ["long"], "nothing else stored");
|
||||
assert!(temp_leftovers("vault-download-") <= before, "temporary download left behind");
|
||||
}
|
||||
|
||||
/// Go's TestMirror: links in HTML and CSS become relative, off-host links
|
||||
/// stay, everything lands under host/.
|
||||
#[test]
|
||||
fn mirror_like_go() {
|
||||
let srv = Arc::new(std::sync::OnceLock::<String>::new());
|
||||
let s2 = Arc::clone(&srv);
|
||||
let server = Server::start(move |p, ua| {
|
||||
if ua != CHROME_UA {
|
||||
return Reply::Full("403 Forbidden", vec![], vec![]);
|
||||
}
|
||||
let base = s2.get().unwrap();
|
||||
match p {
|
||||
"/" => ok(
|
||||
"text/html",
|
||||
&format!(r#"<a href="{base}/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>"#),
|
||||
),
|
||||
"/sub/p.html" => ok("text/html", r#"<a href="/">home</a> <img src="../img/a.png">"#),
|
||||
"/s.css" => ok("text/css", "body{background:url(/img/a.png)}"),
|
||||
"/img/a.png" => ok("", "png"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
srv.set(server.url("")).unwrap();
|
||||
let v = vault();
|
||||
let out = mirror_with(&v, &server.url(""), &Limits::mirror_defaults());
|
||||
assert!(matches!(out, MirrorOutcome::Complete { stored: 4 }));
|
||||
let h = &server.addr;
|
||||
for (name, want) in [
|
||||
(
|
||||
format!("m/{h}/index.html"),
|
||||
r#"<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>"#,
|
||||
),
|
||||
(
|
||||
format!("m/{h}/sub/p.html"),
|
||||
r#"<a href="../index.html">home</a> <img src="../img/a.png">"#,
|
||||
),
|
||||
(format!("m/{h}/s.css"), "body{background:url(img/a.png)}"),
|
||||
(format!("m/{h}/img/a.png"), "png"),
|
||||
] {
|
||||
assert_eq!(String::from_utf8(v.get(&name)).unwrap(), want, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_policies() {
|
||||
let srv = Arc::new(std::sync::OnceLock::<String>::new());
|
||||
let s2 = Arc::clone(&srv);
|
||||
let server = Server::start(move |p, _| {
|
||||
let other = s2.get().unwrap().replace("127.0.0.1", "localhost");
|
||||
match p {
|
||||
"/" => ok(
|
||||
"text/html",
|
||||
r#"<head><base href="/d/"></head><a href=a.html>a</a> <a href="/away">x</a> <a href="/a_b%253Dc">c</a> <a href="/a?b=c">q</a>
|
||||
<script>var s = '<a href="/never">';</script><!-- <a href="/never2"> --><div style="background:url('/i.png')"><style>p{background:url("/j.png")}</style>"#,
|
||||
),
|
||||
"/d/a.html" => ok("text/html", "<p>a</p>"),
|
||||
"/away" => redirect(&format!("{other}/elsewhere")),
|
||||
"/a_b%253Dc" => ok("", "literal"),
|
||||
"/a?b=c" => ok("", "query"),
|
||||
"/i.png" | "/j.png" => ok("", "img"),
|
||||
_ => not_found(),
|
||||
}
|
||||
});
|
||||
srv.set(server.url("")).unwrap();
|
||||
let v = vault();
|
||||
let out = mirror_with(&v, &server.url("/"), &Limits::mirror_defaults());
|
||||
// /away (off-host redirect) and one of the colliding names are skipped.
|
||||
match out {
|
||||
MirrorOutcome::Partial { skipped: 2, stored: 5 } => {}
|
||||
MirrorOutcome::Partial { stored, skipped } => panic!("stored {stored}, skipped {skipped}: {:?}", v.names()),
|
||||
MirrorOutcome::Complete { stored } => panic!("complete, stored {stored}"),
|
||||
}
|
||||
let h = &server.addr;
|
||||
let index = String::from_utf8(v.get(&format!("m/{h}/index.html"))).unwrap();
|
||||
assert!(index.contains(r#"<base href="index.html">"#), "{index}");
|
||||
assert!(index.contains("<a href=d/a.html>"), "base href honoured, unquoted kept: {index}");
|
||||
assert!(
|
||||
index.contains(r#"'<a href="/never">'"#) && index.contains(r#"<!-- <a href="/never2"> -->"#),
|
||||
"{index}"
|
||||
);
|
||||
assert!(index.contains("url('i.png')") && index.contains(r#"url("j.png")"#), "{index}");
|
||||
let names = v.names();
|
||||
assert!(names.contains(&format!("m/{h}/d/a.html")));
|
||||
assert!(!names.iter().any(|n| n.contains("never") || n.contains("elsewhere")), "{names:?}");
|
||||
let collided = [format!("m/{h}/a_b%3Dc")];
|
||||
assert_eq!(
|
||||
names.iter().filter(|n| collided.contains(n)).count(),
|
||||
1,
|
||||
"one of two URLs owns the name"
|
||||
);
|
||||
let kept = String::from_utf8(v.get(&collided[0])).unwrap();
|
||||
assert!(kept == "literal" || kept == "query");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mirror_budgets() {
|
||||
let server = Server::start(|p, _| match p {
|
||||
"/" => ok("text/html", r#"<a href="/1">1</a><a href="/2">2</a><a href="/3">3</a>"#),
|
||||
"/1" | "/2" | "/3" => ok("", &"z".repeat(1000)),
|
||||
_ => not_found(),
|
||||
});
|
||||
let v = vault();
|
||||
let pages = Limits {
|
||||
max_pages: 2,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
match mirror_with(&v, &server.url("/"), &pages) {
|
||||
MirrorOutcome::Partial { stored: 2, .. } => {}
|
||||
_ => panic!("page limit not applied"),
|
||||
}
|
||||
let v = vault();
|
||||
let bytes = Limits {
|
||||
max_bytes: 1500,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
match mirror_with(&v, &server.url("/"), &bytes) {
|
||||
MirrorOutcome::Partial { stored: 2, skipped: 2 } => {}
|
||||
MirrorOutcome::Partial { stored, skipped } => panic!("byte budget: stored {stored}, skipped {skipped}"),
|
||||
MirrorOutcome::Complete { .. } => panic!("byte budget not applied"),
|
||||
}
|
||||
let v = vault();
|
||||
let text = Limits {
|
||||
max_text: 10,
|
||||
..Limits::mirror_defaults()
|
||||
};
|
||||
assert!(matches!(
|
||||
mirror_with(&v, &server.url("/"), &text),
|
||||
MirrorOutcome::Partial { stored: 0, skipped: 1 }
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relative_matches_go() {
|
||||
for r in url::tests::vectors()["relative"].as_array().unwrap() {
|
||||
let r = r.as_array().unwrap();
|
||||
assert_eq!(
|
||||
relative(r[0].as_str().unwrap(), r[1].as_str().unwrap()),
|
||||
r[2].as_str().unwrap(),
|
||||
"{r:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_time_matches_go() {
|
||||
for (s, want) in url::tests::vectors()["http_time"].as_object().unwrap() {
|
||||
assert_eq!(parse_http_time(s), want.as_i64(), "{s:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn proxy_rules() {
|
||||
let u = |s: &str| Url::parse(s).unwrap();
|
||||
assert!(!use_proxy(&u("http://localhost:8080/"), ""));
|
||||
assert!(!use_proxy(&u("http://127.0.0.1/"), ""));
|
||||
assert!(!use_proxy(&u("http://[::1]/"), ""));
|
||||
assert!(use_proxy(&u("http://example.com/"), "other.com"));
|
||||
assert!(!use_proxy(&u("http://example.com/"), "example.com"));
|
||||
assert!(!use_proxy(&u("http://a.example.com/"), "example.com"));
|
||||
assert!(use_proxy(&u("http://example.com/"), ".example.com"));
|
||||
assert!(!use_proxy(&u("http://a.example.com/"), "*.example.com"));
|
||||
assert!(!use_proxy(&u("http://x.com/"), "*"));
|
||||
assert!(use_proxy(&u("http://x.com:81/"), "x.com:80"));
|
||||
assert!(!use_proxy(&u("http://x.com/"), "x.com:80"));
|
||||
assert!(!use_proxy(&u("https://10.1.2.3/"), "10.0.0.0/8"));
|
||||
assert!(use_proxy(&u("https://11.1.2.3/"), "10.0.0.0/8, 192.168.1.1"));
|
||||
assert!(!use_proxy(&u("https://192.168.1.1/"), "10.0.0.0/8, 192.168.1.1"));
|
||||
}
|
||||
+473
@@ -0,0 +1,473 @@
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
//! Just enough of Go's `net/url` to crawl and name pages exactly as the Go
|
||||
//! version did: `Parse`, `ResolveReference` (RFC 3986 with dot-segment
|
||||
//! removal), `String` (re-escaping like `EscapedPath`), `Hostname` and
|
||||
//! `QueryEscape`. `path` is percent-decoded; `raw_query` is kept as sent.
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Eq)]
|
||||
pub struct Url {
|
||||
pub scheme: String,
|
||||
pub opaque: String,
|
||||
/// Userinfo as written (still escaped), without the '@'.
|
||||
pub user: Option<String>,
|
||||
pub host: String,
|
||||
pub path: String,
|
||||
raw_path: String,
|
||||
pub force_query: bool,
|
||||
pub raw_query: String,
|
||||
pub fragment: String,
|
||||
raw_fragment: String,
|
||||
omit_host: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq)]
|
||||
enum Mode {
|
||||
Path,
|
||||
Host,
|
||||
QueryComponent,
|
||||
Fragment,
|
||||
}
|
||||
|
||||
fn should_escape(c: u8, mode: Mode) -> bool {
|
||||
if c.is_ascii_alphanumeric() {
|
||||
return false;
|
||||
}
|
||||
if mode == Mode::Host && b"!$&'()*+,;=:[]<>\"".contains(&c) {
|
||||
return false;
|
||||
}
|
||||
match c {
|
||||
b'-' | b'_' | b'.' | b'~' => return false,
|
||||
b'$' | b'&' | b'+' | b',' | b'/' | b':' | b';' | b'=' | b'?' | b'@' => {
|
||||
return match mode {
|
||||
Mode::Path => c == b'?',
|
||||
Mode::QueryComponent => true,
|
||||
Mode::Fragment => false,
|
||||
Mode::Host => true,
|
||||
};
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
if mode == Mode::Fragment && b"!()*".contains(&c) {
|
||||
return false;
|
||||
}
|
||||
true
|
||||
}
|
||||
|
||||
fn escape(s: &str, mode: Mode) -> String {
|
||||
let mut o = String::with_capacity(s.len());
|
||||
for &c in s.as_bytes() {
|
||||
if c == b' ' && mode == Mode::QueryComponent {
|
||||
o.push('+');
|
||||
} else if should_escape(c, mode) {
|
||||
o.push_str(&format!("%{c:02X}"));
|
||||
} else {
|
||||
o.push(c as char);
|
||||
}
|
||||
}
|
||||
o
|
||||
}
|
||||
|
||||
fn unhex(c: u8) -> Option<u8> {
|
||||
(c as char).to_digit(16).map(|d| d as u8)
|
||||
}
|
||||
|
||||
fn unescape(s: &str, mode: Mode) -> Result<String, String> {
|
||||
let b = s.as_bytes();
|
||||
let mut out = Vec::with_capacity(b.len());
|
||||
let mut i = 0;
|
||||
while i < b.len() {
|
||||
match b[i] {
|
||||
b'%' => {
|
||||
let (h, l) = match (b.get(i + 1).copied().and_then(unhex), b.get(i + 2).copied().and_then(unhex)) {
|
||||
(Some(h), Some(l)) => (h, l),
|
||||
_ => return Err(format!("invalid URL escape {:?}", &s[i..(i + 3).min(s.len())])),
|
||||
};
|
||||
if mode == Mode::Host && h < 8 && &s[i..i + 3] != "%25" {
|
||||
return Err(format!("invalid URL escape {:?}", &s[i..i + 3]));
|
||||
}
|
||||
out.push(h << 4 | l);
|
||||
i += 3;
|
||||
}
|
||||
b'+' => {
|
||||
out.push(if mode == Mode::QueryComponent { b' ' } else { b'+' });
|
||||
i += 1;
|
||||
}
|
||||
c => {
|
||||
if mode == Mode::Host && c < 0x80 && should_escape(c, mode) {
|
||||
return Err(format!("invalid character {:?} in host name", c as char));
|
||||
}
|
||||
out.push(c);
|
||||
i += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
// Go strings may hold any bytes; a path that decodes to invalid UTF-8
|
||||
// cannot become a vault name anyway, so it is refused here.
|
||||
String::from_utf8(out).map_err(|_| "invalid UTF-8 in URL".to_string())
|
||||
}
|
||||
|
||||
pub fn query_escape(s: &str) -> String {
|
||||
escape(s, Mode::QueryComponent)
|
||||
}
|
||||
|
||||
fn valid_encoded(s: &str, mode: Mode) -> bool {
|
||||
s.bytes()
|
||||
.all(|c| b"!$&'()*+,;=:@[]".contains(&c) || c == b'%' || !should_escape(c, mode))
|
||||
}
|
||||
|
||||
fn get_scheme(raw: &str) -> Result<(&str, &str), String> {
|
||||
for (i, c) in raw.bytes().enumerate() {
|
||||
match c {
|
||||
b'a'..=b'z' | b'A'..=b'Z' => {}
|
||||
b'0'..=b'9' | b'+' | b'-' | b'.' if i > 0 => {}
|
||||
b':' if i == 0 => return Err("missing protocol scheme".into()),
|
||||
b':' => return Ok((&raw[..i], &raw[i + 1..])),
|
||||
_ => return Ok(("", raw)),
|
||||
}
|
||||
}
|
||||
Ok(("", raw))
|
||||
}
|
||||
|
||||
fn valid_optional_port(p: &str) -> bool {
|
||||
p.is_empty() || (p.starts_with(':') && p[1..].bytes().all(|c| c.is_ascii_digit()))
|
||||
}
|
||||
|
||||
fn parse_host(host: &str) -> Result<String, String> {
|
||||
if let Some(rest) = host.strip_prefix('[') {
|
||||
let end = rest.find(']').ok_or("missing ']' in host")?;
|
||||
let port = &rest[end + 1..];
|
||||
if !valid_optional_port(port) {
|
||||
return Err(format!("invalid port {port:?} after host"));
|
||||
}
|
||||
} else if let Some(i) = host.rfind(':') {
|
||||
let port = &host[i..];
|
||||
if !valid_optional_port(port) {
|
||||
return Err(format!("invalid port {port:?} after host"));
|
||||
}
|
||||
}
|
||||
unescape(host, Mode::Host)
|
||||
}
|
||||
|
||||
impl Url {
|
||||
pub fn parse(raw: &str) -> Result<Url, String> {
|
||||
let (rest, frag) = match raw.find('#') {
|
||||
Some(i) => (&raw[..i], Some(&raw[i + 1..])),
|
||||
None => (raw, None),
|
||||
};
|
||||
let mut u = Url::parse_without_fragment(rest)?;
|
||||
if let Some(f) = frag {
|
||||
u.fragment = unescape(f, Mode::Fragment)?;
|
||||
u.raw_fragment = if escape(&u.fragment, Mode::Fragment) == f {
|
||||
String::new()
|
||||
} else {
|
||||
f.to_string()
|
||||
};
|
||||
}
|
||||
Ok(u)
|
||||
}
|
||||
|
||||
fn parse_without_fragment(raw: &str) -> Result<Url, String> {
|
||||
if raw.bytes().any(|c| c < 0x20 || c == 0x7f) {
|
||||
return Err("net/url: invalid control character in URL".into());
|
||||
}
|
||||
let mut u = Url::default();
|
||||
if raw.is_empty() {
|
||||
return Ok(u);
|
||||
}
|
||||
let (scheme, mut rest) = get_scheme(raw)?;
|
||||
u.scheme = scheme.to_ascii_lowercase();
|
||||
if rest.ends_with('?') && rest.matches('?').count() == 1 {
|
||||
u.force_query = true;
|
||||
rest = &rest[..rest.len() - 1];
|
||||
} else if let Some(i) = rest.find('?') {
|
||||
u.raw_query = rest[i + 1..].to_string();
|
||||
rest = &rest[..i];
|
||||
}
|
||||
if !rest.starts_with('/') {
|
||||
if !u.scheme.is_empty() {
|
||||
u.opaque = rest.to_string();
|
||||
return Ok(u);
|
||||
}
|
||||
let seg = rest.split('/').next().unwrap_or("");
|
||||
if seg.contains(':') {
|
||||
return Err("first path segment in URL cannot contain colon".into());
|
||||
}
|
||||
}
|
||||
if (!u.scheme.is_empty() || !rest.starts_with("///")) && rest.starts_with("//") {
|
||||
let auth_and_rest = &rest[2..];
|
||||
let (authority, r) = match auth_and_rest.find('/') {
|
||||
Some(i) => (&auth_and_rest[..i], &auth_and_rest[i..]),
|
||||
None => (auth_and_rest, ""),
|
||||
};
|
||||
rest = r;
|
||||
let host = match authority.rfind('@') {
|
||||
Some(i) => {
|
||||
let ui = &authority[..i];
|
||||
if !ui.bytes().all(|c| c.is_ascii_alphanumeric() || b"-._:~!$&'()*+,;=%@".contains(&c)) {
|
||||
return Err("net/url: invalid userinfo".into());
|
||||
}
|
||||
u.user = Some(ui.to_string());
|
||||
&authority[i + 1..]
|
||||
}
|
||||
None => authority,
|
||||
};
|
||||
u.host = parse_host(host)?;
|
||||
} else if !u.scheme.is_empty() && rest.starts_with('/') {
|
||||
u.omit_host = true;
|
||||
}
|
||||
if !rest.is_empty() {
|
||||
u.set_path(rest)?;
|
||||
}
|
||||
Ok(u)
|
||||
}
|
||||
|
||||
fn set_path(&mut self, p: &str) -> Result<(), String> {
|
||||
self.path = unescape(p, Mode::Path)?;
|
||||
self.raw_path = if escape(&self.path, Mode::Path) == p {
|
||||
String::new()
|
||||
} else {
|
||||
p.to_string()
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn escaped_path(&self) -> String {
|
||||
if !self.raw_path.is_empty()
|
||||
&& valid_encoded(&self.raw_path, Mode::Path)
|
||||
&& unescape(&self.raw_path, Mode::Path).ok().as_deref() == Some(&self.path)
|
||||
{
|
||||
return self.raw_path.clone();
|
||||
}
|
||||
if self.path == "*" {
|
||||
return "*".into();
|
||||
}
|
||||
escape(&self.path, Mode::Path)
|
||||
}
|
||||
|
||||
fn escaped_fragment(&self) -> String {
|
||||
if !self.raw_fragment.is_empty()
|
||||
&& valid_encoded(&self.raw_fragment, Mode::Fragment)
|
||||
&& unescape(&self.raw_fragment, Mode::Fragment).ok().as_deref() == Some(&self.fragment)
|
||||
{
|
||||
return self.raw_fragment.clone();
|
||||
}
|
||||
escape(&self.fragment, Mode::Fragment)
|
||||
}
|
||||
|
||||
/// `ref` resolved against `self`, as Go's `URL.Parse`.
|
||||
pub fn join(&self, reference: &str) -> Result<Url, String> {
|
||||
Ok(self.resolve(&Url::parse(reference)?))
|
||||
}
|
||||
|
||||
/// Go's `ResolveReference`.
|
||||
pub fn resolve(&self, r: &Url) -> Url {
|
||||
let mut u = r.clone();
|
||||
if r.scheme.is_empty() {
|
||||
u.scheme = self.scheme.clone();
|
||||
}
|
||||
if !r.scheme.is_empty() || !r.host.is_empty() || r.user.is_some() {
|
||||
let p = resolve_path(&r.escaped_path(), "");
|
||||
let _ = u.set_path(&p);
|
||||
return u;
|
||||
}
|
||||
if !r.opaque.is_empty() {
|
||||
u.user = None;
|
||||
u.host.clear();
|
||||
u.path.clear();
|
||||
return u;
|
||||
}
|
||||
if r.path.is_empty() && !r.force_query && r.raw_query.is_empty() {
|
||||
u.raw_query = self.raw_query.clone();
|
||||
if r.fragment.is_empty() {
|
||||
u.fragment = self.fragment.clone();
|
||||
u.raw_fragment = self.raw_fragment.clone();
|
||||
}
|
||||
}
|
||||
if r.path.is_empty() && !self.opaque.is_empty() {
|
||||
u.opaque = self.opaque.clone();
|
||||
u.user = None;
|
||||
u.host.clear();
|
||||
u.path.clear();
|
||||
return u;
|
||||
}
|
||||
u.host = self.host.clone();
|
||||
u.user = self.user.clone();
|
||||
let p = resolve_path(&self.escaped_path(), &r.escaped_path());
|
||||
let _ = u.set_path(&p);
|
||||
u
|
||||
}
|
||||
|
||||
pub fn clear_fragment(&mut self) {
|
||||
self.fragment.clear();
|
||||
self.raw_fragment.clear();
|
||||
}
|
||||
|
||||
/// The host without any port, IPv6 brackets removed.
|
||||
pub fn hostname(&self) -> &str {
|
||||
let h = &self.host;
|
||||
if let Some(rest) = h.strip_prefix('[') {
|
||||
return rest.find(']').map_or(rest, |i| &rest[..i]);
|
||||
}
|
||||
match h.rfind(':') {
|
||||
Some(i) if valid_optional_port(&h[i..]) => &h[..i],
|
||||
_ => h,
|
||||
}
|
||||
}
|
||||
|
||||
/// The port, or the scheme's default.
|
||||
pub fn port_or_default(&self) -> &str {
|
||||
let h = &self.host;
|
||||
let after = if h.starts_with('[') {
|
||||
h.find(']').map_or("", |i| &h[i + 1..])
|
||||
} else {
|
||||
h.rfind(':').map_or("", |i| &h[i..])
|
||||
};
|
||||
match after.strip_prefix(':') {
|
||||
Some(p) if !p.is_empty() => p,
|
||||
_ if self.scheme == "https" => "443",
|
||||
_ => "80",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Url {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let mut b = String::new();
|
||||
if !self.scheme.is_empty() {
|
||||
b.push_str(&self.scheme);
|
||||
b.push(':');
|
||||
}
|
||||
if !self.opaque.is_empty() {
|
||||
b.push_str(&self.opaque);
|
||||
} else {
|
||||
if (!self.scheme.is_empty() || !self.host.is_empty() || self.user.is_some())
|
||||
&& !(self.omit_host && self.host.is_empty() && self.user.is_none())
|
||||
{
|
||||
if !self.host.is_empty() || !self.path.is_empty() || self.user.is_some() {
|
||||
b.push_str("//");
|
||||
}
|
||||
if let Some(ui) = &self.user {
|
||||
b.push_str(ui);
|
||||
b.push('@');
|
||||
}
|
||||
b.push_str(&escape(&self.host, Mode::Host));
|
||||
}
|
||||
let p = self.escaped_path();
|
||||
if !p.is_empty() && !p.starts_with('/') && !self.host.is_empty() {
|
||||
b.push('/');
|
||||
}
|
||||
if b.is_empty() && p.split('/').next().unwrap_or("").contains(':') {
|
||||
b.push_str("./");
|
||||
}
|
||||
b.push_str(&p);
|
||||
}
|
||||
if self.force_query || !self.raw_query.is_empty() {
|
||||
b.push('?');
|
||||
b.push_str(&self.raw_query);
|
||||
}
|
||||
if !self.fragment.is_empty() {
|
||||
b.push('#');
|
||||
b.push_str(&self.escaped_fragment());
|
||||
}
|
||||
f.write_str(&b)
|
||||
}
|
||||
}
|
||||
|
||||
/// Go's `resolvePath`: merge and remove dot segments.
|
||||
fn resolve_path(base: &str, r: &str) -> String {
|
||||
let full = if r.is_empty() {
|
||||
base.to_string()
|
||||
} else if !r.starts_with('/') {
|
||||
let i = base.rfind('/').map_or(0, |i| i + 1);
|
||||
format!("{}{}", &base[..i], r)
|
||||
} else {
|
||||
r.to_string()
|
||||
};
|
||||
if full.is_empty() {
|
||||
return String::new();
|
||||
}
|
||||
let mut dst = String::from("/");
|
||||
let mut first = true;
|
||||
let mut last = "";
|
||||
for elem in full.split('/') {
|
||||
last = elem;
|
||||
if elem == "." {
|
||||
first = false;
|
||||
continue;
|
||||
}
|
||||
if elem == ".." {
|
||||
let s = dst[1..].to_string();
|
||||
dst = String::from("/");
|
||||
match s.rfind('/') {
|
||||
None => first = true,
|
||||
Some(i) => dst.push_str(&s[..i]),
|
||||
}
|
||||
} else {
|
||||
if !first {
|
||||
dst.push('/');
|
||||
}
|
||||
dst.push_str(elem);
|
||||
first = false;
|
||||
}
|
||||
}
|
||||
if last == "." || last == ".." {
|
||||
dst.push('/');
|
||||
}
|
||||
if dst.len() > 1 && dst.as_bytes()[1] == b'/' {
|
||||
dst.remove(0);
|
||||
}
|
||||
dst
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod tests {
|
||||
use super::*;
|
||||
use serde_json::Value;
|
||||
|
||||
pub(crate) fn vectors() -> Value {
|
||||
serde_json::from_str(include_str!("../../tests/vectors/net.json")).unwrap()
|
||||
}
|
||||
|
||||
fn check(v: &Value, got: Result<Url, String>, ctx: &str) {
|
||||
if v["err"].as_bool().unwrap() {
|
||||
assert!(got.is_err(), "{ctx}: accepted, Go refused: {:?}", got.map(|u| u.to_string()));
|
||||
return;
|
||||
}
|
||||
let u = got.unwrap_or_else(|e| panic!("{ctx}: refused ({e}), Go accepted"));
|
||||
let s = |k: &str| v[k].as_str().unwrap_or("").to_string();
|
||||
assert_eq!(u.to_string(), s("string"), "{ctx} String");
|
||||
assert_eq!(u.scheme, s("scheme"), "{ctx} Scheme");
|
||||
assert_eq!(u.host, s("host"), "{ctx} Host");
|
||||
assert_eq!(u.hostname(), s("hostname"), "{ctx} Hostname");
|
||||
assert_eq!(u.path, s("path"), "{ctx} Path");
|
||||
assert_eq!(u.raw_query, s("raw_query"), "{ctx} RawQuery");
|
||||
assert_eq!(u.fragment, s("fragment"), "{ctx} Fragment");
|
||||
assert_eq!(super::super::local_path(&u), s("local_path"), "{ctx} localPath");
|
||||
assert_eq!(super::super::key(&u), s("key"), "{ctx} key");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_matches_go() {
|
||||
for v in vectors()["parse"].as_array().unwrap() {
|
||||
let input = v["input"].as_str().unwrap();
|
||||
check(v, Url::parse(input), &format!("Parse({input:?})"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_matches_go() {
|
||||
for v in vectors()["resolve"].as_array().unwrap() {
|
||||
let (b, r) = (v["base"].as_str().unwrap(), v["ref"].as_str().unwrap());
|
||||
let base = Url::parse(b).unwrap();
|
||||
check(&v["result"], base.join(r), &format!("{b:?}.Parse({r:?})"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn query_escape_matches_go() {
|
||||
for (k, want) in vectors()["query_escape"].as_object().unwrap() {
|
||||
assert_eq!(query_escape(k), want.as_str().unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user