andy/vault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Network import: bounded add URL and mirror

Browse Source 
An explicit ureq client with platform TLS verification, connect/header/
request deadlines, manual bounded redirects, decoded-byte budgets and
Go-style proxy selection. A Go-compatible URL module checked against
oracle vectors, and a byte-preserving HTML tokenizer for link rewriting
(href/src/action/poster, <style> and style= url(), <base href>). Mirror
keeps to one host across redirects, bounds pages and bytes, refuses
local-name collisions and reports partial results.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
Andy Dixon
2026-10-08 10:34:28 +01:00
co-authored by Claude
parent 24ebede9bb
commit aedc605a82
7 changed files with 5326 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
oracle/gen-vectors.sh
+1
View File
@@ -10,4 +10,5 @@ cd "$(dirname "$0")/.."
out=$PWD/tests/vectors out=$PWD/tests/vectors
mkdir -p "$out" mkdir -p "$out"
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream) (cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream)
(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./)
echo "vectors written to $out" echo "vectors written to $out"
oracle/go/net_oracle_vectors_test.go
+100
View File
@@ -0,0 +1,100 @@
// SPDX-License-Identifier: GPL-3.0-or-later
// Test-only vector generator for the Rust port's URL handling and mirror
// link mapping; see oracle/gen-vectors.sh.
package main
import (
"encoding/json"
"net/http"
"net/url"
"os"
"path/filepath"
"testing"
)
func TestOracleNetVectors(t *testing.T) {
dir := os.Getenv("VAULT_ORACLE_OUT")
if dir == "" {
t.Skip("set VAULT_ORACLE_OUT to write vectors")
}
type parsed struct {
Input string `json:"input"`
Err bool `json:"err"`
String string `json:"string,omitempty"`
Scheme string `json:"scheme,omitempty"`
Host string `json:"host,omitempty"`
Hostname string `json:"hostname,omitempty"`
Path string `json:"path,omitempty"`
RawQuery string `json:"raw_query,omitempty"`
Fragment string `json:"fragment,omitempty"`
Local string `json:"local_path,omitempty"`
Key string `json:"key,omitempty"`
}
describe := func(in string, u *url.URL, err error) parsed {
if err != nil {
return parsed{Input: in, Err: true}
}
return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path,
RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)}
}
refs := []string{
"", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080",
"?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A",
"https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..",
"a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f",
"http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only",
}
bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"}
var parses []parsed
for _, r := range refs {
u, err := url.Parse(r)
parses = append(parses, describe(r, u, err))
}
type resolved struct {
Base string `json:"base"`
Ref string `json:"ref"`
Result parsed `json:"result"`
}
var resolves []resolved
for _, b := range bases {
bu, err := url.Parse(b)
if err != nil {
t.Fatal(err)
}
for _, r := range refs {
u, err := bu.Parse(r)
resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)})
}
}
rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"},
{"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}}
var relOut [][3]string
for _, p := range rels {
relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])})
}
escapes := map[string]string{}
for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} {
escapes[s] = url.QueryEscape(s)
}
times := map[string]any{}
for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} {
if tm, err := http.ParseTime(s); err == nil {
times[s] = tm.Unix()
} else {
times[s] = nil
}
}
b, _ := json.MarshalIndent(map[string]any{
"provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go",
"parse": parses,
"resolve": resolves,
"relative": relOut,
"query_escape": escapes,
"http_time": times,
}, "", " ")
if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil {
t.Fatal(err)
}
}
src/net/html.rs
+503
View File
@@ -0,0 +1,503 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! Link rewriting for mirrored HTML and CSS.
//!
//! HTML is scanned with a small tokenizer following the HTML5 tokenizer's
//! rules for what matters here: tag and attribute syntax (quoted, unquoted
//! and valueless attributes), comments, doctype and other `<!`/`<?`
//! markup, raw-text elements (script, style, xmp, iframe, noembed,
//! noframes), RCDATA (textarea, title) and plaintext. It records byte
//! spans, so rewriting splices new values in and leaves every other byte —
//! quote style included — untouched.
//!
//! Rewritten: the values of `href`, `src`, `action` and `poster` on start
//! tags, and CSS `url(...)` inside `<style>` elements and `style`
//! attributes. CSS files get the `url(...)` rewrite only, with the same
//! matching as the Go version's regular expression
//! `(?i)(url\(\s*["']?)([^"')\s]+)`. Character references in attribute
//! values are decoded before a URL is parsed: numeric ones and the named
//! `amp`, `lt`, `gt`, `quot`, `apos` and `nbsp`; other named references
//! are left as written.
/// Something the tokenizer found.
#[derive(Debug, PartialEq, Eq)]
pub enum Item {
/// An attribute value on a start tag: lower-case tag and attribute
/// names, the value's byte span, and whether it was quoted.
Attr {
tag: String,
name: String,
start: usize,
end: usize,
quoted: bool,
},
/// The raw text inside a `<style>` element.
Style { start: usize, end: usize },
}
fn is_ws(c: u8) -> bool {
matches!(c, b'\t' | b'\n' | b'\x0c' | b'\r' | b' ')
}
fn find(h: &[u8], from: usize, n: &[u8]) -> Option<usize> {
if from > h.len() {
return None;
}
h[from..].windows(n.len()).position(|w| w == n).map(|i| i + from)
}
/// Scans `b` and returns every attribute value and style element, in
/// document order.
pub fn tokenize(b: &[u8]) -> Vec<Item> {
let mut items = Vec::new();
let mut pos = 0;
while let Some(off) = b[pos..].iter().position(|&c| c == b'<') {
let i = pos + off;
let next = b.get(i + 1).copied();
pos = match next {
Some(b'!') => {
if b[i..].starts_with(b"<!--") {
let body = i + 4;
if b[body..].starts_with(b">") {
body + 1
} else if b[body..].starts_with(b"->") {
body + 2
} else {
find(b, body, b"-->").map_or(b.len(), |e| e + 3)
}
} else {
to_gt(b, i + 2)
}
}
Some(b'?') => to_gt(b, i + 2),
Some(b'/') => match b.get(i + 2) {
Some(c) if c.is_ascii_alphabetic() => {
let (_, after) = tag(b, i + 2, &mut Vec::new(), "");
after
}
Some(b'>') => i + 3,
Some(_) => to_gt(b, i + 2),
None => b.len(),
},
Some(c) if c.is_ascii_alphabetic() => {
let (name, after) = tag(b, i + 1, &mut items, "start");
match name.as_str() {
"script" | "style" | "xmp" | "iframe" | "noembed" | "noframes" | "textarea" | "title" => {
let end = raw_text_end(b, after, &name);
if name == "style" {
items.push(Item::Style { start: after, end });
}
end
}
"plaintext" => b.len(),
_ => after,
}
}
_ => i + 1,
};
if pos >= b.len() {
break;
}
}
items
}
fn to_gt(b: &[u8], from: usize) -> usize {
b[from.min(b.len())..]
.iter()
.position(|&c| c == b'>')
.map_or(b.len(), |e| from + e + 1)
}
/// Where raw text started at `from` ends: at `</name` followed by
/// whitespace, `/` or `>` (ASCII case-insensitive), or at end of input.
fn raw_text_end(b: &[u8], from: usize, name: &str) -> usize {
let mut p = from;
while let Some(i) = find(b, p, b"</") {
let n = i + 2;
let candidate = b.get(n..n + name.len());
if candidate.is_some_and(|c| c.eq_ignore_ascii_case(name.as_bytes())) {
match b.get(n + name.len()) {
Some(&c) if is_ws(c) || c == b'/' || c == b'>' => return i,
None => return i,
_ => {}
}
}
p = i + 2;
}
b.len()
}
/// Parses a tag whose name starts at `from`; records start-tag attributes
/// (when `kind` is "start") and returns the lower-case name and the index
/// just past the closing `>` (or the end of input).
fn tag(b: &[u8], from: usize, items: &mut Vec<Item>, kind: &str) -> (String, usize) {
let mut attrs = Vec::new();
let (name, end, closed) = tag_inner(b, from, &mut attrs);
// A tag cut off by the end of input is not a tag at all.
if closed && kind == "start" {
items.extend(attrs);
}
(name, end)
}
fn tag_inner(b: &[u8], from: usize, items: &mut Vec<Item>) -> (String, usize, bool) {
let mut p = from;
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' {
p += 1;
}
let name = String::from_utf8_lossy(&b[from..p]).to_ascii_lowercase();
loop {
while p < b.len() && (is_ws(b[p]) || b[p] == b'/') {
p += 1;
}
if p >= b.len() {
return (name, b.len(), false);
}
if b[p] == b'>' {
return (name, p + 1, true);
}
// Attribute name: the first character may be anything (even '='),
// then up to whitespace, '/', '>' or '='.
let ns = p;
p += 1;
while p < b.len() && !is_ws(b[p]) && b[p] != b'/' && b[p] != b'>' && b[p] != b'=' {
p += 1;
}
let attr = String::from_utf8_lossy(&b[ns..p]).to_ascii_lowercase();
let mut q = p;
while q < b.len() && is_ws(b[q]) {
q += 1;
}
if q >= b.len() || b[q] != b'=' {
p = q;
continue;
}
q += 1;
while q < b.len() && is_ws(b[q]) {
q += 1;
}
if q >= b.len() {
return (name, b.len(), false);
}
let (start, end, quoted, next) = match b[q] {
quote @ (b'"' | b'\'') => {
let s = q + 1;
match b[s..].iter().position(|&c| c == quote) {
Some(e) => (s, s + e, true, s + e + 1),
// An unterminated value runs to the end; the tag is
// incomplete, so nothing is recorded.
None => return (name, b.len(), false),
}
}
b'>' => (q, q, false, q),
_ => {
let mut e = q;
while e < b.len() && !is_ws(b[e]) && b[e] != b'>' {
e += 1;
}
(q, e, false, e)
}
};
items.push(Item::Attr {
tag: name.clone(),
name: attr,
start,
end,
quoted,
});
p = next;
}
}
/// Decodes character references (see the module documentation).
pub fn unescape(s: &str) -> String {
let mut out = String::with_capacity(s.len());
let mut rest = s;
while let Some(i) = rest.find('&') {
out.push_str(&rest[..i]);
rest = &rest[i..];
let (text, used) = reference(rest);
out.push_str(&text);
rest = &rest[used..];
}
out.push_str(rest);
out
}
/// One reference at the start of `s` (which begins with '&'): its text and
/// how many bytes it used; an unknown one is passed through as "&".
fn reference(s: &str) -> (String, usize) {
let b = s.as_bytes();
if b.get(1) == Some(&b'#') {
let (hex, start) = if matches!(b.get(2), Some(b'x' | b'X')) {
(true, 3)
} else {
(false, 2)
};
let mut e = start;
while e < b.len() && (if hex { b[e].is_ascii_hexdigit() } else { b[e].is_ascii_digit() }) {
e += 1;
}
if e == start {
return ("&".into(), 1);
}
let v = u32::from_str_radix(&s[start..e], if hex { 16 } else { 10 }).unwrap_or(0xfffd);
let c = char::from_u32(v).filter(|&c| c != '\0').unwrap_or('\u{fffd}');
let used = if b.get(e) == Some(&b';') { e + 1 } else { e };
return (c.to_string(), used);
}
for (name, text, legacy) in [
("amp", "&", true),
("lt", "<", true),
("gt", ">", true),
("quot", "\"", true),
("apos", "'", false),
("nbsp", "\u{a0}", true),
] {
if let Some(r) = s[1..].strip_prefix(name) {
if r.starts_with(';') {
return (text.into(), name.len() + 2);
}
if legacy {
return (text.into(), name.len() + 1);
}
}
}
("&".into(), 1)
}
/// Escapes a value for an attribute, as Go's `html.EscapeString`.
pub fn escape_attr(s: &str) -> String {
let mut o = String::with_capacity(s.len());
for c in s.chars() {
match c {
'&' => o.push_str("&amp;"),
'"' => o.push_str("&#34;"),
'\'' => o.push_str("&#39;"),
'<' => o.push_str("&lt;"),
'>' => o.push_str("&gt;"),
c => o.push(c),
}
}
o
}
/// Rewrites `url(...)` references in CSS. `map` gets each URL (HTML-
/// unescaped, as the Go version did) and returns its replacement, or None
/// to leave it; replacements are attribute-escaped when `in_attr`. Bytes
/// that are not UTF-8 pass through unchanged.
pub fn rewrite_css(b: &[u8], in_attr: bool, map: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
let mut out = Vec::with_capacity(b.len());
let mut last = 0;
let mut i = 0;
while i + 4 <= b.len() {
if !b[i..i + 4].eq_ignore_ascii_case(b"url(") {
i += 1;
continue;
}
let mut p = i + 4;
while p < b.len() && is_ws(b[p]) {
p += 1;
}
if p < b.len() && (b[p] == b'"' || b[p] == b'\'') {
p += 1;
} else if in_attr {
p += entity_quote(&b[p..]);
}
let start = p;
while p < b.len() && !is_ws(b[p]) && !matches!(b[p], b'"' | b'\'' | b')') && !(in_attr && entity_quote(&b[p..]) > 0) {
p += 1;
}
if p == start {
i += 1;
continue;
}
if let Some(r) = map(&unescape(&String::from_utf8_lossy(&b[start..p]))) {
out.extend_from_slice(&b[last..start]);
out.extend_from_slice(if in_attr { escape_attr(&r) } else { r }.as_bytes());
last = p;
}
i = p;
}
out.extend_from_slice(&b[last..]);
out
}
/// The length of a quote written as a character reference at the start of
/// `b` (inside an attribute value, CSS quotes arrive that way), or 0.
fn entity_quote(b: &[u8]) -> usize {
["&quot;", "&#34;", "&#39;", "&apos;", "&#x22;", "&#x27;"]
.iter()
.find(|q| b.starts_with(q.as_bytes()))
.map_or(0, |q| q.len())
}
/// Rewrites an HTML page. `base` is called with the first `<base href>`
/// value (decoded) before any link after it, and returns what that value
/// becomes; `link` maps a decoded link to its replacement, or None to
/// leave it. Every byte outside a replaced value is kept.
pub fn rewrite_html(page: &[u8], base: &mut dyn FnMut(&str) -> Option<String>, link: &mut dyn FnMut(&str) -> Option<String>) -> Vec<u8> {
let mut out = Vec::with_capacity(page.len());
let mut last = 0;
let mut seen_base = false;
for item in tokenize(page) {
let (start, end, replacement) = match item {
Item::Attr {
tag,
name,
start,
end,
quoted,
} => {
let raw = &page[start..end];
let value = || unescape(&String::from_utf8_lossy(raw));
let r = match name.as_str() {
"href" if tag == "base" && !seen_base => {
seen_base = true;
base(&value())
}
"href" | "src" | "action" | "poster" => link(&value()),
"style" => {
let r = rewrite_css(raw, true, link);
if r != raw {
out.extend_from_slice(&page[last..start]);
out.extend_from_slice(&r);
last = end;
}
continue;
}
_ => None,
};
let Some(r) = r else { continue };
let mut v = escape_attr(&r);
if !quoted && (v.is_empty() || v.bytes().any(|c| is_ws(c) || b"\"'=<>`".contains(&c))) {
v = format!("\"{v}\"");
}
(start, end, v.into_bytes())
}
Item::Style { start, end } => (start, end, rewrite_css(&page[start..end], false, link)),
};
out.extend_from_slice(&page[last..start]);
out.extend_from_slice(&replacement);
last = end;
}
out.extend_from_slice(&page[last..]);
out
}
#[cfg(test)]
mod tests {
use super::*;
fn upper(s: &str) -> Option<String> {
Some(s.to_uppercase())
}
fn rw(page: &str) -> String {
String::from_utf8(rewrite_html(page.as_bytes(), &mut |_| None, &mut upper)).unwrap()
}
#[test]
fn attributes_and_quotes_preserved() {
assert_eq!(
rw(r#"<a href="x">x</a> <link href='y'> <img src=z alt=q>"#),
r#"<a href="X">x</a> <link href='Y'> <img src=Z alt=q>"#
);
assert_eq!(rw(r#"<A HREF = "x" >"#), r#"<A HREF = "X" >"#);
assert_eq!(rw(r#"<form action="/p?a=1&amp;b=2">"#), r#"<form action="/P?A=1&amp;B=2">"#);
assert_eq!(
rw(r#"<video poster="p.jpg" data-src="d">"#),
r#"<video poster="P.JPG" data-src="d">"#
);
assert_eq!(rw("<p>href=\"x\"</p>"), "<p>href=\"x\"</p>", "text is not markup");
}
#[test]
fn comments_scripts_rcdata_untouched() {
let s = r#"<!-- <a href="c"> --><script>var a = '<a href="s">';</script><textarea><a href="t"></textarea><title><img src=u></title><a href="ok">"#;
assert_eq!(rw(s), s.replace("\"ok\"", "\"OK\""));
assert_eq!(rw("<!--><a href=a>"), "<!--><a href=A>");
assert_eq!(rw("<!DOCTYPE html><a href=a>"), "<!DOCTYPE html><a href=A>");
assert_eq!(rw("<script>x</SCRIPT ><a href=a>"), "<script>x</SCRIPT ><a href=A>");
assert_eq!(rw("<script>x</scripty><a href=a>"), "<script>x</scripty><a href=a>");
assert_eq!(rw("<plaintext><a href=a>"), "<plaintext><a href=a>");
}
#[test]
fn css_in_html_and_files() {
assert_eq!(
rw("<style>body{background:url( 'a.png' )}</style>"),
"<style>body{background:url( 'A.PNG' )}</style>"
);
assert_eq!(
rw(r#"<div style="background: URL(&quot;b.png&quot;)">"#),
r#"<div style="background: URL(&quot;B.PNG&quot;)">"#
);
let mut m = |s: &str| Some(format!("<{s}>"));
assert_eq!(
rewrite_css(b"a{b:url(x)} c{d:url()} url(\"y\")", false, &mut m),
b"a{b:url(<x>)} c{d:url()} url(\"<y>\")"
);
}
#[test]
fn base_href_and_escaping() {
let mut seen = Vec::new();
let out = rewrite_html(
br#"<base href="http://h/d/"><base href="x"><a href=a>"#,
&mut |b| {
seen.push(b.to_string());
Some("p.html".into())
},
&mut upper,
);
assert_eq!(seen, ["http://h/d/"]);
assert_eq!(out, br#"<base href="p.html"><base href="X"><a href=A>"#);
let out = rewrite_html(b"<a href=a>", &mut |_| None, &mut |_| Some("a b\"&".into()));
assert_eq!(out, br#"<a href="a b&#34;&amp;">"#);
// Non-UTF-8 bytes around and inside values survive.
let latin1 = b"<p>caf\xe9</p><a href=\"x\xe9\">";
assert_eq!(rewrite_html(latin1, &mut |_| None, &mut |_| None), latin1);
assert_eq!(unescape("a&amp;b&#47;&#x2F;&lt&unknown;&apos;&apos"), "a&b//<&unknown;'&apos");
}
#[test]
fn unterminated_input() {
for s in ["<a href=\"x", "<a href=", "<a", "<", "<!--", "<script>", "</a", "<a href='x' "] {
assert_eq!(rw(s), s, "{s:?}");
}
}
/// Random and mutated pages never panic, and an identity rewrite
/// returns the input byte for byte.
#[test]
fn fuzz_identity() {
let seeds = [
r#"<html><head><base href="/x/"><style>a{b:url(c)}</style></head><body><a href='d'>e</a><img src=f style="g:url(h)"><!-- i --><script>j</script></body></html>"#,
"<a href=\"x\"><b <c/> </d e='f'> <!DOCTYPE> <? pi ?> <textarea></textarea>",
];
let mut rng = crate::rng::os();
let alphabet = b"<>/!-='\" \tabcdefhrsxyzlptu()&;#";
for i in 0..20000 {
let mut page: Vec<u8> = seeds[i % 2].as_bytes().to_vec();
for _ in 0..rng.below(8) {
let at = rng.below(page.len() as u64 + 1) as usize;
match rng.below(3) {
0 => page.insert(at, alphabet[rng.below(alphabet.len() as u64) as usize]),
1 if at < page.len() => {
page.remove(at);
}
_ => page.truncate(at),
}
}
if i % 3 == 0 {
page.push(0xff);
}
assert_eq!(rewrite_html(&page, &mut |_| None, &mut |_| None), page);
// Rewriting everything must not panic either.
let _ = rewrite_html(&page, &mut |b| Some(b.into()), &mut |v| Some(format!("{v}&x")));
let _ = tokenize(&page);
}
}
}
src/net/mod.rs
+657 -1
View File
@@ -1,3 +1,659 @@
// SPDX-License-Identifier: GPL-3.0-or-later // SPDX-License-Identifier: GPL-3.0-or-later
//! Network import: `add URL` and `mirror`. (Being written.) //! Network import: `add URL` and `mirror`.
//!
//! Every request goes through one explicit client: TLS certificates are
//! always verified against the platform's roots, redirects are followed by
//! hand (http and https only, at most [`Limits::max_redirects`], and for
//! `mirror` only within the starting host), and every body is counted
//! after decompression against a byte budget — `Content-Length` is never
//! trusted. Proxies follow Go's `ProxyFromEnvironment`: `HTTPS_PROXY` for
//! https, `HTTP_PROXY` for http (ignored under CGI, when `REQUEST_METHOD`
//! is set), `NO_PROXY` exclusions, and never for localhost or loopback.
//!
//! Defaults (all overridable from the command line, the byte budget never
//! above the 1 TiB file limit):
//!
//! | limit | `add URL` | `mirror` |
//! |------------------------|-----------|-----------------|
//! | connect | 30 s | 30 s |
//! | response headers | 60 s | 60 s |
//! | whole request | 2 h | 5 min each |
//! | bytes | 16 GiB | 4 GiB in total |
//! | redirects per request | 10 | 10 |
//! | pages | — | 10 000 |
//! | HTML/CSS held to rewrite | — | 16 MiB per page |
//!
//! Downloads that are not rewritten are streamed to a private (0600)
//! temporary file, because the vault needs a file's size before storing
//! it; that file holds plaintext until it is removed, on every path, once
//! stored. HTML and CSS for rewriting are held in memory instead.
mod html;
mod url;
#[cfg(test)]
mod tests;
use crate::error::{Error, Result};
use crate::store::{Catalogue, FILE_LIMIT, Store, VaultName};
use crate::util::Instant;
use std::collections::{HashMap, HashSet, VecDeque};
use std::io::{Read, Write};
use std::path::PathBuf;
use std::time::Duration;
pub use url::Url;
pub const CHROME_UA: &str =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36";
#[derive(Clone, Debug)]
pub struct Limits {
pub connect: Duration,
/// Until the response headers have arrived.
pub response: Duration,
/// End to end, body included, for each request.
pub request: Duration,
/// Body bytes: per download for `add`, in total for `mirror`.
pub max_bytes: u64,
pub max_redirects: u32,
pub max_pages: u64,
/// Largest HTML or CSS document held in memory for rewriting.
pub max_text: u64,
}
impl Limits {
pub fn add_defaults() -> Limits {
Limits {
connect: Duration::from_secs(30),
response: Duration::from_secs(60),
request: Duration::from_secs(2 * 3600),
max_bytes: 16 << 30,
max_redirects: 10,
max_pages: 1,
max_text: 16 << 20,
}
}
pub fn mirror_defaults() -> Limits {
Limits {
request: Duration::from_secs(300),
max_bytes: 4 << 30,
max_pages: 10_000,
..Limits::add_defaults()
}
}
}
pub fn is_url(s: &str) -> bool {
s.starts_with("https://") || s.starts_with("http://")
}
/// Why a fetch failed: the byte budget ends a mirror, anything else only
/// skips one resource.
enum FetchError {
Budget(u64),
Other(String),
}
impl std::fmt::Display for FetchError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
FetchError::Budget(n) => write!(f, "download exceeds {n} bytes"),
FetchError::Other(m) => f.write_str(m),
}
}
}
impl From<String> for FetchError {
fn from(s: String) -> Self {
FetchError::Other(s)
}
}
type Response = ureq::http::Response<ureq::Body>;
struct Client {
limits: Limits,
agents: HashMap<Option<String>, ureq::Agent>,
}
impl Client {
fn new(limits: &Limits) -> Client {
Client {
limits: limits.clone(),
agents: HashMap::new(),
}
}
fn agent(&mut self, proxy: Option<String>) -> std::result::Result<&ureq::Agent, String> {
if !self.agents.contains_key(&proxy) {
let p = match &proxy {
Some(p) => Some(ureq::Proxy::new(p).map_err(|e| format!("invalid proxy {p:?}: {e}"))?),
None => None,
};
let tls = ureq::tls::TlsConfig::builder()
.root_certs(ureq::tls::RootCerts::PlatformVerifier)
.build();
let l = &self.limits;
let cfg = ureq::Agent::config_builder()
.tls_config(tls)
.proxy(p)
.max_redirects(0)
.http_status_as_error(false)
.user_agent(CHROME_UA)
.timeout_connect(Some(l.connect))
.timeout_recv_response(Some(l.response))
.timeout_global(Some(l.request))
.build();
self.agents.insert(proxy.clone(), cfg.into());
}
Ok(&self.agents[&proxy])
}
/// GETs `u`, following redirects; with `host`, every hop must stay on
/// it. Returns the final URL and its response, whatever the status.
fn get(&mut self, u: &Url, host: Option<&str>) -> std::result::Result<(Url, Response), String> {
let mut cur = u.clone();
for hop in 0..=self.limits.max_redirects {
if cur.scheme != "http" && cur.scheme != "https" {
return Err(format!("unsupported protocol scheme {:?}", cur.scheme));
}
let agent = self.agent(proxy_for(&cur))?;
let resp = agent.get(cur.to_string()).call().map_err(|e| e.to_string())?;
let code = resp.status().as_u16();
let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_string);
match (code, location) {
(301 | 302 | 303 | 307 | 308, Some(loc)) => {
if hop == self.limits.max_redirects {
return Err(format!("stopped after {} redirects", self.limits.max_redirects));
}
cur = cur.join(&loc).map_err(|e| format!("bad redirect location {loc:?}: {e}"))?;
if host.is_some_and(|h| cur.host.to_lowercase() != h) {
return Err(format!("redirected to another host: {cur}"));
}
}
_ => return Ok((cur, resp)),
}
}
unreachable!("the loop returns on its last hop")
}
}
fn status_error(resp: &Response) -> Option<String> {
let s = resp.status();
(s.as_u16() != 200).then(|| {
format!("{} {}", s.as_u16(), s.canonical_reason().unwrap_or(""))
.trim_end()
.to_string()
})
}
/// Copies at most `budget` decoded body bytes to `dst`; returns the count.
fn copy_body(resp: &mut Response, dst: &mut dyn Write, budget: u64) -> std::result::Result<u64, FetchError> {
let mut r = resp.body_mut().with_config().limit(u64::MAX).reader();
let mut buf = vec![0u8; 64 << 10];
let mut n = 0u64;
loop {
let k = match r.read(&mut buf) {
Ok(0) => return Ok(n),
Ok(k) => k,
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
Err(e) => return Err(FetchError::Other(crate::util::io_reason(&e))),
};
n += k as u64;
if n > budget {
return Err(FetchError::Budget(budget));
}
dst.write_all(&buf[..k])
.map_err(|e| FetchError::Other(crate::util::io_reason(&e)))?;
}
}
/// A private temporary file, removed when dropped.
struct TempFile {
file: Option<std::fs::File>,
path: PathBuf,
}
impl TempFile {
fn new(prefix: &str) -> Result<TempFile> {
let dir = std::env::temp_dir();
let (file, path) = crate::platform::fs::create_temp(&dir, prefix).map_err(|e| Error::io("create temporary file in", &dir, &e))?;
Ok(TempFile { file: Some(file), path })
}
}
impl Drop for TempFile {
fn drop(&mut self) {
drop(self.file.take());
let _ = std::fs::remove_file(&self.path);
}
}
fn vault_name(dest: &str, name: &str) -> Result<VaultName> {
let joined = crate::util::join(&[dest, name]);
VaultName::new(&joined).ok_or_else(|| Error::new(format!("invalid vault path {joined:?}")))
}
/// Downloads `url` and stores it beneath `dest`, named after the last
/// element of the final (post-redirect) URL's path, or its host name.
pub fn add_url(
store: &Store,
cat: &mut Catalogue,
url: &str,
dest: &str,
limits: &Limits,
report: &mut dyn FnMut(&VaultName, bool),
) -> Result<()> {
let u = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
let mut client = Client::new(limits);
let (fin, mut resp) = client.get(&u, None).map_err(|e| Error::new(format!("{url}: {e}")))?;
if let Some(e) = status_error(&resp) {
crate::bail!("{url}: {e}");
}
let mut name = crate::util::base(&fin.path);
if name == "/" || name == "." {
name = fin.hostname().to_string();
}
let vn = vault_name(dest, &name)?;
let mut tmp = TempFile::new("vault-download-")?;
let f = tmp.file.as_mut().expect("open");
copy_body(&mut resp, f, limits.max_bytes.min(FILE_LIMIT)).map_err(|e| Error::new(format!("{url}: {e}")))?;
if let Some(t) = resp
.headers()
.get("last-modified")
.and_then(|v| v.to_str().ok())
.and_then(parse_http_time)
{
let t = Instant { secs: t, nanos: 0 }.to_system();
let _ = f.set_times(std::fs::FileTimes::new().set_accessed(t).set_modified(t));
}
drop(tmp.file.take());
let (added, _) = store.add_file(cat, &tmp.path, &vn)?;
report(&vn, added.replaced);
Ok(())
}
pub enum MirrorOutcome {
Complete { stored: u64 },
Partial { stored: u64, skipped: u64 },
}
/// Where `u` lives inside a mirror, relative to its root (Go's localPath).
pub(crate) fn local_path(u: &Url) -> String {
let mut p = u.path.strip_prefix('/').unwrap_or(&u.path).to_string();
if p.is_empty() || p.ends_with('/') {
p.push_str("index.html");
}
if !u.raw_query.is_empty() {
p.push('_');
p.push_str(&url::query_escape(&u.raw_query));
}
p
}
/// A page's identity, independent of its fragment (Go's key).
pub(crate) fn key(u: &Url) -> String {
let mut c = u.clone();
c.clear_fragment();
c.to_string()
}
/// The link from the file `from` to the file `to`, both relative to the
/// same root (Go's relative).
pub(crate) fn relative(from: &str, to: &str) -> String {
let dir = crate::util::clean(from.rsplit_once('/').map_or(".", |(d, _)| if d.is_empty() { "/" } else { d }));
let mut f: Vec<&str> = dir.split('/').collect();
let mut t: Vec<&str> = to.split('/').collect();
if f[0] == "." {
f.clear();
}
while !f.is_empty() && t.len() > 1 && f[0] == t[0] {
f.remove(0);
t.remove(0);
}
format!("{}{}", "../".repeat(f.len()), t.join("/"))
}
struct Crawl<'a> {
store: &'a Store,
client: Client,
host: String,
root: String,
seen: HashSet<String>,
queue: VecDeque<Url>,
owners: HashMap<String, String>,
budget: u64,
max_pages: u64,
limit_hit: bool,
max_text: u64,
}
/// Crawls the site at `url`, one request at a time, staying on its host,
/// and stores every page and asset beneath `dest/HOST`. Links in HTML and
/// CSS that point into the mirror are rewritten to relative paths. A
/// resource that fails is reported and skipped.
pub fn mirror(
store: &Store,
cat: &mut Catalogue,
url: &str,
dest: &str,
limits: &Limits,
report: &mut dyn FnMut(&VaultName, bool),
) -> Result<MirrorOutcome> {
let mut start = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?;
if start.scheme != "http" && start.scheme != "https" {
crate::bail!("mirror needs an http:// or https:// URL");
}
if start.path.is_empty() && start.raw_query.is_empty() {
// The same page as a link to "/" would name.
start = start.join("/").expect("valid reference");
}
let host = start.host.to_lowercase();
let mut c = Crawl {
store,
client: Client::new(limits),
root: crate::util::join(&[dest, &host]),
host,
seen: HashSet::new(),
queue: VecDeque::new(),
owners: HashMap::new(),
budget: limits.max_bytes.min(FILE_LIMIT),
max_pages: limits.max_pages,
limit_hit: false,
max_text: limits.max_text,
};
c.enqueue(start);
let (mut stored, mut skipped) = (0u64, 0u64);
while let Some(u) = c.queue.pop_front() {
match c.one(cat, &u, report) {
Ok(()) => stored += 1,
Err(FetchError::Budget(n)) => {
eprintln!("skipping {u}: the mirror's {n}-byte budget is used up");
skipped += 1 + c.queue.len() as u64;
c.queue.clear();
}
Err(FetchError::Other(e)) => {
eprintln!("skipping {u}: {e}");
skipped += 1;
}
}
}
if c.limit_hit {
eprintln!("mirror: page limit of {} reached; further links were not followed", c.max_pages);
skipped += 1;
}
Ok(if skipped == 0 {
MirrorOutcome::Complete { stored }
} else {
MirrorOutcome::Partial { stored, skipped }
})
}
impl Crawl<'_> {
fn enqueue(&mut self, u: Url) {
let k = key(&u);
if self.seen.contains(&k) {
return;
}
if self.seen.len() as u64 >= self.max_pages {
self.limit_hit = true;
return;
}
self.seen.insert(k);
self.queue.push_back(u);
}
fn one(&mut self, cat: &mut Catalogue, u: &Url, report: &mut dyn FnMut(&VaultName, bool)) -> std::result::Result<(), FetchError> {
let local = local_path(u);
let k = key(u);
match self.owners.get(&local) {
Some(owner) if *owner != k => return Err(format!("its local name {local} is already used by {owner}").into()),
_ => {
self.owners.insert(local.clone(), k);
}
}
let name = vault_name(&self.root, &local).map_err(|e| e.msg)?;
let (fin, mut resp) = self.client.get(u, Some(&self.host))?;
if let Some(e) = status_error(&resp) {
return Err(e.into());
}
let ct = resp
.headers()
.get("content-type")
.and_then(|v| v.to_str().ok())
.unwrap_or("")
.to_lowercase();
let (is_html, is_css) = (ct.starts_with("text/html"), ct.starts_with("text/css"));
let added = if is_html || is_css {
let mut body = Vec::new();
let cap = self.budget.min(self.max_text);
let n = copy_body(&mut resp, &mut body, cap).map_err(|e| match e {
FetchError::Budget(_) if cap < self.budget => {
FetchError::Other(format!("larger than the {}-byte limit for pages to rewrite", self.max_text))
}
e => e,
})?;
self.budget -= n;
let body = self.rewrite(&body, is_html, &fin, &local);
self.store
.add_reader(cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name)
.map_err(|e| FetchError::Other(e.msg))?
} else {
let mut tmp = TempFile::new("vault-mirror-").map_err(|e| FetchError::Other(e.msg))?;
let n = copy_body(&mut resp, tmp.file.as_mut().expect("open"), self.budget)?;
self.budget -= n;
drop(tmp.file.take());
self.store.add_file(cat, &tmp.path, &name).map_err(|e| FetchError::Other(e.msg))?.0
};
report(&name, added.replaced);
Ok(())
}
/// Rewrites links that point into the mirror, queueing their targets.
fn rewrite(&mut self, body: &[u8], is_html: bool, page: &Url, local: &str) -> Vec<u8> {
let base = std::cell::RefCell::new(page.clone());
let found = std::cell::RefCell::new(Vec::new());
let host = self.host.clone();
let mut link = |v: &str| -> Option<String> {
let mut l = base.borrow().join(v).ok()?;
if (l.scheme != "http" && l.scheme != "https") || l.host.to_lowercase() != host {
return None;
}
let frag = l.fragment.clone();
l.clear_fragment();
if l.path.is_empty() && l.raw_query.is_empty() {
l = l.join("/").ok()?;
}
let mut r = relative(local, &local_path(&l));
found.borrow_mut().push(l);
if !frag.is_empty() {
r.push('#');
r.push_str(&frag);
}
Some(r)
};
let out = if is_html {
let mut on_base = |v: &str| -> Option<String> {
let b = base.borrow().join(v).ok()?;
let same = (b.scheme == "http" || b.scheme == "https") && b.host.to_lowercase() == host;
*base.borrow_mut() = b;
// Point the copy's base at the page itself, so the relative
// links written here resolve as intended.
same.then(|| relative(local, local))
};
html::rewrite_html(body, &mut on_base, &mut link)
} else {
html::rewrite_css(body, false, &mut link)
};
for l in found.into_inner() {
self.enqueue(l);
}
out
}
}
/// Go's `ProxyFromEnvironment`: the proxy URL for `u`, if any.
fn proxy_for(u: &Url) -> Option<String> {
let env = |names: &[&str]| names.iter().find_map(|n| std::env::var(n).ok().filter(|v| !v.is_empty()));
let proxy = if u.scheme == "https" {
env(&["HTTPS_PROXY", "https_proxy"])
} else if std::env::var_os("REQUEST_METHOD").is_some() {
None
} else {
env(&["HTTP_PROXY", "http_proxy"])
}?;
if !use_proxy(u, env(&["NO_PROXY", "no_proxy"]).as_deref().unwrap_or("")) {
return None;
}
Some(if proxy.contains("://") { proxy } else { format!("http://{proxy}") })
}
/// Whether `u` should go through a proxy given `no_proxy` (Go's
/// httpproxy semantics).
fn use_proxy(u: &Url, no_proxy: &str) -> bool {
let host = u.hostname().to_lowercase();
let port = u.port_or_default();
if host == "localhost" {
return false;
}
let ip: Option<std::net::IpAddr> = host.parse().ok();
if ip.is_some_and(|ip| ip.is_loopback()) {
return false;
}
for entry in no_proxy.split(',').map(|e| e.trim().to_lowercase()).filter(|e| !e.is_empty()) {
if entry == "*" {
return false;
}
if let Some((net, bits)) = entry.split_once('/') {
if let (Ok(net), Ok(bits), Some(ip)) = (net.parse::<std::net::IpAddr>(), bits.parse::<u32>(), ip)
&& in_cidr(ip, net, bits)
{
return false;
}
continue;
}
if let Ok(eip) = entry.trim_start_matches('[').trim_end_matches(']').parse::<std::net::IpAddr>() {
if ip == Some(eip) {
return false;
}
continue;
}
let (mut phost, pport) = match entry.rsplit_once(':') {
Some((h, p)) if !h.is_empty() && p.bytes().all(|c| c.is_ascii_digit()) => (h.to_string(), Some(p)),
_ => (entry.clone(), None),
};
if let Some(rest) = phost.strip_prefix("*.") {
phost = format!(".{rest}");
}
let match_host = !phost.starts_with('.');
if match_host {
phost.insert(0, '.');
}
if (host.ends_with(&phost) || (match_host && host == phost[1..])) && pport.is_none_or(|p| p == port) {
return false;
}
}
true
}
fn in_cidr(ip: std::net::IpAddr, net: std::net::IpAddr, bits: u32) -> bool {
let (a, b, width): (u128, u128, u32) = match (ip, net) {
(std::net::IpAddr::V4(a), std::net::IpAddr::V4(b)) => (u32::from(a) as u128, u32::from(b) as u128, 32),
(std::net::IpAddr::V6(a), std::net::IpAddr::V6(b)) => (u128::from(a), u128::from(b), 128),
_ => return false,
};
if bits > width {
return false;
}
let shift = width - bits;
shift >= width || (a >> shift) == (b >> shift)
}
/// Go's `http.ParseTime`: RFC 1123 with "GMT", RFC 850, or ANSI C, as Unix
/// seconds.
fn parse_http_time(s: &str) -> Option<i64> {
const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
const DAYS: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"];
const LONG: [&str; 7] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"];
let month = |m: &str| MONTHS.iter().position(|x| *x == m).map(|i| i as i64 + 1);
let num = |t: &str, n: usize| {
(t.len() == n && t.bytes().all(|c| c.is_ascii_digit()))
.then(|| t.parse::<i64>().ok())
.flatten()
};
let clock = |t: &str| -> Option<i64> {
let p: Vec<&str> = t.split(':').collect();
let (h, m, s) = (num(p.first()?, 2)?, num(p.get(1)?, 2)?, num(p.get(2)?, 2)?);
(p.len() == 3 && h < 24 && m < 60 && s < 60).then_some(h * 3600 + m * 60 + s)
};
let date = |y: i64, mo: i64, d: i64| -> Option<i64> {
let dim = [
31,
if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 { 29 } else { 28 },
31,
30,
31,
30,
31,
31,
30,
31,
30,
31,
];
(d >= 1 && d <= dim[(mo - 1) as usize]).then(|| days_from_civil(y, mo, d) * 86400)
};
// Mon, 02 Jan 2006 15:04:05 GMT
if let Some(rest) = s.get(..5).filter(|d| DAYS.contains(&&d[..3]) && &d[3..] == ", ").map(|_| &s[5..]) {
let p: Vec<&str> = rest.split(' ').collect();
if p.len() == 5 && p[4] == "GMT" {
return Some(date(num(p[2], 4)?, month(p[1])?, num(p[0], 2)?)? + clock(p[3])?);
}
return None;
}
// Monday, 02-Jan-06 15:04:05 MST
if let Some((day, rest)) = s.split_once(", ") {
if !LONG.contains(&day) {
return None;
}
let p: Vec<&str> = rest.split(' ').collect();
if p.len() != 3 || !(3..=5).contains(&p[2].len()) || !p[2].bytes().all(|c| c.is_ascii_uppercase()) {
return None;
}
let d: Vec<&str> = p[0].split('-').collect();
if d.len() != 3 {
return None;
}
let yy = num(d[2], 2)?;
return Some(date(if yy >= 69 { 1900 + yy } else { 2000 + yy }, month(d[1])?, num(d[0], 2)?)? + clock(p[1])?);
}
// Mon Jan _2 15:04:05 2006
let (head, rest) = (s.get(..8)?, s.get(8..)?);
if !DAYS.contains(&&head[..3]) || &head[3..4] != " " || &head[7..8] != " " {
return None;
}
let mo = month(&head[4..7])?;
let day = if let Some(r) = rest.strip_prefix(' ') {
(r, num(r.get(..1)?, 1)?)
} else {
(rest, num(rest.get(..2)?, 2)?)
};
let (r, d) = (day.0, day.1);
let after = &r[if rest.starts_with(' ') { 1 } else { 2 }..];
let p: Vec<&str> = after.strip_prefix(' ')?.split(' ').collect();
if p.len() != 2 {
return None;
}
Some(date(num(p[1], 4)?, mo, d)? + clock(p[0])?)
}
/// Days since 1970-01-01 of a proleptic Gregorian date.
fn days_from_civil(y: i64, m: i64, d: i64) -> i64 {
let y = if m <= 2 { y - 1 } else { y };
let era = y.div_euclid(400);
let yoe = y - era * 400;
let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1;
era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468
}
src/net/tests.rs
+412
View File
@@ -0,0 +1,412 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! Network tests against local HTTP fixtures; nothing leaves 127.0.0.1.
use super::*;
use crate::crypto::kem::{Identity, SUITE_ID};
use crate::pipeline::Pool;
use std::io::BufRead;
use std::net::{TcpListener, TcpStream};
use std::sync::Arc;
/// What a fixture handler sends back for a request.
enum Reply {
/// Status line text, headers, body; Content-Length is added.
Full(&'static str, Vec<(String, String)>, Vec<u8>),
/// Raw bytes, then the connection closes.
Raw(Vec<u8>),
/// Headers then a stall.
Stall,
}
struct Server {
addr: String,
}
impl Server {
/// Serves `handler(path, user_agent)` on 127.0.0.1, one thread per
/// connection, for the life of the test process.
fn start(handler: impl Fn(&str, &str) -> Reply + Send + Sync + 'static) -> Server {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let addr = l.local_addr().unwrap().to_string();
let handler = Arc::new(handler);
std::thread::spawn(move || {
for c in l.incoming().flatten() {
let h = Arc::clone(&handler);
std::thread::spawn(move || serve(c, &*h));
}
});
Server { addr }
}
fn url(&self, p: &str) -> String {
format!("http://{}{p}", self.addr)
}
}
fn serve(mut c: TcpStream, h: &dyn Fn(&str, &str) -> Reply) {
let mut r = std::io::BufReader::new(c.try_clone().unwrap());
let mut line = String::new();
if r.read_line(&mut line).is_err() {
return;
}
let path = line.split(' ').nth(1).unwrap_or("/").to_string();
let mut ua = String::new();
loop {
let mut l = String::new();
if r.read_line(&mut l).unwrap_or(0) == 0 || l == "\r\n" {
break;
}
if let Some(v) = l.strip_prefix("user-agent: ").or_else(|| l.strip_prefix("User-Agent: ")) {
ua = v.trim().to_string();
}
}
let out = match h(&path, &ua) {
Reply::Full(status, headers, body) => {
let mut o = format!("HTTP/1.1 {status}\r\nConnection: close\r\nContent-Length: {}\r\n", body.len());
for (k, v) in headers {
o.push_str(&format!("{k}: {v}\r\n"));
}
o.push_str("\r\n");
[o.into_bytes(), body].concat()
}
Reply::Raw(b) => b,
Reply::Stall => {
let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc");
std::thread::sleep(Duration::from_secs(5));
return;
}
};
let _ = c.write_all(&out);
}
fn ok(ct: &str, body: &str) -> Reply {
let headers = if ct.is_empty() {
vec![]
} else {
vec![("Content-Type".to_string(), ct.to_string())]
};
Reply::Full("200 OK", headers, body.as_bytes().to_vec())
}
fn not_found() -> Reply {
Reply::Full("404 Not Found", vec![], b"no".to_vec())
}
fn redirect(to: &str) -> Reply {
Reply::Full("302 Found", vec![("Location".into(), to.into())], vec![])
}
struct Vault {
dir: PathBuf,
store: Store,
}
impl Drop for Vault {
fn drop(&mut self) {
let _ = std::fs::remove_dir_all(&self.dir);
}
}
fn vault() -> Vault {
let r: [u8; 8] = crate::rng::os().array();
let dir = std::env::temp_dir().join(format!("vault-net-{}-{}", std::process::id(), crate::store::hex(&r)));
std::fs::create_dir_all(&dir).unwrap();
let root = dir.join("store");
let pool = Arc::new(Pool::new(2));
Store::init(&root, 0, Arc::clone(&pool)).unwrap();
let store = Store::open(&root, Identity::from_suite_seed(SUITE_ID, &[1; 32]).unwrap(), pool).unwrap();
Vault { dir, store }
}
impl Vault {
fn get(&self, name: &str) -> Vec<u8> {
let cat = self.store.catalogue().unwrap();
let rec = cat
.current(name)
.unwrap_or_else(|| panic!("{name} not stored; have {:?}", cat.names().collect::<Vec<_>>()));
let out = format!("out-{}", rec.id.hex());
self.store
.extract(rec, &crate::platform::fs::Dir::open(&self.dir).unwrap(), std::ffi::OsStr::new(&out))
.unwrap();
std::fs::read(self.dir.join(out)).unwrap()
}
fn names(&self) -> Vec<String> {
self.store.catalogue().unwrap().names().map(String::from).collect()
}
}
fn mirror_with(v: &Vault, url: &str, limits: &Limits) -> MirrorOutcome {
let mut cat = v.store.catalogue().unwrap();
mirror(&v.store, &mut cat, url, "m", limits, &mut |_, _| {}).unwrap()
}
fn temp_leftovers(prefix: &str) -> usize {
std::fs::read_dir(std::env::temp_dir())
.unwrap()
.flatten()
.filter(|e| e.file_name().to_string_lossy().starts_with(prefix))
.count()
}
#[test]
fn add_url_names_and_user_agent() {
let srv = Server::start(|p, ua| {
if ua != CHROME_UA {
return Reply::Full("403 Forbidden", vec![], b"bad agent".to_vec());
}
match p {
"/files/page.html" => Reply::Full(
"200 OK",
vec![("Last-Modified".into(), "Tue, 15 Nov 1994 08:12:31 GMT".into())],
b"body".to_vec(),
),
"/old" => redirect("/files/new.txt"),
"/files/new.txt" => ok("", "moved"),
"/" => ok("", "root"),
_ => not_found(),
}
});
let v = vault();
let mut cat = v.store.catalogue().unwrap();
let mut reported = Vec::new();
let lim = Limits::add_defaults();
add_url(&v.store, &mut cat, &srv.url("/files/page.html"), "web", &lim, &mut |n, r| {
reported.push((n.to_string(), r))
})
.unwrap();
add_url(&v.store, &mut cat, &srv.url("/old"), "web", &lim, &mut |n, r| {
reported.push((n.to_string(), r))
})
.unwrap();
add_url(&v.store, &mut cat, &srv.url(""), "", &lim, &mut |n, r| {
reported.push((n.to_string(), r))
})
.unwrap();
assert_eq!(
reported,
[
("web/page.html".into(), false),
("web/new.txt".into(), false),
("127.0.0.1".into(), false)
]
);
assert_eq!(v.get("web/page.html"), b"body");
assert_eq!(v.get("web/new.txt"), b"moved", "named after the final URL");
assert_eq!(
v.store.catalogue().unwrap().current("web/page.html").unwrap().modified.secs,
784887151
);
let e = add_url(&v.store, &mut cat, &srv.url("/missing"), "", &lim, &mut |_, _| {}).unwrap_err();
assert!(e.msg.ends_with("404 Not Found"), "{e}");
}
#[test]
fn add_url_limits() {
let gz = {
let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best());
e.write_all(&vec![b'a'; 1 << 20]).unwrap();
e.finish().unwrap()
};
let srv = Server::start(move |p, _| match p {
"/stall" => Reply::Stall,
"/long" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabcdefgh".to_vec()),
"/short" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nabc".to_vec()),
"/bomb" => Reply::Full("200 OK", vec![("Content-Encoding".into(), "gzip".into())], gz.clone()),
"/loop" => redirect("/loop"),
"/big" => ok("", &"x".repeat(5000)),
_ => not_found(),
});
let v = vault();
let mut cat = v.store.catalogue().unwrap();
let before = temp_leftovers("vault-download-");
let small = Limits {
request: Duration::from_millis(500),
max_bytes: 4096,
..Limits::add_defaults()
};
let mut try_add = |p: &str| add_url(&v.store, &mut cat, &srv.url(p), "", &small, &mut |_, _| {});
let e = try_add("/stall").unwrap_err();
assert!(e.msg.contains("timeout") || e.msg.contains("timed out"), "{e}");
// A server sending more than it declared is cut at its Content-Length.
try_add("/long").unwrap();
assert!(try_add("/short").is_err(), "a body shorter than declared is an error");
let e = try_add("/bomb").unwrap_err();
assert!(e.msg.contains("exceeds 4096 bytes"), "decoded bytes count: {e}");
let e = try_add("/loop").unwrap_err();
assert!(e.msg.contains("stopped after 10 redirects"), "{e}");
assert!(try_add("/big").unwrap_err().msg.contains("exceeds"));
assert_eq!(v.get("long"), b"abc");
assert_eq!(v.names(), ["long"], "nothing else stored");
assert!(temp_leftovers("vault-download-") <= before, "temporary download left behind");
}
/// Go's TestMirror: links in HTML and CSS become relative, off-host links
/// stay, everything lands under host/.
#[test]
fn mirror_like_go() {
let srv = Arc::new(std::sync::OnceLock::<String>::new());
let s2 = Arc::clone(&srv);
let server = Server::start(move |p, ua| {
if ua != CHROME_UA {
return Reply::Full("403 Forbidden", vec![], vec![]);
}
let base = s2.get().unwrap();
match p {
"/" => ok(
"text/html",
&format!(r#"<a href="{base}/sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='/s.css'>"#),
),
"/sub/p.html" => ok("text/html", r#"<a href="/">home</a> <img src="../img/a.png">"#),
"/s.css" => ok("text/css", "body{background:url(/img/a.png)}"),
"/img/a.png" => ok("", "png"),
_ => not_found(),
}
});
srv.set(server.url("")).unwrap();
let v = vault();
let out = mirror_with(&v, &server.url(""), &Limits::mirror_defaults());
assert!(matches!(out, MirrorOutcome::Complete { stored: 4 }));
let h = &server.addr;
for (name, want) in [
(
format!("m/{h}/index.html"),
r#"<a href="sub/p.html#x">p</a> <a href="https://elsewhere.example/">e</a> <link href='s.css'>"#,
),
(
format!("m/{h}/sub/p.html"),
r#"<a href="../index.html">home</a> <img src="../img/a.png">"#,
),
(format!("m/{h}/s.css"), "body{background:url(img/a.png)}"),
(format!("m/{h}/img/a.png"), "png"),
] {
assert_eq!(String::from_utf8(v.get(&name)).unwrap(), want, "{name}");
}
}
#[test]
fn mirror_policies() {
let srv = Arc::new(std::sync::OnceLock::<String>::new());
let s2 = Arc::clone(&srv);
let server = Server::start(move |p, _| {
let other = s2.get().unwrap().replace("127.0.0.1", "localhost");
match p {
"/" => ok(
"text/html",
r#"<head><base href="/d/"></head><a href=a.html>a</a> <a href="/away">x</a> <a href="/a_b%253Dc">c</a> <a href="/a?b=c">q</a>
<script>var s = '<a href="/never">';</script><!-- <a href="/never2"> --><div style="background:url('/i.png')"><style>p{background:url("/j.png")}</style>"#,
),
"/d/a.html" => ok("text/html", "<p>a</p>"),
"/away" => redirect(&format!("{other}/elsewhere")),
"/a_b%253Dc" => ok("", "literal"),
"/a?b=c" => ok("", "query"),
"/i.png" | "/j.png" => ok("", "img"),
_ => not_found(),
}
});
srv.set(server.url("")).unwrap();
let v = vault();
let out = mirror_with(&v, &server.url("/"), &Limits::mirror_defaults());
// /away (off-host redirect) and one of the colliding names are skipped.
match out {
MirrorOutcome::Partial { skipped: 2, stored: 5 } => {}
MirrorOutcome::Partial { stored, skipped } => panic!("stored {stored}, skipped {skipped}: {:?}", v.names()),
MirrorOutcome::Complete { stored } => panic!("complete, stored {stored}"),
}
let h = &server.addr;
let index = String::from_utf8(v.get(&format!("m/{h}/index.html"))).unwrap();
assert!(index.contains(r#"<base href="index.html">"#), "{index}");
assert!(index.contains("<a href=d/a.html>"), "base href honoured, unquoted kept: {index}");
assert!(
index.contains(r#"'<a href="/never">'"#) && index.contains(r#"<!-- <a href="/never2"> -->"#),
"{index}"
);
assert!(index.contains("url('i.png')") && index.contains(r#"url("j.png")"#), "{index}");
let names = v.names();
assert!(names.contains(&format!("m/{h}/d/a.html")));
assert!(!names.iter().any(|n| n.contains("never") || n.contains("elsewhere")), "{names:?}");
let collided = [format!("m/{h}/a_b%3Dc")];
assert_eq!(
names.iter().filter(|n| collided.contains(n)).count(),
1,
"one of two URLs owns the name"
);
let kept = String::from_utf8(v.get(&collided[0])).unwrap();
assert!(kept == "literal" || kept == "query");
}
#[test]
fn mirror_budgets() {
let server = Server::start(|p, _| match p {
"/" => ok("text/html", r#"<a href="/1">1</a><a href="/2">2</a><a href="/3">3</a>"#),
"/1" | "/2" | "/3" => ok("", &"z".repeat(1000)),
_ => not_found(),
});
let v = vault();
let pages = Limits {
max_pages: 2,
..Limits::mirror_defaults()
};
match mirror_with(&v, &server.url("/"), &pages) {
MirrorOutcome::Partial { stored: 2, .. } => {}
_ => panic!("page limit not applied"),
}
let v = vault();
let bytes = Limits {
max_bytes: 1500,
..Limits::mirror_defaults()
};
match mirror_with(&v, &server.url("/"), &bytes) {
MirrorOutcome::Partial { stored: 2, skipped: 2 } => {}
MirrorOutcome::Partial { stored, skipped } => panic!("byte budget: stored {stored}, skipped {skipped}"),
MirrorOutcome::Complete { .. } => panic!("byte budget not applied"),
}
let v = vault();
let text = Limits {
max_text: 10,
..Limits::mirror_defaults()
};
assert!(matches!(
mirror_with(&v, &server.url("/"), &text),
MirrorOutcome::Partial { stored: 0, skipped: 1 }
));
}
#[test]
fn relative_matches_go() {
for r in url::tests::vectors()["relative"].as_array().unwrap() {
let r = r.as_array().unwrap();
assert_eq!(
relative(r[0].as_str().unwrap(), r[1].as_str().unwrap()),
r[2].as_str().unwrap(),
"{r:?}"
);
}
}
#[test]
fn http_time_matches_go() {
for (s, want) in url::tests::vectors()["http_time"].as_object().unwrap() {
assert_eq!(parse_http_time(s), want.as_i64(), "{s:?}");
}
}
#[test]
fn proxy_rules() {
let u = |s: &str| Url::parse(s).unwrap();
assert!(!use_proxy(&u("http://localhost:8080/"), ""));
assert!(!use_proxy(&u("http://127.0.0.1/"), ""));
assert!(!use_proxy(&u("http://[::1]/"), ""));
assert!(use_proxy(&u("http://example.com/"), "other.com"));
assert!(!use_proxy(&u("http://example.com/"), "example.com"));
assert!(!use_proxy(&u("http://a.example.com/"), "example.com"));
assert!(use_proxy(&u("http://example.com/"), ".example.com"));
assert!(!use_proxy(&u("http://a.example.com/"), "*.example.com"));
assert!(!use_proxy(&u("http://x.com/"), "*"));
assert!(use_proxy(&u("http://x.com:81/"), "x.com:80"));
assert!(!use_proxy(&u("http://x.com/"), "x.com:80"));
assert!(!use_proxy(&u("https://10.1.2.3/"), "10.0.0.0/8"));
assert!(use_proxy(&u("https://11.1.2.3/"), "10.0.0.0/8, 192.168.1.1"));
assert!(!use_proxy(&u("https://192.168.1.1/"), "10.0.0.0/8, 192.168.1.1"));
}
src/net/url.rs
+473
View File
@@ -0,0 +1,473 @@
// SPDX-License-Identifier: GPL-3.0-or-later
//! Just enough of Go's `net/url` to crawl and name pages exactly as the Go
//! version did: `Parse`, `ResolveReference` (RFC 3986 with dot-segment
//! removal), `String` (re-escaping like `EscapedPath`), `Hostname` and
//! `QueryEscape`. `path` is percent-decoded; `raw_query` is kept as sent.
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Url {
pub scheme: String,
pub opaque: String,
/// Userinfo as written (still escaped), without the '@'.
pub user: Option<String>,
pub host: String,
pub path: String,
raw_path: String,
pub force_query: bool,
pub raw_query: String,
pub fragment: String,
raw_fragment: String,
omit_host: bool,
}
#[derive(Clone, Copy, PartialEq)]
enum Mode {
Path,
Host,
QueryComponent,
Fragment,
}
fn should_escape(c: u8, mode: Mode) -> bool {
if c.is_ascii_alphanumeric() {
return false;
}
if mode == Mode::Host && b"!$&'()*+,;=:[]<>\"".contains(&c) {
return false;
}
match c {
b'-' | b'_' | b'.' | b'~' => return false,
b'$' | b'&' | b'+' | b',' | b'/' | b':' | b';' | b'=' | b'?' | b'@' => {
return match mode {
Mode::Path => c == b'?',
Mode::QueryComponent => true,
Mode::Fragment => false,
Mode::Host => true,
};
}
_ => {}
}
if mode == Mode::Fragment && b"!()*".contains(&c) {
return false;
}
true
}
fn escape(s: &str, mode: Mode) -> String {
let mut o = String::with_capacity(s.len());
for &c in s.as_bytes() {
if c == b' ' && mode == Mode::QueryComponent {
o.push('+');
} else if should_escape(c, mode) {
o.push_str(&format!("%{c:02X}"));
} else {
o.push(c as char);
}
}
o
}
fn unhex(c: u8) -> Option<u8> {
(c as char).to_digit(16).map(|d| d as u8)
}
fn unescape(s: &str, mode: Mode) -> Result<String, String> {
let b = s.as_bytes();
let mut out = Vec::with_capacity(b.len());
let mut i = 0;
while i < b.len() {
match b[i] {
b'%' => {
let (h, l) = match (b.get(i + 1).copied().and_then(unhex), b.get(i + 2).copied().and_then(unhex)) {
(Some(h), Some(l)) => (h, l),
_ => return Err(format!("invalid URL escape {:?}", &s[i..(i + 3).min(s.len())])),
};
if mode == Mode::Host && h < 8 && &s[i..i + 3] != "%25" {
return Err(format!("invalid URL escape {:?}", &s[i..i + 3]));
}
out.push(h << 4 | l);
i += 3;
}
b'+' => {
out.push(if mode == Mode::QueryComponent { b' ' } else { b'+' });
i += 1;
}
c => {
if mode == Mode::Host && c < 0x80 && should_escape(c, mode) {
return Err(format!("invalid character {:?} in host name", c as char));
}
out.push(c);
i += 1;
}
}
}
// Go strings may hold any bytes; a path that decodes to invalid UTF-8
// cannot become a vault name anyway, so it is refused here.
String::from_utf8(out).map_err(|_| "invalid UTF-8 in URL".to_string())
}
pub fn query_escape(s: &str) -> String {
escape(s, Mode::QueryComponent)
}
fn valid_encoded(s: &str, mode: Mode) -> bool {
s.bytes()
.all(|c| b"!$&'()*+,;=:@[]".contains(&c) || c == b'%' || !should_escape(c, mode))
}
fn get_scheme(raw: &str) -> Result<(&str, &str), String> {
for (i, c) in raw.bytes().enumerate() {
match c {
b'a'..=b'z' | b'A'..=b'Z' => {}
b'0'..=b'9' | b'+' | b'-' | b'.' if i > 0 => {}
b':' if i == 0 => return Err("missing protocol scheme".into()),
b':' => return Ok((&raw[..i], &raw[i + 1..])),
_ => return Ok(("", raw)),
}
}
Ok(("", raw))
}
fn valid_optional_port(p: &str) -> bool {
p.is_empty() || (p.starts_with(':') && p[1..].bytes().all(|c| c.is_ascii_digit()))
}
fn parse_host(host: &str) -> Result<String, String> {
if let Some(rest) = host.strip_prefix('[') {
let end = rest.find(']').ok_or("missing ']' in host")?;
let port = &rest[end + 1..];
if !valid_optional_port(port) {
return Err(format!("invalid port {port:?} after host"));
}
} else if let Some(i) = host.rfind(':') {
let port = &host[i..];
if !valid_optional_port(port) {
return Err(format!("invalid port {port:?} after host"));
}
}
unescape(host, Mode::Host)
}
impl Url {
pub fn parse(raw: &str) -> Result<Url, String> {
let (rest, frag) = match raw.find('#') {
Some(i) => (&raw[..i], Some(&raw[i + 1..])),
None => (raw, None),
};
let mut u = Url::parse_without_fragment(rest)?;
if let Some(f) = frag {
u.fragment = unescape(f, Mode::Fragment)?;
u.raw_fragment = if escape(&u.fragment, Mode::Fragment) == f {
String::new()
} else {
f.to_string()
};
}
Ok(u)
}
fn parse_without_fragment(raw: &str) -> Result<Url, String> {
if raw.bytes().any(|c| c < 0x20 || c == 0x7f) {
return Err("net/url: invalid control character in URL".into());
}
let mut u = Url::default();
if raw.is_empty() {
return Ok(u);
}
let (scheme, mut rest) = get_scheme(raw)?;
u.scheme = scheme.to_ascii_lowercase();
if rest.ends_with('?') && rest.matches('?').count() == 1 {
u.force_query = true;
rest = &rest[..rest.len() - 1];
} else if let Some(i) = rest.find('?') {
u.raw_query = rest[i + 1..].to_string();
rest = &rest[..i];
}
if !rest.starts_with('/') {
if !u.scheme.is_empty() {
u.opaque = rest.to_string();
return Ok(u);
}
let seg = rest.split('/').next().unwrap_or("");
if seg.contains(':') {
return Err("first path segment in URL cannot contain colon".into());
}
}
if (!u.scheme.is_empty() || !rest.starts_with("///")) && rest.starts_with("//") {
let auth_and_rest = &rest[2..];
let (authority, r) = match auth_and_rest.find('/') {
Some(i) => (&auth_and_rest[..i], &auth_and_rest[i..]),
None => (auth_and_rest, ""),
};
rest = r;
let host = match authority.rfind('@') {
Some(i) => {
let ui = &authority[..i];
if !ui.bytes().all(|c| c.is_ascii_alphanumeric() || b"-._:~!$&'()*+,;=%@".contains(&c)) {
return Err("net/url: invalid userinfo".into());
}
u.user = Some(ui.to_string());
&authority[i + 1..]
}
None => authority,
};
u.host = parse_host(host)?;
} else if !u.scheme.is_empty() && rest.starts_with('/') {
u.omit_host = true;
}
if !rest.is_empty() {
u.set_path(rest)?;
}
Ok(u)
}
fn set_path(&mut self, p: &str) -> Result<(), String> {
self.path = unescape(p, Mode::Path)?;
self.raw_path = if escape(&self.path, Mode::Path) == p {
String::new()
} else {
p.to_string()
};
Ok(())
}
pub fn escaped_path(&self) -> String {
if !self.raw_path.is_empty()
&& valid_encoded(&self.raw_path, Mode::Path)
&& unescape(&self.raw_path, Mode::Path).ok().as_deref() == Some(&self.path)
{
return self.raw_path.clone();
}
if self.path == "*" {
return "*".into();
}
escape(&self.path, Mode::Path)
}
fn escaped_fragment(&self) -> String {
if !self.raw_fragment.is_empty()
&& valid_encoded(&self.raw_fragment, Mode::Fragment)
&& unescape(&self.raw_fragment, Mode::Fragment).ok().as_deref() == Some(&self.fragment)
{
return self.raw_fragment.clone();
}
escape(&self.fragment, Mode::Fragment)
}
/// `ref` resolved against `self`, as Go's `URL.Parse`.
pub fn join(&self, reference: &str) -> Result<Url, String> {
Ok(self.resolve(&Url::parse(reference)?))
}
/// Go's `ResolveReference`.
pub fn resolve(&self, r: &Url) -> Url {
let mut u = r.clone();
if r.scheme.is_empty() {
u.scheme = self.scheme.clone();
}
if !r.scheme.is_empty() || !r.host.is_empty() || r.user.is_some() {
let p = resolve_path(&r.escaped_path(), "");
let _ = u.set_path(&p);
return u;
}
if !r.opaque.is_empty() {
u.user = None;
u.host.clear();
u.path.clear();
return u;
}
if r.path.is_empty() && !r.force_query && r.raw_query.is_empty() {
u.raw_query = self.raw_query.clone();
if r.fragment.is_empty() {
u.fragment = self.fragment.clone();
u.raw_fragment = self.raw_fragment.clone();
}
}
if r.path.is_empty() && !self.opaque.is_empty() {
u.opaque = self.opaque.clone();
u.user = None;
u.host.clear();
u.path.clear();
return u;
}
u.host = self.host.clone();
u.user = self.user.clone();
let p = resolve_path(&self.escaped_path(), &r.escaped_path());
let _ = u.set_path(&p);
u
}
pub fn clear_fragment(&mut self) {
self.fragment.clear();
self.raw_fragment.clear();
}
/// The host without any port, IPv6 brackets removed.
pub fn hostname(&self) -> &str {
let h = &self.host;
if let Some(rest) = h.strip_prefix('[') {
return rest.find(']').map_or(rest, |i| &rest[..i]);
}
match h.rfind(':') {
Some(i) if valid_optional_port(&h[i..]) => &h[..i],
_ => h,
}
}
/// The port, or the scheme's default.
pub fn port_or_default(&self) -> &str {
let h = &self.host;
let after = if h.starts_with('[') {
h.find(']').map_or("", |i| &h[i + 1..])
} else {
h.rfind(':').map_or("", |i| &h[i..])
};
match after.strip_prefix(':') {
Some(p) if !p.is_empty() => p,
_ if self.scheme == "https" => "443",
_ => "80",
}
}
}
impl std::fmt::Display for Url {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let mut b = String::new();
if !self.scheme.is_empty() {
b.push_str(&self.scheme);
b.push(':');
}
if !self.opaque.is_empty() {
b.push_str(&self.opaque);
} else {
if (!self.scheme.is_empty() || !self.host.is_empty() || self.user.is_some())
&& !(self.omit_host && self.host.is_empty() && self.user.is_none())
{
if !self.host.is_empty() || !self.path.is_empty() || self.user.is_some() {
b.push_str("//");
}
if let Some(ui) = &self.user {
b.push_str(ui);
b.push('@');
}
b.push_str(&escape(&self.host, Mode::Host));
}
let p = self.escaped_path();
if !p.is_empty() && !p.starts_with('/') && !self.host.is_empty() {
b.push('/');
}
if b.is_empty() && p.split('/').next().unwrap_or("").contains(':') {
b.push_str("./");
}
b.push_str(&p);
}
if self.force_query || !self.raw_query.is_empty() {
b.push('?');
b.push_str(&self.raw_query);
}
if !self.fragment.is_empty() {
b.push('#');
b.push_str(&self.escaped_fragment());
}
f.write_str(&b)
}
}
/// Go's `resolvePath`: merge and remove dot segments.
fn resolve_path(base: &str, r: &str) -> String {
let full = if r.is_empty() {
base.to_string()
} else if !r.starts_with('/') {
let i = base.rfind('/').map_or(0, |i| i + 1);
format!("{}{}", &base[..i], r)
} else {
r.to_string()
};
if full.is_empty() {
return String::new();
}
let mut dst = String::from("/");
let mut first = true;
let mut last = "";
for elem in full.split('/') {
last = elem;
if elem == "." {
first = false;
continue;
}
if elem == ".." {
let s = dst[1..].to_string();
dst = String::from("/");
match s.rfind('/') {
None => first = true,
Some(i) => dst.push_str(&s[..i]),
}
} else {
if !first {
dst.push('/');
}
dst.push_str(elem);
first = false;
}
}
if last == "." || last == ".." {
dst.push('/');
}
if dst.len() > 1 && dst.as_bytes()[1] == b'/' {
dst.remove(0);
}
dst
}
#[cfg(test)]
pub(crate) mod tests {
use super::*;
use serde_json::Value;
pub(crate) fn vectors() -> Value {
serde_json::from_str(include_str!("../../tests/vectors/net.json")).unwrap()
}
fn check(v: &Value, got: Result<Url, String>, ctx: &str) {
if v["err"].as_bool().unwrap() {
assert!(got.is_err(), "{ctx}: accepted, Go refused: {:?}", got.map(|u| u.to_string()));
return;
}
let u = got.unwrap_or_else(|e| panic!("{ctx}: refused ({e}), Go accepted"));
let s = |k: &str| v[k].as_str().unwrap_or("").to_string();
assert_eq!(u.to_string(), s("string"), "{ctx} String");
assert_eq!(u.scheme, s("scheme"), "{ctx} Scheme");
assert_eq!(u.host, s("host"), "{ctx} Host");
assert_eq!(u.hostname(), s("hostname"), "{ctx} Hostname");
assert_eq!(u.path, s("path"), "{ctx} Path");
assert_eq!(u.raw_query, s("raw_query"), "{ctx} RawQuery");
assert_eq!(u.fragment, s("fragment"), "{ctx} Fragment");
assert_eq!(super::super::local_path(&u), s("local_path"), "{ctx} localPath");
assert_eq!(super::super::key(&u), s("key"), "{ctx} key");
}
#[test]
fn parse_matches_go() {
for v in vectors()["parse"].as_array().unwrap() {
let input = v["input"].as_str().unwrap();
check(v, Url::parse(input), &format!("Parse({input:?})"));
}
}
#[test]
fn resolve_matches_go() {
for v in vectors()["resolve"].as_array().unwrap() {
let (b, r) = (v["base"].as_str().unwrap(), v["ref"].as_str().unwrap());
let base = Url::parse(b).unwrap();
check(&v["result"], base.join(r), &format!("{b:?}.Parse({r:?})"));
}
}
#[test]
fn query_escape_matches_go() {
for (k, want) in vectors()["query_escape"].as_object().unwrap() {
assert_eq!(query_escape(k), want.as_str().unwrap());
}
}
}
tests/vectors/net.json
+3180
View File
File diff suppressed because it is too large Load Diff