diff --git a/oracle/gen-vectors.sh b/oracle/gen-vectors.sh index 44cd62b..c2e18cb 100755 --- a/oracle/gen-vectors.sh +++ b/oracle/gen-vectors.sh @@ -10,4 +10,5 @@ cd "$(dirname "$0")/.." out=$PWD/tests/vectors mkdir -p "$out" (cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleVectors$' ./ ./internal/pqcrypt ./internal/stream) +(cd oracle/go && VAULT_ORACLE_OUT=$out go test -count=1 -run '^TestOracleNetVectors$' ./) echo "vectors written to $out" diff --git a/oracle/go/net_oracle_vectors_test.go b/oracle/go/net_oracle_vectors_test.go new file mode 100644 index 0000000..ef06739 --- /dev/null +++ b/oracle/go/net_oracle_vectors_test.go @@ -0,0 +1,100 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +// Test-only vector generator for the Rust port's URL handling and mirror +// link mapping; see oracle/gen-vectors.sh. + +package main + +import ( + "encoding/json" + "net/http" + "net/url" + "os" + "path/filepath" + "testing" +) + +func TestOracleNetVectors(t *testing.T) { + dir := os.Getenv("VAULT_ORACLE_OUT") + if dir == "" { + t.Skip("set VAULT_ORACLE_OUT to write vectors") + } + type parsed struct { + Input string `json:"input"` + Err bool `json:"err"` + String string `json:"string,omitempty"` + Scheme string `json:"scheme,omitempty"` + Host string `json:"host,omitempty"` + Hostname string `json:"hostname,omitempty"` + Path string `json:"path,omitempty"` + RawQuery string `json:"raw_query,omitempty"` + Fragment string `json:"fragment,omitempty"` + Local string `json:"local_path,omitempty"` + Key string `json:"key,omitempty"` + } + describe := func(in string, u *url.URL, err error) parsed { + if err != nil { + return parsed{Input: in, Err: true} + } + return parsed{Input: in, String: u.String(), Scheme: u.Scheme, Host: u.Host, Hostname: u.Hostname(), Path: u.Path, + RawQuery: u.RawQuery, Fragment: u.Fragment, Local: localPath(u), Key: key(u)} + } + refs := []string{ + "", "a", "a/b", "./a", "../a", "../../../../a", "/a/./b/../c", "/", "//other.example/x", "//OTHER.example:8080", + "?q=1", "?", "#frag", "a#f", "a?x=1&y=2#f", "/sp ace", "/%41%2fb", "/%zz", "/caf%C3%A9", "/café", "http://H.Example:80/A", + "https://x/a/b/../../..", "HTTP://x", "mailto:a@b", "javascript:alert(1)", "ftp://x/y", "x:y", "a:b/c", "/a?b?c", "a/..", + "a/.", ".", "..", "/a//b", "https://user:pw@x/p", "http://[::1]:8080/v6", "http://x:port/", "http://x:/p", "/a%20b?c%20d#e%20f", + "http://x/\x01", "/a+b", "/q?a+b=c d", "#", "?#", "/p#a%23b", "//x/?", "http://x?only", + } + bases := []string{"http://h.example/dir/page.html?q=1#top", "https://h.example", "http://h.example:8080/a/b/", "http://h.example/a/b"} + var parses []parsed + for _, r := range refs { + u, err := url.Parse(r) + parses = append(parses, describe(r, u, err)) + } + type resolved struct { + Base string `json:"base"` + Ref string `json:"ref"` + Result parsed `json:"result"` + } + var resolves []resolved + for _, b := range bases { + bu, err := url.Parse(b) + if err != nil { + t.Fatal(err) + } + for _, r := range refs { + u, err := bu.Parse(r) + resolves = append(resolves, resolved{Base: b, Ref: r, Result: describe(r, u, err)}) + } + } + rels := [][2]string{{"index.html", "sub/p.html"}, {"sub/p.html", "index.html"}, {"sub/p.html", "img/a.png"}, {"a/b/c.html", "a/b/d.html"}, + {"a/b/c.html", "a/x/y/z.css"}, {"x.html", "x.html"}, {"a/b/", "a/c"}, {"s.css", "img/a.png"}, {"a/b/c", "a"}} + var relOut [][3]string + for _, p := range rels { + relOut = append(relOut, [3]string{p[0], p[1], relative(p[0], p[1])}) + } + escapes := map[string]string{} + for _, s := range []string{"a=1&b=2", "x y", "é/?#", "~-_.!*'()", "a+b"} { + escapes[s] = url.QueryEscape(s) + } + times := map[string]any{} + for _, s := range []string{"Mon, 02 Jan 2006 15:04:05 GMT", "Monday, 02-Jan-06 15:04:05 GMT", "Mon Jan 2 15:04:05 2006", "Tue, 15 Nov 1994 08:12:31 GMT", "bogus", "Mon, 02 Jan 2006 15:04:05 UTC", "Mon Jan 12 15:04:05 2006"} { + if tm, err := http.ParseTime(s); err == nil { + times[s] = tm.Unix() + } else { + times[s] = nil + } + } + b, _ := json.MarshalIndent(map[string]any{ + "provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go", + "parse": parses, + "resolve": resolves, + "relative": relOut, + "query_escape": escapes, + "http_time": times, + }, "", " ") + if err := os.WriteFile(filepath.Join(dir, "net.json"), append(b, '\n'), 0o644); err != nil { + t.Fatal(err) + } +} diff --git a/src/net/html.rs b/src/net/html.rs new file mode 100644 index 0000000..8362c97 --- /dev/null +++ b/src/net/html.rs @@ -0,0 +1,503 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! Link rewriting for mirrored HTML and CSS. +//! +//! HTML is scanned with a small tokenizer following the HTML5 tokenizer's +//! rules for what matters here: tag and attribute syntax (quoted, unquoted +//! and valueless attributes), comments, doctype and other `` elements and `style` +//! attributes. CSS files get the `url(...)` rewrite only, with the same +//! matching as the Go version's regular expression +//! `(?i)(url\(\s*["']?)([^"')\s]+)`. Character references in attribute +//! values are decoded before a URL is parsed: numeric ones and the named +//! `amp`, `lt`, `gt`, `quot`, `apos` and `nbsp`; other named references +//! are left as written. + +/// Something the tokenizer found. +#[derive(Debug, PartialEq, Eq)] +pub enum Item { + /// An attribute value on a start tag: lower-case tag and attribute + /// names, the value's byte span, and whether it was quoted. + Attr { + tag: String, + name: String, + start: usize, + end: usize, + quoted: bool, + }, + /// The raw text inside a `"), + "" + ); + assert_eq!( + rw(r#"
"#), + r#"
"# + ); + let mut m = |s: &str| Some(format!("<{s}>")); + assert_eq!( + rewrite_css(b"a{b:url(x)} c{d:url()} url(\"y\")", false, &mut m), + b"a{b:url()} c{d:url()} url(\"\")" + ); + } + + #[test] + fn base_href_and_escaping() { + let mut seen = Vec::new(); + let out = rewrite_html( + br#""#, + &mut |b| { + seen.push(b.to_string()); + Some("p.html".into()) + }, + &mut upper, + ); + assert_eq!(seen, ["http://h/d/"]); + assert_eq!(out, br#""#); + let out = rewrite_html(b"", &mut |_| None, &mut |_| Some("a b\"&".into())); + assert_eq!(out, br#""#); + // Non-UTF-8 bytes around and inside values survive. + let latin1 = b"

caf\xe9

"; + assert_eq!(rewrite_html(latin1, &mut |_| None, &mut |_| None), latin1); + assert_eq!(unescape("a&b//<&unknown;'&apos"), "a&b//<&unknown;'&apos"); + } + + #[test] + fn unterminated_input() { + for s in ["", "e"#, + " ", + ]; + let mut rng = crate::rng::os(); + let alphabet = b"<>/!-='\" \tabcdefhrsxyzlptu()&;#"; + for i in 0..20000 { + let mut page: Vec = seeds[i % 2].as_bytes().to_vec(); + for _ in 0..rng.below(8) { + let at = rng.below(page.len() as u64 + 1) as usize; + match rng.below(3) { + 0 => page.insert(at, alphabet[rng.below(alphabet.len() as u64) as usize]), + 1 if at < page.len() => { + page.remove(at); + } + _ => page.truncate(at), + } + } + if i % 3 == 0 { + page.push(0xff); + } + assert_eq!(rewrite_html(&page, &mut |_| None, &mut |_| None), page); + // Rewriting everything must not panic either. + let _ = rewrite_html(&page, &mut |b| Some(b.into()), &mut |v| Some(format!("{v}&x"))); + let _ = tokenize(&page); + } + } +} diff --git a/src/net/mod.rs b/src/net/mod.rs index 38369af..45b38b8 100644 --- a/src/net/mod.rs +++ b/src/net/mod.rs @@ -1,3 +1,659 @@ // SPDX-License-Identifier: GPL-3.0-or-later -//! Network import: `add URL` and `mirror`. (Being written.) +//! Network import: `add URL` and `mirror`. +//! +//! Every request goes through one explicit client: TLS certificates are +//! always verified against the platform's roots, redirects are followed by +//! hand (http and https only, at most [`Limits::max_redirects`], and for +//! `mirror` only within the starting host), and every body is counted +//! after decompression against a byte budget — `Content-Length` is never +//! trusted. Proxies follow Go's `ProxyFromEnvironment`: `HTTPS_PROXY` for +//! https, `HTTP_PROXY` for http (ignored under CGI, when `REQUEST_METHOD` +//! is set), `NO_PROXY` exclusions, and never for localhost or loopback. +//! +//! Defaults (all overridable from the command line, the byte budget never +//! above the 1 TiB file limit): +//! +//! | limit | `add URL` | `mirror` | +//! |------------------------|-----------|-----------------| +//! | connect | 30 s | 30 s | +//! | response headers | 60 s | 60 s | +//! | whole request | 2 h | 5 min each | +//! | bytes | 16 GiB | 4 GiB in total | +//! | redirects per request | 10 | 10 | +//! | pages | — | 10 000 | +//! | HTML/CSS held to rewrite | — | 16 MiB per page | +//! +//! Downloads that are not rewritten are streamed to a private (0600) +//! temporary file, because the vault needs a file's size before storing +//! it; that file holds plaintext until it is removed, on every path, once +//! stored. HTML and CSS for rewriting are held in memory instead. + +mod html; +mod url; + +#[cfg(test)] +mod tests; + +use crate::error::{Error, Result}; +use crate::store::{Catalogue, FILE_LIMIT, Store, VaultName}; +use crate::util::Instant; +use std::collections::{HashMap, HashSet, VecDeque}; +use std::io::{Read, Write}; +use std::path::PathBuf; +use std::time::Duration; +pub use url::Url; + +pub const CHROME_UA: &str = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0.0.0 Safari/537.36"; + +#[derive(Clone, Debug)] +pub struct Limits { + pub connect: Duration, + /// Until the response headers have arrived. + pub response: Duration, + /// End to end, body included, for each request. + pub request: Duration, + /// Body bytes: per download for `add`, in total for `mirror`. + pub max_bytes: u64, + pub max_redirects: u32, + pub max_pages: u64, + /// Largest HTML or CSS document held in memory for rewriting. + pub max_text: u64, +} + +impl Limits { + pub fn add_defaults() -> Limits { + Limits { + connect: Duration::from_secs(30), + response: Duration::from_secs(60), + request: Duration::from_secs(2 * 3600), + max_bytes: 16 << 30, + max_redirects: 10, + max_pages: 1, + max_text: 16 << 20, + } + } + + pub fn mirror_defaults() -> Limits { + Limits { + request: Duration::from_secs(300), + max_bytes: 4 << 30, + max_pages: 10_000, + ..Limits::add_defaults() + } + } +} + +pub fn is_url(s: &str) -> bool { + s.starts_with("https://") || s.starts_with("http://") +} + +/// Why a fetch failed: the byte budget ends a mirror, anything else only +/// skips one resource. +enum FetchError { + Budget(u64), + Other(String), +} + +impl std::fmt::Display for FetchError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + FetchError::Budget(n) => write!(f, "download exceeds {n} bytes"), + FetchError::Other(m) => f.write_str(m), + } + } +} + +impl From for FetchError { + fn from(s: String) -> Self { + FetchError::Other(s) + } +} + +type Response = ureq::http::Response; + +struct Client { + limits: Limits, + agents: HashMap, ureq::Agent>, +} + +impl Client { + fn new(limits: &Limits) -> Client { + Client { + limits: limits.clone(), + agents: HashMap::new(), + } + } + + fn agent(&mut self, proxy: Option) -> std::result::Result<&ureq::Agent, String> { + if !self.agents.contains_key(&proxy) { + let p = match &proxy { + Some(p) => Some(ureq::Proxy::new(p).map_err(|e| format!("invalid proxy {p:?}: {e}"))?), + None => None, + }; + let tls = ureq::tls::TlsConfig::builder() + .root_certs(ureq::tls::RootCerts::PlatformVerifier) + .build(); + let l = &self.limits; + let cfg = ureq::Agent::config_builder() + .tls_config(tls) + .proxy(p) + .max_redirects(0) + .http_status_as_error(false) + .user_agent(CHROME_UA) + .timeout_connect(Some(l.connect)) + .timeout_recv_response(Some(l.response)) + .timeout_global(Some(l.request)) + .build(); + self.agents.insert(proxy.clone(), cfg.into()); + } + Ok(&self.agents[&proxy]) + } + + /// GETs `u`, following redirects; with `host`, every hop must stay on + /// it. Returns the final URL and its response, whatever the status. + fn get(&mut self, u: &Url, host: Option<&str>) -> std::result::Result<(Url, Response), String> { + let mut cur = u.clone(); + for hop in 0..=self.limits.max_redirects { + if cur.scheme != "http" && cur.scheme != "https" { + return Err(format!("unsupported protocol scheme {:?}", cur.scheme)); + } + let agent = self.agent(proxy_for(&cur))?; + let resp = agent.get(cur.to_string()).call().map_err(|e| e.to_string())?; + let code = resp.status().as_u16(); + let location = resp.headers().get("location").and_then(|v| v.to_str().ok()).map(str::to_string); + match (code, location) { + (301 | 302 | 303 | 307 | 308, Some(loc)) => { + if hop == self.limits.max_redirects { + return Err(format!("stopped after {} redirects", self.limits.max_redirects)); + } + cur = cur.join(&loc).map_err(|e| format!("bad redirect location {loc:?}: {e}"))?; + if host.is_some_and(|h| cur.host.to_lowercase() != h) { + return Err(format!("redirected to another host: {cur}")); + } + } + _ => return Ok((cur, resp)), + } + } + unreachable!("the loop returns on its last hop") + } +} + +fn status_error(resp: &Response) -> Option { + let s = resp.status(); + (s.as_u16() != 200).then(|| { + format!("{} {}", s.as_u16(), s.canonical_reason().unwrap_or("")) + .trim_end() + .to_string() + }) +} + +/// Copies at most `budget` decoded body bytes to `dst`; returns the count. +fn copy_body(resp: &mut Response, dst: &mut dyn Write, budget: u64) -> std::result::Result { + let mut r = resp.body_mut().with_config().limit(u64::MAX).reader(); + let mut buf = vec![0u8; 64 << 10]; + let mut n = 0u64; + loop { + let k = match r.read(&mut buf) { + Ok(0) => return Ok(n), + Ok(k) => k, + Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue, + Err(e) => return Err(FetchError::Other(crate::util::io_reason(&e))), + }; + n += k as u64; + if n > budget { + return Err(FetchError::Budget(budget)); + } + dst.write_all(&buf[..k]) + .map_err(|e| FetchError::Other(crate::util::io_reason(&e)))?; + } +} + +/// A private temporary file, removed when dropped. +struct TempFile { + file: Option, + path: PathBuf, +} + +impl TempFile { + fn new(prefix: &str) -> Result { + let dir = std::env::temp_dir(); + let (file, path) = crate::platform::fs::create_temp(&dir, prefix).map_err(|e| Error::io("create temporary file in", &dir, &e))?; + Ok(TempFile { file: Some(file), path }) + } +} + +impl Drop for TempFile { + fn drop(&mut self) { + drop(self.file.take()); + let _ = std::fs::remove_file(&self.path); + } +} + +fn vault_name(dest: &str, name: &str) -> Result { + let joined = crate::util::join(&[dest, name]); + VaultName::new(&joined).ok_or_else(|| Error::new(format!("invalid vault path {joined:?}"))) +} + +/// Downloads `url` and stores it beneath `dest`, named after the last +/// element of the final (post-redirect) URL's path, or its host name. +pub fn add_url( + store: &Store, + cat: &mut Catalogue, + url: &str, + dest: &str, + limits: &Limits, + report: &mut dyn FnMut(&VaultName, bool), +) -> Result<()> { + let u = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?; + let mut client = Client::new(limits); + let (fin, mut resp) = client.get(&u, None).map_err(|e| Error::new(format!("{url}: {e}")))?; + if let Some(e) = status_error(&resp) { + crate::bail!("{url}: {e}"); + } + let mut name = crate::util::base(&fin.path); + if name == "/" || name == "." { + name = fin.hostname().to_string(); + } + let vn = vault_name(dest, &name)?; + let mut tmp = TempFile::new("vault-download-")?; + let f = tmp.file.as_mut().expect("open"); + copy_body(&mut resp, f, limits.max_bytes.min(FILE_LIMIT)).map_err(|e| Error::new(format!("{url}: {e}")))?; + if let Some(t) = resp + .headers() + .get("last-modified") + .and_then(|v| v.to_str().ok()) + .and_then(parse_http_time) + { + let t = Instant { secs: t, nanos: 0 }.to_system(); + let _ = f.set_times(std::fs::FileTimes::new().set_accessed(t).set_modified(t)); + } + drop(tmp.file.take()); + let (added, _) = store.add_file(cat, &tmp.path, &vn)?; + report(&vn, added.replaced); + Ok(()) +} + +pub enum MirrorOutcome { + Complete { stored: u64 }, + Partial { stored: u64, skipped: u64 }, +} + +/// Where `u` lives inside a mirror, relative to its root (Go's localPath). +pub(crate) fn local_path(u: &Url) -> String { + let mut p = u.path.strip_prefix('/').unwrap_or(&u.path).to_string(); + if p.is_empty() || p.ends_with('/') { + p.push_str("index.html"); + } + if !u.raw_query.is_empty() { + p.push('_'); + p.push_str(&url::query_escape(&u.raw_query)); + } + p +} + +/// A page's identity, independent of its fragment (Go's key). +pub(crate) fn key(u: &Url) -> String { + let mut c = u.clone(); + c.clear_fragment(); + c.to_string() +} + +/// The link from the file `from` to the file `to`, both relative to the +/// same root (Go's relative). +pub(crate) fn relative(from: &str, to: &str) -> String { + let dir = crate::util::clean(from.rsplit_once('/').map_or(".", |(d, _)| if d.is_empty() { "/" } else { d })); + let mut f: Vec<&str> = dir.split('/').collect(); + let mut t: Vec<&str> = to.split('/').collect(); + if f[0] == "." { + f.clear(); + } + while !f.is_empty() && t.len() > 1 && f[0] == t[0] { + f.remove(0); + t.remove(0); + } + format!("{}{}", "../".repeat(f.len()), t.join("/")) +} + +struct Crawl<'a> { + store: &'a Store, + client: Client, + host: String, + root: String, + seen: HashSet, + queue: VecDeque, + owners: HashMap, + budget: u64, + max_pages: u64, + limit_hit: bool, + max_text: u64, +} + +/// Crawls the site at `url`, one request at a time, staying on its host, +/// and stores every page and asset beneath `dest/HOST`. Links in HTML and +/// CSS that point into the mirror are rewritten to relative paths. A +/// resource that fails is reported and skipped. +pub fn mirror( + store: &Store, + cat: &mut Catalogue, + url: &str, + dest: &str, + limits: &Limits, + report: &mut dyn FnMut(&VaultName, bool), +) -> Result { + let mut start = Url::parse(url).map_err(|e| Error::new(format!("{url}: {e}")))?; + if start.scheme != "http" && start.scheme != "https" { + crate::bail!("mirror needs an http:// or https:// URL"); + } + if start.path.is_empty() && start.raw_query.is_empty() { + // The same page as a link to "/" would name. + start = start.join("/").expect("valid reference"); + } + let host = start.host.to_lowercase(); + let mut c = Crawl { + store, + client: Client::new(limits), + root: crate::util::join(&[dest, &host]), + host, + seen: HashSet::new(), + queue: VecDeque::new(), + owners: HashMap::new(), + budget: limits.max_bytes.min(FILE_LIMIT), + max_pages: limits.max_pages, + limit_hit: false, + max_text: limits.max_text, + }; + c.enqueue(start); + let (mut stored, mut skipped) = (0u64, 0u64); + while let Some(u) = c.queue.pop_front() { + match c.one(cat, &u, report) { + Ok(()) => stored += 1, + Err(FetchError::Budget(n)) => { + eprintln!("skipping {u}: the mirror's {n}-byte budget is used up"); + skipped += 1 + c.queue.len() as u64; + c.queue.clear(); + } + Err(FetchError::Other(e)) => { + eprintln!("skipping {u}: {e}"); + skipped += 1; + } + } + } + if c.limit_hit { + eprintln!("mirror: page limit of {} reached; further links were not followed", c.max_pages); + skipped += 1; + } + Ok(if skipped == 0 { + MirrorOutcome::Complete { stored } + } else { + MirrorOutcome::Partial { stored, skipped } + }) +} + +impl Crawl<'_> { + fn enqueue(&mut self, u: Url) { + let k = key(&u); + if self.seen.contains(&k) { + return; + } + if self.seen.len() as u64 >= self.max_pages { + self.limit_hit = true; + return; + } + self.seen.insert(k); + self.queue.push_back(u); + } + + fn one(&mut self, cat: &mut Catalogue, u: &Url, report: &mut dyn FnMut(&VaultName, bool)) -> std::result::Result<(), FetchError> { + let local = local_path(u); + let k = key(u); + match self.owners.get(&local) { + Some(owner) if *owner != k => return Err(format!("its local name {local} is already used by {owner}").into()), + _ => { + self.owners.insert(local.clone(), k); + } + } + let name = vault_name(&self.root, &local).map_err(|e| e.msg)?; + let (fin, mut resp) = self.client.get(u, Some(&self.host))?; + if let Some(e) = status_error(&resp) { + return Err(e.into()); + } + let ct = resp + .headers() + .get("content-type") + .and_then(|v| v.to_str().ok()) + .unwrap_or("") + .to_lowercase(); + let (is_html, is_css) = (ct.starts_with("text/html"), ct.starts_with("text/css")); + let added = if is_html || is_css { + let mut body = Vec::new(); + let cap = self.budget.min(self.max_text); + let n = copy_body(&mut resp, &mut body, cap).map_err(|e| match e { + FetchError::Budget(_) if cap < self.budget => { + FetchError::Other(format!("larger than the {}-byte limit for pages to rewrite", self.max_text)) + } + e => e, + })?; + self.budget -= n; + let body = self.rewrite(&body, is_html, &fin, &local); + self.store + .add_reader(cat, &mut &body[..], body.len() as u64, 0o600, Instant::now(), &name) + .map_err(|e| FetchError::Other(e.msg))? + } else { + let mut tmp = TempFile::new("vault-mirror-").map_err(|e| FetchError::Other(e.msg))?; + let n = copy_body(&mut resp, tmp.file.as_mut().expect("open"), self.budget)?; + self.budget -= n; + drop(tmp.file.take()); + self.store.add_file(cat, &tmp.path, &name).map_err(|e| FetchError::Other(e.msg))?.0 + }; + report(&name, added.replaced); + Ok(()) + } + + /// Rewrites links that point into the mirror, queueing their targets. + fn rewrite(&mut self, body: &[u8], is_html: bool, page: &Url, local: &str) -> Vec { + let base = std::cell::RefCell::new(page.clone()); + let found = std::cell::RefCell::new(Vec::new()); + let host = self.host.clone(); + let mut link = |v: &str| -> Option { + let mut l = base.borrow().join(v).ok()?; + if (l.scheme != "http" && l.scheme != "https") || l.host.to_lowercase() != host { + return None; + } + let frag = l.fragment.clone(); + l.clear_fragment(); + if l.path.is_empty() && l.raw_query.is_empty() { + l = l.join("/").ok()?; + } + let mut r = relative(local, &local_path(&l)); + found.borrow_mut().push(l); + if !frag.is_empty() { + r.push('#'); + r.push_str(&frag); + } + Some(r) + }; + let out = if is_html { + let mut on_base = |v: &str| -> Option { + let b = base.borrow().join(v).ok()?; + let same = (b.scheme == "http" || b.scheme == "https") && b.host.to_lowercase() == host; + *base.borrow_mut() = b; + // Point the copy's base at the page itself, so the relative + // links written here resolve as intended. + same.then(|| relative(local, local)) + }; + html::rewrite_html(body, &mut on_base, &mut link) + } else { + html::rewrite_css(body, false, &mut link) + }; + for l in found.into_inner() { + self.enqueue(l); + } + out + } +} + +/// Go's `ProxyFromEnvironment`: the proxy URL for `u`, if any. +fn proxy_for(u: &Url) -> Option { + let env = |names: &[&str]| names.iter().find_map(|n| std::env::var(n).ok().filter(|v| !v.is_empty())); + let proxy = if u.scheme == "https" { + env(&["HTTPS_PROXY", "https_proxy"]) + } else if std::env::var_os("REQUEST_METHOD").is_some() { + None + } else { + env(&["HTTP_PROXY", "http_proxy"]) + }?; + if !use_proxy(u, env(&["NO_PROXY", "no_proxy"]).as_deref().unwrap_or("")) { + return None; + } + Some(if proxy.contains("://") { proxy } else { format!("http://{proxy}") }) +} + +/// Whether `u` should go through a proxy given `no_proxy` (Go's +/// httpproxy semantics). +fn use_proxy(u: &Url, no_proxy: &str) -> bool { + let host = u.hostname().to_lowercase(); + let port = u.port_or_default(); + if host == "localhost" { + return false; + } + let ip: Option = host.parse().ok(); + if ip.is_some_and(|ip| ip.is_loopback()) { + return false; + } + for entry in no_proxy.split(',').map(|e| e.trim().to_lowercase()).filter(|e| !e.is_empty()) { + if entry == "*" { + return false; + } + if let Some((net, bits)) = entry.split_once('/') { + if let (Ok(net), Ok(bits), Some(ip)) = (net.parse::(), bits.parse::(), ip) + && in_cidr(ip, net, bits) + { + return false; + } + continue; + } + if let Ok(eip) = entry.trim_start_matches('[').trim_end_matches(']').parse::() { + if ip == Some(eip) { + return false; + } + continue; + } + let (mut phost, pport) = match entry.rsplit_once(':') { + Some((h, p)) if !h.is_empty() && p.bytes().all(|c| c.is_ascii_digit()) => (h.to_string(), Some(p)), + _ => (entry.clone(), None), + }; + if let Some(rest) = phost.strip_prefix("*.") { + phost = format!(".{rest}"); + } + let match_host = !phost.starts_with('.'); + if match_host { + phost.insert(0, '.'); + } + if (host.ends_with(&phost) || (match_host && host == phost[1..])) && pport.is_none_or(|p| p == port) { + return false; + } + } + true +} + +fn in_cidr(ip: std::net::IpAddr, net: std::net::IpAddr, bits: u32) -> bool { + let (a, b, width): (u128, u128, u32) = match (ip, net) { + (std::net::IpAddr::V4(a), std::net::IpAddr::V4(b)) => (u32::from(a) as u128, u32::from(b) as u128, 32), + (std::net::IpAddr::V6(a), std::net::IpAddr::V6(b)) => (u128::from(a), u128::from(b), 128), + _ => return false, + }; + if bits > width { + return false; + } + let shift = width - bits; + shift >= width || (a >> shift) == (b >> shift) +} + +/// Go's `http.ParseTime`: RFC 1123 with "GMT", RFC 850, or ANSI C, as Unix +/// seconds. +fn parse_http_time(s: &str) -> Option { + const MONTHS: [&str; 12] = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + const DAYS: [&str; 7] = ["Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun"]; + const LONG: [&str; 7] = ["Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday", "Sunday"]; + let month = |m: &str| MONTHS.iter().position(|x| *x == m).map(|i| i as i64 + 1); + let num = |t: &str, n: usize| { + (t.len() == n && t.bytes().all(|c| c.is_ascii_digit())) + .then(|| t.parse::().ok()) + .flatten() + }; + let clock = |t: &str| -> Option { + let p: Vec<&str> = t.split(':').collect(); + let (h, m, s) = (num(p.first()?, 2)?, num(p.get(1)?, 2)?, num(p.get(2)?, 2)?); + (p.len() == 3 && h < 24 && m < 60 && s < 60).then_some(h * 3600 + m * 60 + s) + }; + let date = |y: i64, mo: i64, d: i64| -> Option { + let dim = [ + 31, + if (y % 4 == 0 && y % 100 != 0) || y % 400 == 0 { 29 } else { 28 }, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31, + ]; + (d >= 1 && d <= dim[(mo - 1) as usize]).then(|| days_from_civil(y, mo, d) * 86400) + }; + // Mon, 02 Jan 2006 15:04:05 GMT + if let Some(rest) = s.get(..5).filter(|d| DAYS.contains(&&d[..3]) && &d[3..] == ", ").map(|_| &s[5..]) { + let p: Vec<&str> = rest.split(' ').collect(); + if p.len() == 5 && p[4] == "GMT" { + return Some(date(num(p[2], 4)?, month(p[1])?, num(p[0], 2)?)? + clock(p[3])?); + } + return None; + } + // Monday, 02-Jan-06 15:04:05 MST + if let Some((day, rest)) = s.split_once(", ") { + if !LONG.contains(&day) { + return None; + } + let p: Vec<&str> = rest.split(' ').collect(); + if p.len() != 3 || !(3..=5).contains(&p[2].len()) || !p[2].bytes().all(|c| c.is_ascii_uppercase()) { + return None; + } + let d: Vec<&str> = p[0].split('-').collect(); + if d.len() != 3 { + return None; + } + let yy = num(d[2], 2)?; + return Some(date(if yy >= 69 { 1900 + yy } else { 2000 + yy }, month(d[1])?, num(d[0], 2)?)? + clock(p[1])?); + } + // Mon Jan _2 15:04:05 2006 + let (head, rest) = (s.get(..8)?, s.get(8..)?); + if !DAYS.contains(&&head[..3]) || &head[3..4] != " " || &head[7..8] != " " { + return None; + } + let mo = month(&head[4..7])?; + let day = if let Some(r) = rest.strip_prefix(' ') { + (r, num(r.get(..1)?, 1)?) + } else { + (rest, num(rest.get(..2)?, 2)?) + }; + let (r, d) = (day.0, day.1); + let after = &r[if rest.starts_with(' ') { 1 } else { 2 }..]; + let p: Vec<&str> = after.strip_prefix(' ')?.split(' ').collect(); + if p.len() != 2 { + return None; + } + Some(date(num(p[1], 4)?, mo, d)? + clock(p[0])?) +} + +/// Days since 1970-01-01 of a proleptic Gregorian date. +fn days_from_civil(y: i64, m: i64, d: i64) -> i64 { + let y = if m <= 2 { y - 1 } else { y }; + let era = y.div_euclid(400); + let yoe = y - era * 400; + let doy = (153 * ((m + 9) % 12) + 2) / 5 + d - 1; + era * 146_097 + yoe * 365 + yoe / 4 - yoe / 100 + doy - 719_468 +} diff --git a/src/net/tests.rs b/src/net/tests.rs new file mode 100644 index 0000000..82b8f54 --- /dev/null +++ b/src/net/tests.rs @@ -0,0 +1,412 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! Network tests against local HTTP fixtures; nothing leaves 127.0.0.1. + +use super::*; +use crate::crypto::kem::{Identity, SUITE_ID}; +use crate::pipeline::Pool; +use std::io::BufRead; +use std::net::{TcpListener, TcpStream}; +use std::sync::Arc; + +/// What a fixture handler sends back for a request. +enum Reply { + /// Status line text, headers, body; Content-Length is added. + Full(&'static str, Vec<(String, String)>, Vec), + /// Raw bytes, then the connection closes. + Raw(Vec), + /// Headers then a stall. + Stall, +} + +struct Server { + addr: String, +} + +impl Server { + /// Serves `handler(path, user_agent)` on 127.0.0.1, one thread per + /// connection, for the life of the test process. + fn start(handler: impl Fn(&str, &str) -> Reply + Send + Sync + 'static) -> Server { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = l.local_addr().unwrap().to_string(); + let handler = Arc::new(handler); + std::thread::spawn(move || { + for c in l.incoming().flatten() { + let h = Arc::clone(&handler); + std::thread::spawn(move || serve(c, &*h)); + } + }); + Server { addr } + } + + fn url(&self, p: &str) -> String { + format!("http://{}{p}", self.addr) + } +} + +fn serve(mut c: TcpStream, h: &dyn Fn(&str, &str) -> Reply) { + let mut r = std::io::BufReader::new(c.try_clone().unwrap()); + let mut line = String::new(); + if r.read_line(&mut line).is_err() { + return; + } + let path = line.split(' ').nth(1).unwrap_or("/").to_string(); + let mut ua = String::new(); + loop { + let mut l = String::new(); + if r.read_line(&mut l).unwrap_or(0) == 0 || l == "\r\n" { + break; + } + if let Some(v) = l.strip_prefix("user-agent: ").or_else(|| l.strip_prefix("User-Agent: ")) { + ua = v.trim().to_string(); + } + } + let out = match h(&path, &ua) { + Reply::Full(status, headers, body) => { + let mut o = format!("HTTP/1.1 {status}\r\nConnection: close\r\nContent-Length: {}\r\n", body.len()); + for (k, v) in headers { + o.push_str(&format!("{k}: {v}\r\n")); + } + o.push_str("\r\n"); + [o.into_bytes(), body].concat() + } + Reply::Raw(b) => b, + Reply::Stall => { + let _ = c.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\n\r\nabc"); + std::thread::sleep(Duration::from_secs(5)); + return; + } + }; + let _ = c.write_all(&out); +} + +fn ok(ct: &str, body: &str) -> Reply { + let headers = if ct.is_empty() { + vec![] + } else { + vec![("Content-Type".to_string(), ct.to_string())] + }; + Reply::Full("200 OK", headers, body.as_bytes().to_vec()) +} + +fn not_found() -> Reply { + Reply::Full("404 Not Found", vec![], b"no".to_vec()) +} + +fn redirect(to: &str) -> Reply { + Reply::Full("302 Found", vec![("Location".into(), to.into())], vec![]) +} + +struct Vault { + dir: PathBuf, + store: Store, +} + +impl Drop for Vault { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.dir); + } +} + +fn vault() -> Vault { + let r: [u8; 8] = crate::rng::os().array(); + let dir = std::env::temp_dir().join(format!("vault-net-{}-{}", std::process::id(), crate::store::hex(&r))); + std::fs::create_dir_all(&dir).unwrap(); + let root = dir.join("store"); + let pool = Arc::new(Pool::new(2)); + Store::init(&root, 0, Arc::clone(&pool)).unwrap(); + let store = Store::open(&root, Identity::from_suite_seed(SUITE_ID, &[1; 32]).unwrap(), pool).unwrap(); + Vault { dir, store } +} + +impl Vault { + fn get(&self, name: &str) -> Vec { + let cat = self.store.catalogue().unwrap(); + let rec = cat + .current(name) + .unwrap_or_else(|| panic!("{name} not stored; have {:?}", cat.names().collect::>())); + let out = format!("out-{}", rec.id.hex()); + self.store + .extract(rec, &crate::platform::fs::Dir::open(&self.dir).unwrap(), std::ffi::OsStr::new(&out)) + .unwrap(); + std::fs::read(self.dir.join(out)).unwrap() + } + + fn names(&self) -> Vec { + self.store.catalogue().unwrap().names().map(String::from).collect() + } +} + +fn mirror_with(v: &Vault, url: &str, limits: &Limits) -> MirrorOutcome { + let mut cat = v.store.catalogue().unwrap(); + mirror(&v.store, &mut cat, url, "m", limits, &mut |_, _| {}).unwrap() +} + +fn temp_leftovers(prefix: &str) -> usize { + std::fs::read_dir(std::env::temp_dir()) + .unwrap() + .flatten() + .filter(|e| e.file_name().to_string_lossy().starts_with(prefix)) + .count() +} + +#[test] +fn add_url_names_and_user_agent() { + let srv = Server::start(|p, ua| { + if ua != CHROME_UA { + return Reply::Full("403 Forbidden", vec![], b"bad agent".to_vec()); + } + match p { + "/files/page.html" => Reply::Full( + "200 OK", + vec![("Last-Modified".into(), "Tue, 15 Nov 1994 08:12:31 GMT".into())], + b"body".to_vec(), + ), + "/old" => redirect("/files/new.txt"), + "/files/new.txt" => ok("", "moved"), + "/" => ok("", "root"), + _ => not_found(), + } + }); + let v = vault(); + let mut cat = v.store.catalogue().unwrap(); + let mut reported = Vec::new(); + let lim = Limits::add_defaults(); + add_url(&v.store, &mut cat, &srv.url("/files/page.html"), "web", &lim, &mut |n, r| { + reported.push((n.to_string(), r)) + }) + .unwrap(); + add_url(&v.store, &mut cat, &srv.url("/old"), "web", &lim, &mut |n, r| { + reported.push((n.to_string(), r)) + }) + .unwrap(); + add_url(&v.store, &mut cat, &srv.url(""), "", &lim, &mut |n, r| { + reported.push((n.to_string(), r)) + }) + .unwrap(); + assert_eq!( + reported, + [ + ("web/page.html".into(), false), + ("web/new.txt".into(), false), + ("127.0.0.1".into(), false) + ] + ); + assert_eq!(v.get("web/page.html"), b"body"); + assert_eq!(v.get("web/new.txt"), b"moved", "named after the final URL"); + assert_eq!( + v.store.catalogue().unwrap().current("web/page.html").unwrap().modified.secs, + 784887151 + ); + let e = add_url(&v.store, &mut cat, &srv.url("/missing"), "", &lim, &mut |_, _| {}).unwrap_err(); + assert!(e.msg.ends_with("404 Not Found"), "{e}"); +} + +#[test] +fn add_url_limits() { + let gz = { + let mut e = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::best()); + e.write_all(&vec![b'a'; 1 << 20]).unwrap(); + e.finish().unwrap() + }; + let srv = Server::start(move |p, _| match p { + "/stall" => Reply::Stall, + "/long" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 3\r\nConnection: close\r\n\r\nabcdefgh".to_vec()), + "/short" => Reply::Raw(b"HTTP/1.1 200 OK\r\nContent-Length: 100\r\nConnection: close\r\n\r\nabc".to_vec()), + "/bomb" => Reply::Full("200 OK", vec![("Content-Encoding".into(), "gzip".into())], gz.clone()), + "/loop" => redirect("/loop"), + "/big" => ok("", &"x".repeat(5000)), + _ => not_found(), + }); + let v = vault(); + let mut cat = v.store.catalogue().unwrap(); + let before = temp_leftovers("vault-download-"); + let small = Limits { + request: Duration::from_millis(500), + max_bytes: 4096, + ..Limits::add_defaults() + }; + let mut try_add = |p: &str| add_url(&v.store, &mut cat, &srv.url(p), "", &small, &mut |_, _| {}); + let e = try_add("/stall").unwrap_err(); + assert!(e.msg.contains("timeout") || e.msg.contains("timed out"), "{e}"); + // A server sending more than it declared is cut at its Content-Length. + try_add("/long").unwrap(); + assert!(try_add("/short").is_err(), "a body shorter than declared is an error"); + let e = try_add("/bomb").unwrap_err(); + assert!(e.msg.contains("exceeds 4096 bytes"), "decoded bytes count: {e}"); + let e = try_add("/loop").unwrap_err(); + assert!(e.msg.contains("stopped after 10 redirects"), "{e}"); + assert!(try_add("/big").unwrap_err().msg.contains("exceeds")); + assert_eq!(v.get("long"), b"abc"); + assert_eq!(v.names(), ["long"], "nothing else stored"); + assert!(temp_leftovers("vault-download-") <= before, "temporary download left behind"); +} + +/// Go's TestMirror: links in HTML and CSS become relative, off-host links +/// stay, everything lands under host/. +#[test] +fn mirror_like_go() { + let srv = Arc::new(std::sync::OnceLock::::new()); + let s2 = Arc::clone(&srv); + let server = Server::start(move |p, ua| { + if ua != CHROME_UA { + return Reply::Full("403 Forbidden", vec![], vec![]); + } + let base = s2.get().unwrap(); + match p { + "/" => ok( + "text/html", + &format!(r#"p e "#), + ), + "/sub/p.html" => ok("text/html", r#"home "#), + "/s.css" => ok("text/css", "body{background:url(/img/a.png)}"), + "/img/a.png" => ok("", "png"), + _ => not_found(), + } + }); + srv.set(server.url("")).unwrap(); + let v = vault(); + let out = mirror_with(&v, &server.url(""), &Limits::mirror_defaults()); + assert!(matches!(out, MirrorOutcome::Complete { stored: 4 })); + let h = &server.addr; + for (name, want) in [ + ( + format!("m/{h}/index.html"), + r#"p e "#, + ), + ( + format!("m/{h}/sub/p.html"), + r#"home "#, + ), + (format!("m/{h}/s.css"), "body{background:url(img/a.png)}"), + (format!("m/{h}/img/a.png"), "png"), + ] { + assert_eq!(String::from_utf8(v.get(&name)).unwrap(), want, "{name}"); + } +} + +#[test] +fn mirror_policies() { + let srv = Arc::new(std::sync::OnceLock::::new()); + let s2 = Arc::clone(&srv); + let server = Server::start(move |p, _| { + let other = s2.get().unwrap().replace("127.0.0.1", "localhost"); + match p { + "/" => ok( + "text/html", + r#"a x c q +
"#, + ), + "/d/a.html" => ok("text/html", "

a

"), + "/away" => redirect(&format!("{other}/elsewhere")), + "/a_b%253Dc" => ok("", "literal"), + "/a?b=c" => ok("", "query"), + "/i.png" | "/j.png" => ok("", "img"), + _ => not_found(), + } + }); + srv.set(server.url("")).unwrap(); + let v = vault(); + let out = mirror_with(&v, &server.url("/"), &Limits::mirror_defaults()); + // /away (off-host redirect) and one of the colliding names are skipped. + match out { + MirrorOutcome::Partial { skipped: 2, stored: 5 } => {} + MirrorOutcome::Partial { stored, skipped } => panic!("stored {stored}, skipped {skipped}: {:?}", v.names()), + MirrorOutcome::Complete { stored } => panic!("complete, stored {stored}"), + } + let h = &server.addr; + let index = String::from_utf8(v.get(&format!("m/{h}/index.html"))).unwrap(); + assert!(index.contains(r#""#), "{index}"); + assert!(index.contains(""), "base href honoured, unquoted kept: {index}"); + assert!( + index.contains(r#"''"#) && index.contains(r#""#), + "{index}" + ); + assert!(index.contains("url('i.png')") && index.contains(r#"url("j.png")"#), "{index}"); + let names = v.names(); + assert!(names.contains(&format!("m/{h}/d/a.html"))); + assert!(!names.iter().any(|n| n.contains("never") || n.contains("elsewhere")), "{names:?}"); + let collided = [format!("m/{h}/a_b%3Dc")]; + assert_eq!( + names.iter().filter(|n| collided.contains(n)).count(), + 1, + "one of two URLs owns the name" + ); + let kept = String::from_utf8(v.get(&collided[0])).unwrap(); + assert!(kept == "literal" || kept == "query"); +} + +#[test] +fn mirror_budgets() { + let server = Server::start(|p, _| match p { + "/" => ok("text/html", r#"123"#), + "/1" | "/2" | "/3" => ok("", &"z".repeat(1000)), + _ => not_found(), + }); + let v = vault(); + let pages = Limits { + max_pages: 2, + ..Limits::mirror_defaults() + }; + match mirror_with(&v, &server.url("/"), &pages) { + MirrorOutcome::Partial { stored: 2, .. } => {} + _ => panic!("page limit not applied"), + } + let v = vault(); + let bytes = Limits { + max_bytes: 1500, + ..Limits::mirror_defaults() + }; + match mirror_with(&v, &server.url("/"), &bytes) { + MirrorOutcome::Partial { stored: 2, skipped: 2 } => {} + MirrorOutcome::Partial { stored, skipped } => panic!("byte budget: stored {stored}, skipped {skipped}"), + MirrorOutcome::Complete { .. } => panic!("byte budget not applied"), + } + let v = vault(); + let text = Limits { + max_text: 10, + ..Limits::mirror_defaults() + }; + assert!(matches!( + mirror_with(&v, &server.url("/"), &text), + MirrorOutcome::Partial { stored: 0, skipped: 1 } + )); +} + +#[test] +fn relative_matches_go() { + for r in url::tests::vectors()["relative"].as_array().unwrap() { + let r = r.as_array().unwrap(); + assert_eq!( + relative(r[0].as_str().unwrap(), r[1].as_str().unwrap()), + r[2].as_str().unwrap(), + "{r:?}" + ); + } +} + +#[test] +fn http_time_matches_go() { + for (s, want) in url::tests::vectors()["http_time"].as_object().unwrap() { + assert_eq!(parse_http_time(s), want.as_i64(), "{s:?}"); + } +} + +#[test] +fn proxy_rules() { + let u = |s: &str| Url::parse(s).unwrap(); + assert!(!use_proxy(&u("http://localhost:8080/"), "")); + assert!(!use_proxy(&u("http://127.0.0.1/"), "")); + assert!(!use_proxy(&u("http://[::1]/"), "")); + assert!(use_proxy(&u("http://example.com/"), "other.com")); + assert!(!use_proxy(&u("http://example.com/"), "example.com")); + assert!(!use_proxy(&u("http://a.example.com/"), "example.com")); + assert!(use_proxy(&u("http://example.com/"), ".example.com")); + assert!(!use_proxy(&u("http://a.example.com/"), "*.example.com")); + assert!(!use_proxy(&u("http://x.com/"), "*")); + assert!(use_proxy(&u("http://x.com:81/"), "x.com:80")); + assert!(!use_proxy(&u("http://x.com/"), "x.com:80")); + assert!(!use_proxy(&u("https://10.1.2.3/"), "10.0.0.0/8")); + assert!(use_proxy(&u("https://11.1.2.3/"), "10.0.0.0/8, 192.168.1.1")); + assert!(!use_proxy(&u("https://192.168.1.1/"), "10.0.0.0/8, 192.168.1.1")); +} diff --git a/src/net/url.rs b/src/net/url.rs new file mode 100644 index 0000000..bf08431 --- /dev/null +++ b/src/net/url.rs @@ -0,0 +1,473 @@ +// SPDX-License-Identifier: GPL-3.0-or-later + +//! Just enough of Go's `net/url` to crawl and name pages exactly as the Go +//! version did: `Parse`, `ResolveReference` (RFC 3986 with dot-segment +//! removal), `String` (re-escaping like `EscapedPath`), `Hostname` and +//! `QueryEscape`. `path` is percent-decoded; `raw_query` is kept as sent. + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct Url { + pub scheme: String, + pub opaque: String, + /// Userinfo as written (still escaped), without the '@'. + pub user: Option, + pub host: String, + pub path: String, + raw_path: String, + pub force_query: bool, + pub raw_query: String, + pub fragment: String, + raw_fragment: String, + omit_host: bool, +} + +#[derive(Clone, Copy, PartialEq)] +enum Mode { + Path, + Host, + QueryComponent, + Fragment, +} + +fn should_escape(c: u8, mode: Mode) -> bool { + if c.is_ascii_alphanumeric() { + return false; + } + if mode == Mode::Host && b"!$&'()*+,;=:[]<>\"".contains(&c) { + return false; + } + match c { + b'-' | b'_' | b'.' | b'~' => return false, + b'$' | b'&' | b'+' | b',' | b'/' | b':' | b';' | b'=' | b'?' | b'@' => { + return match mode { + Mode::Path => c == b'?', + Mode::QueryComponent => true, + Mode::Fragment => false, + Mode::Host => true, + }; + } + _ => {} + } + if mode == Mode::Fragment && b"!()*".contains(&c) { + return false; + } + true +} + +fn escape(s: &str, mode: Mode) -> String { + let mut o = String::with_capacity(s.len()); + for &c in s.as_bytes() { + if c == b' ' && mode == Mode::QueryComponent { + o.push('+'); + } else if should_escape(c, mode) { + o.push_str(&format!("%{c:02X}")); + } else { + o.push(c as char); + } + } + o +} + +fn unhex(c: u8) -> Option { + (c as char).to_digit(16).map(|d| d as u8) +} + +fn unescape(s: &str, mode: Mode) -> Result { + let b = s.as_bytes(); + let mut out = Vec::with_capacity(b.len()); + let mut i = 0; + while i < b.len() { + match b[i] { + b'%' => { + let (h, l) = match (b.get(i + 1).copied().and_then(unhex), b.get(i + 2).copied().and_then(unhex)) { + (Some(h), Some(l)) => (h, l), + _ => return Err(format!("invalid URL escape {:?}", &s[i..(i + 3).min(s.len())])), + }; + if mode == Mode::Host && h < 8 && &s[i..i + 3] != "%25" { + return Err(format!("invalid URL escape {:?}", &s[i..i + 3])); + } + out.push(h << 4 | l); + i += 3; + } + b'+' => { + out.push(if mode == Mode::QueryComponent { b' ' } else { b'+' }); + i += 1; + } + c => { + if mode == Mode::Host && c < 0x80 && should_escape(c, mode) { + return Err(format!("invalid character {:?} in host name", c as char)); + } + out.push(c); + i += 1; + } + } + } + // Go strings may hold any bytes; a path that decodes to invalid UTF-8 + // cannot become a vault name anyway, so it is refused here. + String::from_utf8(out).map_err(|_| "invalid UTF-8 in URL".to_string()) +} + +pub fn query_escape(s: &str) -> String { + escape(s, Mode::QueryComponent) +} + +fn valid_encoded(s: &str, mode: Mode) -> bool { + s.bytes() + .all(|c| b"!$&'()*+,;=:@[]".contains(&c) || c == b'%' || !should_escape(c, mode)) +} + +fn get_scheme(raw: &str) -> Result<(&str, &str), String> { + for (i, c) in raw.bytes().enumerate() { + match c { + b'a'..=b'z' | b'A'..=b'Z' => {} + b'0'..=b'9' | b'+' | b'-' | b'.' if i > 0 => {} + b':' if i == 0 => return Err("missing protocol scheme".into()), + b':' => return Ok((&raw[..i], &raw[i + 1..])), + _ => return Ok(("", raw)), + } + } + Ok(("", raw)) +} + +fn valid_optional_port(p: &str) -> bool { + p.is_empty() || (p.starts_with(':') && p[1..].bytes().all(|c| c.is_ascii_digit())) +} + +fn parse_host(host: &str) -> Result { + if let Some(rest) = host.strip_prefix('[') { + let end = rest.find(']').ok_or("missing ']' in host")?; + let port = &rest[end + 1..]; + if !valid_optional_port(port) { + return Err(format!("invalid port {port:?} after host")); + } + } else if let Some(i) = host.rfind(':') { + let port = &host[i..]; + if !valid_optional_port(port) { + return Err(format!("invalid port {port:?} after host")); + } + } + unescape(host, Mode::Host) +} + +impl Url { + pub fn parse(raw: &str) -> Result { + let (rest, frag) = match raw.find('#') { + Some(i) => (&raw[..i], Some(&raw[i + 1..])), + None => (raw, None), + }; + let mut u = Url::parse_without_fragment(rest)?; + if let Some(f) = frag { + u.fragment = unescape(f, Mode::Fragment)?; + u.raw_fragment = if escape(&u.fragment, Mode::Fragment) == f { + String::new() + } else { + f.to_string() + }; + } + Ok(u) + } + + fn parse_without_fragment(raw: &str) -> Result { + if raw.bytes().any(|c| c < 0x20 || c == 0x7f) { + return Err("net/url: invalid control character in URL".into()); + } + let mut u = Url::default(); + if raw.is_empty() { + return Ok(u); + } + let (scheme, mut rest) = get_scheme(raw)?; + u.scheme = scheme.to_ascii_lowercase(); + if rest.ends_with('?') && rest.matches('?').count() == 1 { + u.force_query = true; + rest = &rest[..rest.len() - 1]; + } else if let Some(i) = rest.find('?') { + u.raw_query = rest[i + 1..].to_string(); + rest = &rest[..i]; + } + if !rest.starts_with('/') { + if !u.scheme.is_empty() { + u.opaque = rest.to_string(); + return Ok(u); + } + let seg = rest.split('/').next().unwrap_or(""); + if seg.contains(':') { + return Err("first path segment in URL cannot contain colon".into()); + } + } + if (!u.scheme.is_empty() || !rest.starts_with("///")) && rest.starts_with("//") { + let auth_and_rest = &rest[2..]; + let (authority, r) = match auth_and_rest.find('/') { + Some(i) => (&auth_and_rest[..i], &auth_and_rest[i..]), + None => (auth_and_rest, ""), + }; + rest = r; + let host = match authority.rfind('@') { + Some(i) => { + let ui = &authority[..i]; + if !ui.bytes().all(|c| c.is_ascii_alphanumeric() || b"-._:~!$&'()*+,;=%@".contains(&c)) { + return Err("net/url: invalid userinfo".into()); + } + u.user = Some(ui.to_string()); + &authority[i + 1..] + } + None => authority, + }; + u.host = parse_host(host)?; + } else if !u.scheme.is_empty() && rest.starts_with('/') { + u.omit_host = true; + } + if !rest.is_empty() { + u.set_path(rest)?; + } + Ok(u) + } + + fn set_path(&mut self, p: &str) -> Result<(), String> { + self.path = unescape(p, Mode::Path)?; + self.raw_path = if escape(&self.path, Mode::Path) == p { + String::new() + } else { + p.to_string() + }; + Ok(()) + } + + pub fn escaped_path(&self) -> String { + if !self.raw_path.is_empty() + && valid_encoded(&self.raw_path, Mode::Path) + && unescape(&self.raw_path, Mode::Path).ok().as_deref() == Some(&self.path) + { + return self.raw_path.clone(); + } + if self.path == "*" { + return "*".into(); + } + escape(&self.path, Mode::Path) + } + + fn escaped_fragment(&self) -> String { + if !self.raw_fragment.is_empty() + && valid_encoded(&self.raw_fragment, Mode::Fragment) + && unescape(&self.raw_fragment, Mode::Fragment).ok().as_deref() == Some(&self.fragment) + { + return self.raw_fragment.clone(); + } + escape(&self.fragment, Mode::Fragment) + } + + /// `ref` resolved against `self`, as Go's `URL.Parse`. + pub fn join(&self, reference: &str) -> Result { + Ok(self.resolve(&Url::parse(reference)?)) + } + + /// Go's `ResolveReference`. + pub fn resolve(&self, r: &Url) -> Url { + let mut u = r.clone(); + if r.scheme.is_empty() { + u.scheme = self.scheme.clone(); + } + if !r.scheme.is_empty() || !r.host.is_empty() || r.user.is_some() { + let p = resolve_path(&r.escaped_path(), ""); + let _ = u.set_path(&p); + return u; + } + if !r.opaque.is_empty() { + u.user = None; + u.host.clear(); + u.path.clear(); + return u; + } + if r.path.is_empty() && !r.force_query && r.raw_query.is_empty() { + u.raw_query = self.raw_query.clone(); + if r.fragment.is_empty() { + u.fragment = self.fragment.clone(); + u.raw_fragment = self.raw_fragment.clone(); + } + } + if r.path.is_empty() && !self.opaque.is_empty() { + u.opaque = self.opaque.clone(); + u.user = None; + u.host.clear(); + u.path.clear(); + return u; + } + u.host = self.host.clone(); + u.user = self.user.clone(); + let p = resolve_path(&self.escaped_path(), &r.escaped_path()); + let _ = u.set_path(&p); + u + } + + pub fn clear_fragment(&mut self) { + self.fragment.clear(); + self.raw_fragment.clear(); + } + + /// The host without any port, IPv6 brackets removed. + pub fn hostname(&self) -> &str { + let h = &self.host; + if let Some(rest) = h.strip_prefix('[') { + return rest.find(']').map_or(rest, |i| &rest[..i]); + } + match h.rfind(':') { + Some(i) if valid_optional_port(&h[i..]) => &h[..i], + _ => h, + } + } + + /// The port, or the scheme's default. + pub fn port_or_default(&self) -> &str { + let h = &self.host; + let after = if h.starts_with('[') { + h.find(']').map_or("", |i| &h[i + 1..]) + } else { + h.rfind(':').map_or("", |i| &h[i..]) + }; + match after.strip_prefix(':') { + Some(p) if !p.is_empty() => p, + _ if self.scheme == "https" => "443", + _ => "80", + } + } +} + +impl std::fmt::Display for Url { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut b = String::new(); + if !self.scheme.is_empty() { + b.push_str(&self.scheme); + b.push(':'); + } + if !self.opaque.is_empty() { + b.push_str(&self.opaque); + } else { + if (!self.scheme.is_empty() || !self.host.is_empty() || self.user.is_some()) + && !(self.omit_host && self.host.is_empty() && self.user.is_none()) + { + if !self.host.is_empty() || !self.path.is_empty() || self.user.is_some() { + b.push_str("//"); + } + if let Some(ui) = &self.user { + b.push_str(ui); + b.push('@'); + } + b.push_str(&escape(&self.host, Mode::Host)); + } + let p = self.escaped_path(); + if !p.is_empty() && !p.starts_with('/') && !self.host.is_empty() { + b.push('/'); + } + if b.is_empty() && p.split('/').next().unwrap_or("").contains(':') { + b.push_str("./"); + } + b.push_str(&p); + } + if self.force_query || !self.raw_query.is_empty() { + b.push('?'); + b.push_str(&self.raw_query); + } + if !self.fragment.is_empty() { + b.push('#'); + b.push_str(&self.escaped_fragment()); + } + f.write_str(&b) + } +} + +/// Go's `resolvePath`: merge and remove dot segments. +fn resolve_path(base: &str, r: &str) -> String { + let full = if r.is_empty() { + base.to_string() + } else if !r.starts_with('/') { + let i = base.rfind('/').map_or(0, |i| i + 1); + format!("{}{}", &base[..i], r) + } else { + r.to_string() + }; + if full.is_empty() { + return String::new(); + } + let mut dst = String::from("/"); + let mut first = true; + let mut last = ""; + for elem in full.split('/') { + last = elem; + if elem == "." { + first = false; + continue; + } + if elem == ".." { + let s = dst[1..].to_string(); + dst = String::from("/"); + match s.rfind('/') { + None => first = true, + Some(i) => dst.push_str(&s[..i]), + } + } else { + if !first { + dst.push('/'); + } + dst.push_str(elem); + first = false; + } + } + if last == "." || last == ".." { + dst.push('/'); + } + if dst.len() > 1 && dst.as_bytes()[1] == b'/' { + dst.remove(0); + } + dst +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use serde_json::Value; + + pub(crate) fn vectors() -> Value { + serde_json::from_str(include_str!("../../tests/vectors/net.json")).unwrap() + } + + fn check(v: &Value, got: Result, ctx: &str) { + if v["err"].as_bool().unwrap() { + assert!(got.is_err(), "{ctx}: accepted, Go refused: {:?}", got.map(|u| u.to_string())); + return; + } + let u = got.unwrap_or_else(|e| panic!("{ctx}: refused ({e}), Go accepted")); + let s = |k: &str| v[k].as_str().unwrap_or("").to_string(); + assert_eq!(u.to_string(), s("string"), "{ctx} String"); + assert_eq!(u.scheme, s("scheme"), "{ctx} Scheme"); + assert_eq!(u.host, s("host"), "{ctx} Host"); + assert_eq!(u.hostname(), s("hostname"), "{ctx} Hostname"); + assert_eq!(u.path, s("path"), "{ctx} Path"); + assert_eq!(u.raw_query, s("raw_query"), "{ctx} RawQuery"); + assert_eq!(u.fragment, s("fragment"), "{ctx} Fragment"); + assert_eq!(super::super::local_path(&u), s("local_path"), "{ctx} localPath"); + assert_eq!(super::super::key(&u), s("key"), "{ctx} key"); + } + + #[test] + fn parse_matches_go() { + for v in vectors()["parse"].as_array().unwrap() { + let input = v["input"].as_str().unwrap(); + check(v, Url::parse(input), &format!("Parse({input:?})")); + } + } + + #[test] + fn resolve_matches_go() { + for v in vectors()["resolve"].as_array().unwrap() { + let (b, r) = (v["base"].as_str().unwrap(), v["ref"].as_str().unwrap()); + let base = Url::parse(b).unwrap(); + check(&v["result"], base.join(r), &format!("{b:?}.Parse({r:?})")); + } + } + + #[test] + fn query_escape_matches_go() { + for (k, want) in vectors()["query_escape"].as_object().unwrap() { + assert_eq!(query_escape(k), want.as_str().unwrap()); + } + } +} diff --git a/tests/vectors/net.json b/tests/vectors/net.json new file mode 100644 index 0000000..e2f2ebf --- /dev/null +++ b/tests/vectors/net.json @@ -0,0 +1,3180 @@ +{ + "http_time": { + "Mon Jan 2 15:04:05 2006": 1136214245, + "Mon Jan 12 15:04:05 2006": 1137078245, + "Mon, 02 Jan 2006 15:04:05 GMT": 1136214245, + "Mon, 02 Jan 2006 15:04:05 UTC": null, + "Monday, 02-Jan-06 15:04:05 GMT": 1136214245, + "Tue, 15 Nov 1994 08:12:31 GMT": 784887151, + "bogus": null + }, + "parse": [ + { + "input": "", + "err": false, + "local_path": "index.html" + }, + { + "input": "a", + "err": false, + "string": "a", + "path": "a", + "local_path": "a", + "key": "a" + }, + { + "input": "a/b", + "err": false, + "string": "a/b", + "path": "a/b", + "local_path": "a/b", + "key": "a/b" + }, + { + "input": "./a", + "err": false, + "string": "./a", + "path": "./a", + "local_path": "./a", + "key": "./a" + }, + { + "input": "../a", + "err": false, + "string": "../a", + "path": "../a", + "local_path": "../a", + "key": "../a" + }, + { + "input": "../../../../a", + "err": false, + "string": "../../../../a", + "path": "../../../../a", + "local_path": "../../../../a", + "key": "../../../../a" + }, + { + "input": "/a/./b/../c", + "err": false, + "string": "/a/./b/../c", + "path": "/a/./b/../c", + "local_path": "a/./b/../c", + "key": "/a/./b/../c" + }, + { + "input": "/", + "err": false, + "string": "/", + "path": "/", + "local_path": "index.html", + "key": "/" + }, + { + "input": "//other.example/x", + "err": false, + "string": "//other.example/x", + "host": "other.example", + "hostname": "other.example", + "path": "/x", + "local_path": "x", + "key": "//other.example/x" + }, + { + "input": "//OTHER.example:8080", + "err": false, + "string": "//OTHER.example:8080", + "host": "OTHER.example:8080", + "hostname": "OTHER.example", + "local_path": "index.html", + "key": "//OTHER.example:8080" + }, + { + "input": "?q=1", + "err": false, + "string": "?q=1", + "raw_query": "q=1", + "local_path": "index.html_q%3D1", + "key": "?q=1" + }, + { + "input": "?", + "err": false, + "string": "?", + "local_path": "index.html", + "key": "?" + }, + { + "input": "#frag", + "err": false, + "string": "#frag", + "fragment": "frag", + "local_path": "index.html" + }, + { + "input": "a#f", + "err": false, + "string": "a#f", + "path": "a", + "fragment": "f", + "local_path": "a", + "key": "a" + }, + { + "input": "a?x=1\u0026y=2#f", + "err": false, + "string": "a?x=1\u0026y=2#f", + "path": "a", + "raw_query": "x=1\u0026y=2", + "fragment": "f", + "local_path": "a_x%3D1%26y%3D2", + "key": "a?x=1\u0026y=2" + }, + { + "input": "/sp ace", + "err": false, + "string": "/sp%20ace", + "path": "/sp ace", + "local_path": "sp ace", + "key": "/sp%20ace" + }, + { + "input": "/%41%2fb", + "err": false, + "string": "/%41%2fb", + "path": "/A/b", + "local_path": "A/b", + "key": "/%41%2fb" + }, + { + "input": "/%zz", + "err": true + }, + { + "input": "/caf%C3%A9", + "err": false, + "string": "/caf%C3%A9", + "path": "/café", + "local_path": "café", + "key": "/caf%C3%A9" + }, + { + "input": "/café", + "err": false, + "string": "/caf%C3%A9", + "path": "/café", + "local_path": "café", + "key": "/caf%C3%A9" + }, + { + "input": "http://H.Example:80/A", + "err": false, + "string": "http://H.Example:80/A", + "scheme": "http", + "host": "H.Example:80", + "hostname": "H.Example", + "path": "/A", + "local_path": "A", + "key": "http://H.Example:80/A" + }, + { + "input": "https://x/a/b/../../..", + "err": false, + "string": "https://x/a/b/../../..", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/a/b/../../..", + "local_path": "a/b/../../..", + "key": "https://x/a/b/../../.." + }, + { + "input": "HTTP://x", + "err": false, + "string": "http://x", + "scheme": "http", + "host": "x", + "hostname": "x", + "local_path": "index.html", + "key": "http://x" + }, + { + "input": "mailto:a@b", + "err": false, + "string": "mailto:a@b", + "scheme": "mailto", + "local_path": "index.html", + "key": "mailto:a@b" + }, + { + "input": "javascript:alert(1)", + "err": false, + "string": "javascript:alert(1)", + "scheme": "javascript", + "local_path": "index.html", + "key": "javascript:alert(1)" + }, + { + "input": "ftp://x/y", + "err": false, + "string": "ftp://x/y", + "scheme": "ftp", + "host": "x", + "hostname": "x", + "path": "/y", + "local_path": "y", + "key": "ftp://x/y" + }, + { + "input": "x:y", + "err": false, + "string": "x:y", + "scheme": "x", + "local_path": "index.html", + "key": "x:y" + }, + { + "input": "a:b/c", + "err": false, + "string": "a:b/c", + "scheme": "a", + "local_path": "index.html", + "key": "a:b/c" + }, + { + "input": "/a?b?c", + "err": false, + "string": "/a?b?c", + "path": "/a", + "raw_query": "b?c", + "local_path": "a_b%3Fc", + "key": "/a?b?c" + }, + { + "input": "a/..", + "err": false, + "string": "a/..", + "path": "a/..", + "local_path": "a/..", + "key": "a/.." + }, + { + "input": "a/.", + "err": false, + "string": "a/.", + "path": "a/.", + "local_path": "a/.", + "key": "a/." + }, + { + "input": ".", + "err": false, + "string": ".", + "path": ".", + "local_path": ".", + "key": "." + }, + { + "input": "..", + "err": false, + "string": "..", + "path": "..", + "local_path": "..", + "key": ".." + }, + { + "input": "/a//b", + "err": false, + "string": "/a//b", + "path": "/a//b", + "local_path": "a//b", + "key": "/a//b" + }, + { + "input": "https://user:pw@x/p", + "err": false, + "string": "https://user:pw@x/p", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "https://user:pw@x/p" + }, + { + "input": "http://[::1]:8080/v6", + "err": false, + "string": "http://[::1]:8080/v6", + "scheme": "http", + "host": "[::1]:8080", + "hostname": "::1", + "path": "/v6", + "local_path": "v6", + "key": "http://[::1]:8080/v6" + }, + { + "input": "http://x:port/", + "err": true + }, + { + "input": "http://x:/p", + "err": false, + "string": "http://x:/p", + "scheme": "http", + "host": "x:", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "http://x:/p" + }, + { + "input": "/a%20b?c%20d#e%20f", + "err": false, + "string": "/a%20b?c%20d#e%20f", + "path": "/a b", + "raw_query": "c%20d", + "fragment": "e f", + "local_path": "a b_c%2520d", + "key": "/a%20b?c%20d" + }, + { + "input": "http://x/\u0001", + "err": true + }, + { + "input": "/a+b", + "err": false, + "string": "/a+b", + "path": "/a+b", + "local_path": "a+b", + "key": "/a+b" + }, + { + "input": "/q?a+b=c d", + "err": false, + "string": "/q?a+b=c d", + "path": "/q", + "raw_query": "a+b=c d", + "local_path": "q_a%2Bb%3Dc+d", + "key": "/q?a+b=c d" + }, + { + "input": "#", + "err": false, + "local_path": "index.html" + }, + { + "input": "?#", + "err": false, + "string": "?", + "local_path": "index.html", + "key": "?" + }, + { + "input": "/p#a%23b", + "err": false, + "string": "/p#a%23b", + "path": "/p", + "fragment": "a#b", + "local_path": "p", + "key": "/p" + }, + { + "input": "//x/?", + "err": false, + "string": "//x/?", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "//x/?" + }, + { + "input": "http://x?only", + "err": false, + "string": "http://x?only", + "scheme": "http", + "host": "x", + "hostname": "x", + "raw_query": "only", + "local_path": "index.html_only", + "key": "http://x?only" + } + ], + "provenance": "oracle/go at 4c1507c, net_oracle_vectors_test.go", + "query_escape": { + "a+b": "a%2Bb", + "a=1\u0026b=2": "a%3D1%26b%3D2", + "x y": "x+y", + "~-_.!*'()": "~-_.%21%2A%27%28%29", + "é/?#": "%C3%A9%2F%3F%23" + }, + "relative": [ + [ + "index.html", + "sub/p.html", + "sub/p.html" + ], + [ + "sub/p.html", + "index.html", + "../index.html" + ], + [ + "sub/p.html", + "img/a.png", + "../img/a.png" + ], + [ + "a/b/c.html", + "a/b/d.html", + "d.html" + ], + [ + "a/b/c.html", + "a/x/y/z.css", + "../x/y/z.css" + ], + [ + "x.html", + "x.html", + "x.html" + ], + [ + "a/b/", + "a/c", + "../c" + ], + [ + "s.css", + "img/a.png", + "img/a.png" + ], + [ + "a/b/c", + "a", + "../../a" + ] + ], + "resolve": [ + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "", + "result": { + "input": "", + "err": false, + "string": "http://h.example/dir/page.html?q=1#top", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "raw_query": "q=1", + "fragment": "top", + "local_path": "dir/page.html_q%3D1", + "key": "http://h.example/dir/page.html?q=1" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a", + "result": { + "input": "a", + "err": false, + "string": "http://h.example/dir/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a", + "local_path": "dir/a", + "key": "http://h.example/dir/a" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a/b", + "result": { + "input": "a/b", + "err": false, + "string": "http://h.example/dir/a/b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a/b", + "local_path": "dir/a/b", + "key": "http://h.example/dir/a/b" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "./a", + "result": { + "input": "./a", + "err": false, + "string": "http://h.example/dir/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a", + "local_path": "dir/a", + "key": "http://h.example/dir/a" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "../a", + "result": { + "input": "../a", + "err": false, + "string": "http://h.example/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "http://h.example/a" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "../../../../a", + "result": { + "input": "../../../../a", + "err": false, + "string": "http://h.example/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "http://h.example/a" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/a/./b/../c", + "result": { + "input": "/a/./b/../c", + "err": false, + "string": "http://h.example/a/c", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/c", + "local_path": "a/c", + "key": "http://h.example/a/c" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/", + "result": { + "input": "/", + "err": false, + "string": "http://h.example/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "http://h.example/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "//other.example/x", + "result": { + "input": "//other.example/x", + "err": false, + "string": "http://other.example/x", + "scheme": "http", + "host": "other.example", + "hostname": "other.example", + "path": "/x", + "local_path": "x", + "key": "http://other.example/x" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "//OTHER.example:8080", + "result": { + "input": "//OTHER.example:8080", + "err": false, + "string": "http://OTHER.example:8080", + "scheme": "http", + "host": "OTHER.example:8080", + "hostname": "OTHER.example", + "local_path": "index.html", + "key": "http://OTHER.example:8080" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "?q=1", + "result": { + "input": "?q=1", + "err": false, + "string": "http://h.example/dir/page.html?q=1", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "raw_query": "q=1", + "local_path": "dir/page.html_q%3D1", + "key": "http://h.example/dir/page.html?q=1" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "?", + "result": { + "input": "?", + "err": false, + "string": "http://h.example/dir/page.html?", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "local_path": "dir/page.html", + "key": "http://h.example/dir/page.html?" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "#frag", + "result": { + "input": "#frag", + "err": false, + "string": "http://h.example/dir/page.html?q=1#frag", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "raw_query": "q=1", + "fragment": "frag", + "local_path": "dir/page.html_q%3D1", + "key": "http://h.example/dir/page.html?q=1" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a#f", + "result": { + "input": "a#f", + "err": false, + "string": "http://h.example/dir/a#f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a", + "fragment": "f", + "local_path": "dir/a", + "key": "http://h.example/dir/a" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a?x=1\u0026y=2#f", + "result": { + "input": "a?x=1\u0026y=2#f", + "err": false, + "string": "http://h.example/dir/a?x=1\u0026y=2#f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a", + "raw_query": "x=1\u0026y=2", + "fragment": "f", + "local_path": "dir/a_x%3D1%26y%3D2", + "key": "http://h.example/dir/a?x=1\u0026y=2" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/sp ace", + "result": { + "input": "/sp ace", + "err": false, + "string": "http://h.example/sp%20ace", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/sp ace", + "local_path": "sp ace", + "key": "http://h.example/sp%20ace" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/%41%2fb", + "result": { + "input": "/%41%2fb", + "err": false, + "string": "http://h.example/%41%2fb", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/A/b", + "local_path": "A/b", + "key": "http://h.example/%41%2fb" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/%zz", + "result": { + "input": "/%zz", + "err": true + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/caf%C3%A9", + "result": { + "input": "/caf%C3%A9", + "err": false, + "string": "http://h.example/caf%C3%A9", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example/caf%C3%A9" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/café", + "result": { + "input": "/café", + "err": false, + "string": "http://h.example/caf%C3%A9", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example/caf%C3%A9" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://H.Example:80/A", + "result": { + "input": "http://H.Example:80/A", + "err": false, + "string": "http://H.Example:80/A", + "scheme": "http", + "host": "H.Example:80", + "hostname": "H.Example", + "path": "/A", + "local_path": "A", + "key": "http://H.Example:80/A" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "https://x/a/b/../../..", + "result": { + "input": "https://x/a/b/../../..", + "err": false, + "string": "https://x/", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "https://x/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "HTTP://x", + "result": { + "input": "HTTP://x", + "err": false, + "string": "http://x", + "scheme": "http", + "host": "x", + "hostname": "x", + "local_path": "index.html", + "key": "http://x" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "mailto:a@b", + "result": { + "input": "mailto:a@b", + "err": false, + "string": "mailto:a@b", + "scheme": "mailto", + "local_path": "index.html", + "key": "mailto:a@b" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "javascript:alert(1)", + "result": { + "input": "javascript:alert(1)", + "err": false, + "string": "javascript:alert(1)", + "scheme": "javascript", + "local_path": "index.html", + "key": "javascript:alert(1)" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "ftp://x/y", + "result": { + "input": "ftp://x/y", + "err": false, + "string": "ftp://x/y", + "scheme": "ftp", + "host": "x", + "hostname": "x", + "path": "/y", + "local_path": "y", + "key": "ftp://x/y" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "x:y", + "result": { + "input": "x:y", + "err": false, + "string": "x:y", + "scheme": "x", + "local_path": "index.html", + "key": "x:y" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a:b/c", + "result": { + "input": "a:b/c", + "err": false, + "string": "a:b/c", + "scheme": "a", + "local_path": "index.html", + "key": "a:b/c" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/a?b?c", + "result": { + "input": "/a?b?c", + "err": false, + "string": "http://h.example/a?b?c", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "raw_query": "b?c", + "local_path": "a_b%3Fc", + "key": "http://h.example/a?b?c" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a/..", + "result": { + "input": "a/..", + "err": false, + "string": "http://h.example/dir/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/", + "local_path": "dir/index.html", + "key": "http://h.example/dir/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "a/.", + "result": { + "input": "a/.", + "err": false, + "string": "http://h.example/dir/a/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/a/", + "local_path": "dir/a/index.html", + "key": "http://h.example/dir/a/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": ".", + "result": { + "input": ".", + "err": false, + "string": "http://h.example/dir/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/", + "local_path": "dir/index.html", + "key": "http://h.example/dir/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "..", + "result": { + "input": "..", + "err": false, + "string": "http://h.example/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "http://h.example/" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/a//b", + "result": { + "input": "/a//b", + "err": false, + "string": "http://h.example/a//b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a//b", + "local_path": "a//b", + "key": "http://h.example/a//b" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "https://user:pw@x/p", + "result": { + "input": "https://user:pw@x/p", + "err": false, + "string": "https://user:pw@x/p", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "https://user:pw@x/p" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://[::1]:8080/v6", + "result": { + "input": "http://[::1]:8080/v6", + "err": false, + "string": "http://[::1]:8080/v6", + "scheme": "http", + "host": "[::1]:8080", + "hostname": "::1", + "path": "/v6", + "local_path": "v6", + "key": "http://[::1]:8080/v6" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://x:port/", + "result": { + "input": "http://x:port/", + "err": true + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://x:/p", + "result": { + "input": "http://x:/p", + "err": false, + "string": "http://x:/p", + "scheme": "http", + "host": "x:", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "http://x:/p" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/a%20b?c%20d#e%20f", + "result": { + "input": "/a%20b?c%20d#e%20f", + "err": false, + "string": "http://h.example/a%20b?c%20d#e%20f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a b", + "raw_query": "c%20d", + "fragment": "e f", + "local_path": "a b_c%2520d", + "key": "http://h.example/a%20b?c%20d" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://x/\u0001", + "result": { + "input": "http://x/\u0001", + "err": true + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/a+b", + "result": { + "input": "/a+b", + "err": false, + "string": "http://h.example/a+b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a+b", + "local_path": "a+b", + "key": "http://h.example/a+b" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/q?a+b=c d", + "result": { + "input": "/q?a+b=c d", + "err": false, + "string": "http://h.example/q?a+b=c d", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/q", + "raw_query": "a+b=c d", + "local_path": "q_a%2Bb%3Dc+d", + "key": "http://h.example/q?a+b=c d" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "#", + "result": { + "input": "#", + "err": false, + "string": "http://h.example/dir/page.html?q=1#top", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "raw_query": "q=1", + "fragment": "top", + "local_path": "dir/page.html_q%3D1", + "key": "http://h.example/dir/page.html?q=1" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "?#", + "result": { + "input": "?#", + "err": false, + "string": "http://h.example/dir/page.html?", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/dir/page.html", + "local_path": "dir/page.html", + "key": "http://h.example/dir/page.html?" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "/p#a%23b", + "result": { + "input": "/p#a%23b", + "err": false, + "string": "http://h.example/p#a%23b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/p", + "fragment": "a#b", + "local_path": "p", + "key": "http://h.example/p" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "//x/?", + "result": { + "input": "//x/?", + "err": false, + "string": "http://x/?", + "scheme": "http", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "http://x/?" + } + }, + { + "base": "http://h.example/dir/page.html?q=1#top", + "ref": "http://x?only", + "result": { + "input": "http://x?only", + "err": false, + "string": "http://x?only", + "scheme": "http", + "host": "x", + "hostname": "x", + "raw_query": "only", + "local_path": "index.html_only", + "key": "http://x?only" + } + }, + { + "base": "https://h.example", + "ref": "", + "result": { + "input": "", + "err": false, + "string": "https://h.example", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "local_path": "index.html", + "key": "https://h.example" + } + }, + { + "base": "https://h.example", + "ref": "a", + "result": { + "input": "a", + "err": false, + "string": "https://h.example/a", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "https://h.example/a" + } + }, + { + "base": "https://h.example", + "ref": "a/b", + "result": { + "input": "a/b", + "err": false, + "string": "https://h.example/a/b", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "local_path": "a/b", + "key": "https://h.example/a/b" + } + }, + { + "base": "https://h.example", + "ref": "./a", + "result": { + "input": "./a", + "err": false, + "string": "https://h.example/a", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "https://h.example/a" + } + }, + { + "base": "https://h.example", + "ref": "../a", + "result": { + "input": "../a", + "err": false, + "string": "https://h.example/a", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "https://h.example/a" + } + }, + { + "base": "https://h.example", + "ref": "../../../../a", + "result": { + "input": "../../../../a", + "err": false, + "string": "https://h.example/a", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "https://h.example/a" + } + }, + { + "base": "https://h.example", + "ref": "/a/./b/../c", + "result": { + "input": "/a/./b/../c", + "err": false, + "string": "https://h.example/a/c", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a/c", + "local_path": "a/c", + "key": "https://h.example/a/c" + } + }, + { + "base": "https://h.example", + "ref": "/", + "result": { + "input": "/", + "err": false, + "string": "https://h.example/", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "https://h.example/" + } + }, + { + "base": "https://h.example", + "ref": "//other.example/x", + "result": { + "input": "//other.example/x", + "err": false, + "string": "https://other.example/x", + "scheme": "https", + "host": "other.example", + "hostname": "other.example", + "path": "/x", + "local_path": "x", + "key": "https://other.example/x" + } + }, + { + "base": "https://h.example", + "ref": "//OTHER.example:8080", + "result": { + "input": "//OTHER.example:8080", + "err": false, + "string": "https://OTHER.example:8080", + "scheme": "https", + "host": "OTHER.example:8080", + "hostname": "OTHER.example", + "local_path": "index.html", + "key": "https://OTHER.example:8080" + } + }, + { + "base": "https://h.example", + "ref": "?q=1", + "result": { + "input": "?q=1", + "err": false, + "string": "https://h.example?q=1", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "raw_query": "q=1", + "local_path": "index.html_q%3D1", + "key": "https://h.example?q=1" + } + }, + { + "base": "https://h.example", + "ref": "?", + "result": { + "input": "?", + "err": false, + "string": "https://h.example?", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "local_path": "index.html", + "key": "https://h.example?" + } + }, + { + "base": "https://h.example", + "ref": "#frag", + "result": { + "input": "#frag", + "err": false, + "string": "https://h.example#frag", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "fragment": "frag", + "local_path": "index.html", + "key": "https://h.example" + } + }, + { + "base": "https://h.example", + "ref": "a#f", + "result": { + "input": "a#f", + "err": false, + "string": "https://h.example/a#f", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "fragment": "f", + "local_path": "a", + "key": "https://h.example/a" + } + }, + { + "base": "https://h.example", + "ref": "a?x=1\u0026y=2#f", + "result": { + "input": "a?x=1\u0026y=2#f", + "err": false, + "string": "https://h.example/a?x=1\u0026y=2#f", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "raw_query": "x=1\u0026y=2", + "fragment": "f", + "local_path": "a_x%3D1%26y%3D2", + "key": "https://h.example/a?x=1\u0026y=2" + } + }, + { + "base": "https://h.example", + "ref": "/sp ace", + "result": { + "input": "/sp ace", + "err": false, + "string": "https://h.example/sp%20ace", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/sp ace", + "local_path": "sp ace", + "key": "https://h.example/sp%20ace" + } + }, + { + "base": "https://h.example", + "ref": "/%41%2fb", + "result": { + "input": "/%41%2fb", + "err": false, + "string": "https://h.example/%41%2fb", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/A/b", + "local_path": "A/b", + "key": "https://h.example/%41%2fb" + } + }, + { + "base": "https://h.example", + "ref": "/%zz", + "result": { + "input": "/%zz", + "err": true + } + }, + { + "base": "https://h.example", + "ref": "/caf%C3%A9", + "result": { + "input": "/caf%C3%A9", + "err": false, + "string": "https://h.example/caf%C3%A9", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "https://h.example/caf%C3%A9" + } + }, + { + "base": "https://h.example", + "ref": "/café", + "result": { + "input": "/café", + "err": false, + "string": "https://h.example/caf%C3%A9", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "https://h.example/caf%C3%A9" + } + }, + { + "base": "https://h.example", + "ref": "http://H.Example:80/A", + "result": { + "input": "http://H.Example:80/A", + "err": false, + "string": "http://H.Example:80/A", + "scheme": "http", + "host": "H.Example:80", + "hostname": "H.Example", + "path": "/A", + "local_path": "A", + "key": "http://H.Example:80/A" + } + }, + { + "base": "https://h.example", + "ref": "https://x/a/b/../../..", + "result": { + "input": "https://x/a/b/../../..", + "err": false, + "string": "https://x/", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "https://x/" + } + }, + { + "base": "https://h.example", + "ref": "HTTP://x", + "result": { + "input": "HTTP://x", + "err": false, + "string": "http://x", + "scheme": "http", + "host": "x", + "hostname": "x", + "local_path": "index.html", + "key": "http://x" + } + }, + { + "base": "https://h.example", + "ref": "mailto:a@b", + "result": { + "input": "mailto:a@b", + "err": false, + "string": "mailto:a@b", + "scheme": "mailto", + "local_path": "index.html", + "key": "mailto:a@b" + } + }, + { + "base": "https://h.example", + "ref": "javascript:alert(1)", + "result": { + "input": "javascript:alert(1)", + "err": false, + "string": "javascript:alert(1)", + "scheme": "javascript", + "local_path": "index.html", + "key": "javascript:alert(1)" + } + }, + { + "base": "https://h.example", + "ref": "ftp://x/y", + "result": { + "input": "ftp://x/y", + "err": false, + "string": "ftp://x/y", + "scheme": "ftp", + "host": "x", + "hostname": "x", + "path": "/y", + "local_path": "y", + "key": "ftp://x/y" + } + }, + { + "base": "https://h.example", + "ref": "x:y", + "result": { + "input": "x:y", + "err": false, + "string": "x:y", + "scheme": "x", + "local_path": "index.html", + "key": "x:y" + } + }, + { + "base": "https://h.example", + "ref": "a:b/c", + "result": { + "input": "a:b/c", + "err": false, + "string": "a:b/c", + "scheme": "a", + "local_path": "index.html", + "key": "a:b/c" + } + }, + { + "base": "https://h.example", + "ref": "/a?b?c", + "result": { + "input": "/a?b?c", + "err": false, + "string": "https://h.example/a?b?c", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "raw_query": "b?c", + "local_path": "a_b%3Fc", + "key": "https://h.example/a?b?c" + } + }, + { + "base": "https://h.example", + "ref": "a/..", + "result": { + "input": "a/..", + "err": false, + "string": "https://h.example/", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "https://h.example/" + } + }, + { + "base": "https://h.example", + "ref": "a/.", + "result": { + "input": "a/.", + "err": false, + "string": "https://h.example/a/", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a/", + "local_path": "a/index.html", + "key": "https://h.example/a/" + } + }, + { + "base": "https://h.example", + "ref": ".", + "result": { + "input": ".", + "err": false, + "string": "https://h.example/", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "https://h.example/" + } + }, + { + "base": "https://h.example", + "ref": "..", + "result": { + "input": "..", + "err": false, + "string": "https://h.example/", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "https://h.example/" + } + }, + { + "base": "https://h.example", + "ref": "/a//b", + "result": { + "input": "/a//b", + "err": false, + "string": "https://h.example/a//b", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a//b", + "local_path": "a//b", + "key": "https://h.example/a//b" + } + }, + { + "base": "https://h.example", + "ref": "https://user:pw@x/p", + "result": { + "input": "https://user:pw@x/p", + "err": false, + "string": "https://user:pw@x/p", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "https://user:pw@x/p" + } + }, + { + "base": "https://h.example", + "ref": "http://[::1]:8080/v6", + "result": { + "input": "http://[::1]:8080/v6", + "err": false, + "string": "http://[::1]:8080/v6", + "scheme": "http", + "host": "[::1]:8080", + "hostname": "::1", + "path": "/v6", + "local_path": "v6", + "key": "http://[::1]:8080/v6" + } + }, + { + "base": "https://h.example", + "ref": "http://x:port/", + "result": { + "input": "http://x:port/", + "err": true + } + }, + { + "base": "https://h.example", + "ref": "http://x:/p", + "result": { + "input": "http://x:/p", + "err": false, + "string": "http://x:/p", + "scheme": "http", + "host": "x:", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "http://x:/p" + } + }, + { + "base": "https://h.example", + "ref": "/a%20b?c%20d#e%20f", + "result": { + "input": "/a%20b?c%20d#e%20f", + "err": false, + "string": "https://h.example/a%20b?c%20d#e%20f", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a b", + "raw_query": "c%20d", + "fragment": "e f", + "local_path": "a b_c%2520d", + "key": "https://h.example/a%20b?c%20d" + } + }, + { + "base": "https://h.example", + "ref": "http://x/\u0001", + "result": { + "input": "http://x/\u0001", + "err": true + } + }, + { + "base": "https://h.example", + "ref": "/a+b", + "result": { + "input": "/a+b", + "err": false, + "string": "https://h.example/a+b", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/a+b", + "local_path": "a+b", + "key": "https://h.example/a+b" + } + }, + { + "base": "https://h.example", + "ref": "/q?a+b=c d", + "result": { + "input": "/q?a+b=c d", + "err": false, + "string": "https://h.example/q?a+b=c d", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/q", + "raw_query": "a+b=c d", + "local_path": "q_a%2Bb%3Dc+d", + "key": "https://h.example/q?a+b=c d" + } + }, + { + "base": "https://h.example", + "ref": "#", + "result": { + "input": "#", + "err": false, + "string": "https://h.example", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "local_path": "index.html", + "key": "https://h.example" + } + }, + { + "base": "https://h.example", + "ref": "?#", + "result": { + "input": "?#", + "err": false, + "string": "https://h.example?", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "local_path": "index.html", + "key": "https://h.example?" + } + }, + { + "base": "https://h.example", + "ref": "/p#a%23b", + "result": { + "input": "/p#a%23b", + "err": false, + "string": "https://h.example/p#a%23b", + "scheme": "https", + "host": "h.example", + "hostname": "h.example", + "path": "/p", + "fragment": "a#b", + "local_path": "p", + "key": "https://h.example/p" + } + }, + { + "base": "https://h.example", + "ref": "//x/?", + "result": { + "input": "//x/?", + "err": false, + "string": "https://x/?", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "https://x/?" + } + }, + { + "base": "https://h.example", + "ref": "http://x?only", + "result": { + "input": "http://x?only", + "err": false, + "string": "http://x?only", + "scheme": "http", + "host": "x", + "hostname": "x", + "raw_query": "only", + "local_path": "index.html_only", + "key": "http://x?only" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "", + "result": { + "input": "", + "err": false, + "string": "http://h.example:8080/a/b/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a", + "result": { + "input": "a", + "err": false, + "string": "http://h.example:8080/a/b/a", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a", + "local_path": "a/b/a", + "key": "http://h.example:8080/a/b/a" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a/b", + "result": { + "input": "a/b", + "err": false, + "string": "http://h.example:8080/a/b/a/b", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a/b", + "local_path": "a/b/a/b", + "key": "http://h.example:8080/a/b/a/b" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "./a", + "result": { + "input": "./a", + "err": false, + "string": "http://h.example:8080/a/b/a", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a", + "local_path": "a/b/a", + "key": "http://h.example:8080/a/b/a" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "../a", + "result": { + "input": "../a", + "err": false, + "string": "http://h.example:8080/a/a", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/a", + "local_path": "a/a", + "key": "http://h.example:8080/a/a" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "../../../../a", + "result": { + "input": "../../../../a", + "err": false, + "string": "http://h.example:8080/a", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "http://h.example:8080/a" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/a/./b/../c", + "result": { + "input": "/a/./b/../c", + "err": false, + "string": "http://h.example:8080/a/c", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/c", + "local_path": "a/c", + "key": "http://h.example:8080/a/c" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/", + "result": { + "input": "/", + "err": false, + "string": "http://h.example:8080/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "http://h.example:8080/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "//other.example/x", + "result": { + "input": "//other.example/x", + "err": false, + "string": "http://other.example/x", + "scheme": "http", + "host": "other.example", + "hostname": "other.example", + "path": "/x", + "local_path": "x", + "key": "http://other.example/x" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "//OTHER.example:8080", + "result": { + "input": "//OTHER.example:8080", + "err": false, + "string": "http://OTHER.example:8080", + "scheme": "http", + "host": "OTHER.example:8080", + "hostname": "OTHER.example", + "local_path": "index.html", + "key": "http://OTHER.example:8080" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "?q=1", + "result": { + "input": "?q=1", + "err": false, + "string": "http://h.example:8080/a/b/?q=1", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "raw_query": "q=1", + "local_path": "a/b/index.html_q%3D1", + "key": "http://h.example:8080/a/b/?q=1" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "?", + "result": { + "input": "?", + "err": false, + "string": "http://h.example:8080/a/b/?", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/?" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "#frag", + "result": { + "input": "#frag", + "err": false, + "string": "http://h.example:8080/a/b/#frag", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "fragment": "frag", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a#f", + "result": { + "input": "a#f", + "err": false, + "string": "http://h.example:8080/a/b/a#f", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a", + "fragment": "f", + "local_path": "a/b/a", + "key": "http://h.example:8080/a/b/a" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a?x=1\u0026y=2#f", + "result": { + "input": "a?x=1\u0026y=2#f", + "err": false, + "string": "http://h.example:8080/a/b/a?x=1\u0026y=2#f", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a", + "raw_query": "x=1\u0026y=2", + "fragment": "f", + "local_path": "a/b/a_x%3D1%26y%3D2", + "key": "http://h.example:8080/a/b/a?x=1\u0026y=2" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/sp ace", + "result": { + "input": "/sp ace", + "err": false, + "string": "http://h.example:8080/sp%20ace", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/sp ace", + "local_path": "sp ace", + "key": "http://h.example:8080/sp%20ace" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/%41%2fb", + "result": { + "input": "/%41%2fb", + "err": false, + "string": "http://h.example:8080/%41%2fb", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/A/b", + "local_path": "A/b", + "key": "http://h.example:8080/%41%2fb" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/%zz", + "result": { + "input": "/%zz", + "err": true + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/caf%C3%A9", + "result": { + "input": "/caf%C3%A9", + "err": false, + "string": "http://h.example:8080/caf%C3%A9", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example:8080/caf%C3%A9" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/café", + "result": { + "input": "/café", + "err": false, + "string": "http://h.example:8080/caf%C3%A9", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example:8080/caf%C3%A9" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://H.Example:80/A", + "result": { + "input": "http://H.Example:80/A", + "err": false, + "string": "http://H.Example:80/A", + "scheme": "http", + "host": "H.Example:80", + "hostname": "H.Example", + "path": "/A", + "local_path": "A", + "key": "http://H.Example:80/A" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "https://x/a/b/../../..", + "result": { + "input": "https://x/a/b/../../..", + "err": false, + "string": "https://x/", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "https://x/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "HTTP://x", + "result": { + "input": "HTTP://x", + "err": false, + "string": "http://x", + "scheme": "http", + "host": "x", + "hostname": "x", + "local_path": "index.html", + "key": "http://x" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "mailto:a@b", + "result": { + "input": "mailto:a@b", + "err": false, + "string": "mailto:a@b", + "scheme": "mailto", + "local_path": "index.html", + "key": "mailto:a@b" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "javascript:alert(1)", + "result": { + "input": "javascript:alert(1)", + "err": false, + "string": "javascript:alert(1)", + "scheme": "javascript", + "local_path": "index.html", + "key": "javascript:alert(1)" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "ftp://x/y", + "result": { + "input": "ftp://x/y", + "err": false, + "string": "ftp://x/y", + "scheme": "ftp", + "host": "x", + "hostname": "x", + "path": "/y", + "local_path": "y", + "key": "ftp://x/y" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "x:y", + "result": { + "input": "x:y", + "err": false, + "string": "x:y", + "scheme": "x", + "local_path": "index.html", + "key": "x:y" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a:b/c", + "result": { + "input": "a:b/c", + "err": false, + "string": "a:b/c", + "scheme": "a", + "local_path": "index.html", + "key": "a:b/c" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/a?b?c", + "result": { + "input": "/a?b?c", + "err": false, + "string": "http://h.example:8080/a?b?c", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a", + "raw_query": "b?c", + "local_path": "a_b%3Fc", + "key": "http://h.example:8080/a?b?c" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a/..", + "result": { + "input": "a/..", + "err": false, + "string": "http://h.example:8080/a/b/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "a/.", + "result": { + "input": "a/.", + "err": false, + "string": "http://h.example:8080/a/b/a/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/a/", + "local_path": "a/b/a/index.html", + "key": "http://h.example:8080/a/b/a/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": ".", + "result": { + "input": ".", + "err": false, + "string": "http://h.example:8080/a/b/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "..", + "result": { + "input": "..", + "err": false, + "string": "http://h.example:8080/a/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/", + "local_path": "a/index.html", + "key": "http://h.example:8080/a/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/a//b", + "result": { + "input": "/a//b", + "err": false, + "string": "http://h.example:8080/a//b", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a//b", + "local_path": "a//b", + "key": "http://h.example:8080/a//b" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "https://user:pw@x/p", + "result": { + "input": "https://user:pw@x/p", + "err": false, + "string": "https://user:pw@x/p", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "https://user:pw@x/p" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://[::1]:8080/v6", + "result": { + "input": "http://[::1]:8080/v6", + "err": false, + "string": "http://[::1]:8080/v6", + "scheme": "http", + "host": "[::1]:8080", + "hostname": "::1", + "path": "/v6", + "local_path": "v6", + "key": "http://[::1]:8080/v6" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://x:port/", + "result": { + "input": "http://x:port/", + "err": true + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://x:/p", + "result": { + "input": "http://x:/p", + "err": false, + "string": "http://x:/p", + "scheme": "http", + "host": "x:", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "http://x:/p" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/a%20b?c%20d#e%20f", + "result": { + "input": "/a%20b?c%20d#e%20f", + "err": false, + "string": "http://h.example:8080/a%20b?c%20d#e%20f", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a b", + "raw_query": "c%20d", + "fragment": "e f", + "local_path": "a b_c%2520d", + "key": "http://h.example:8080/a%20b?c%20d" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://x/\u0001", + "result": { + "input": "http://x/\u0001", + "err": true + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/a+b", + "result": { + "input": "/a+b", + "err": false, + "string": "http://h.example:8080/a+b", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a+b", + "local_path": "a+b", + "key": "http://h.example:8080/a+b" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/q?a+b=c d", + "result": { + "input": "/q?a+b=c d", + "err": false, + "string": "http://h.example:8080/q?a+b=c d", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/q", + "raw_query": "a+b=c d", + "local_path": "q_a%2Bb%3Dc+d", + "key": "http://h.example:8080/q?a+b=c d" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "#", + "result": { + "input": "#", + "err": false, + "string": "http://h.example:8080/a/b/", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "?#", + "result": { + "input": "?#", + "err": false, + "string": "http://h.example:8080/a/b/?", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/a/b/", + "local_path": "a/b/index.html", + "key": "http://h.example:8080/a/b/?" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "/p#a%23b", + "result": { + "input": "/p#a%23b", + "err": false, + "string": "http://h.example:8080/p#a%23b", + "scheme": "http", + "host": "h.example:8080", + "hostname": "h.example", + "path": "/p", + "fragment": "a#b", + "local_path": "p", + "key": "http://h.example:8080/p" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "//x/?", + "result": { + "input": "//x/?", + "err": false, + "string": "http://x/?", + "scheme": "http", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "http://x/?" + } + }, + { + "base": "http://h.example:8080/a/b/", + "ref": "http://x?only", + "result": { + "input": "http://x?only", + "err": false, + "string": "http://x?only", + "scheme": "http", + "host": "x", + "hostname": "x", + "raw_query": "only", + "local_path": "index.html_only", + "key": "http://x?only" + } + }, + { + "base": "http://h.example/a/b", + "ref": "", + "result": { + "input": "", + "err": false, + "string": "http://h.example/a/b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "local_path": "a/b", + "key": "http://h.example/a/b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a", + "result": { + "input": "a", + "err": false, + "string": "http://h.example/a/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a", + "local_path": "a/a", + "key": "http://h.example/a/a" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a/b", + "result": { + "input": "a/b", + "err": false, + "string": "http://h.example/a/a/b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a/b", + "local_path": "a/a/b", + "key": "http://h.example/a/a/b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "./a", + "result": { + "input": "./a", + "err": false, + "string": "http://h.example/a/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a", + "local_path": "a/a", + "key": "http://h.example/a/a" + } + }, + { + "base": "http://h.example/a/b", + "ref": "../a", + "result": { + "input": "../a", + "err": false, + "string": "http://h.example/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "http://h.example/a" + } + }, + { + "base": "http://h.example/a/b", + "ref": "../../../../a", + "result": { + "input": "../../../../a", + "err": false, + "string": "http://h.example/a", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "local_path": "a", + "key": "http://h.example/a" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/a/./b/../c", + "result": { + "input": "/a/./b/../c", + "err": false, + "string": "http://h.example/a/c", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/c", + "local_path": "a/c", + "key": "http://h.example/a/c" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/", + "result": { + "input": "/", + "err": false, + "string": "http://h.example/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "http://h.example/" + } + }, + { + "base": "http://h.example/a/b", + "ref": "//other.example/x", + "result": { + "input": "//other.example/x", + "err": false, + "string": "http://other.example/x", + "scheme": "http", + "host": "other.example", + "hostname": "other.example", + "path": "/x", + "local_path": "x", + "key": "http://other.example/x" + } + }, + { + "base": "http://h.example/a/b", + "ref": "//OTHER.example:8080", + "result": { + "input": "//OTHER.example:8080", + "err": false, + "string": "http://OTHER.example:8080", + "scheme": "http", + "host": "OTHER.example:8080", + "hostname": "OTHER.example", + "local_path": "index.html", + "key": "http://OTHER.example:8080" + } + }, + { + "base": "http://h.example/a/b", + "ref": "?q=1", + "result": { + "input": "?q=1", + "err": false, + "string": "http://h.example/a/b?q=1", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "raw_query": "q=1", + "local_path": "a/b_q%3D1", + "key": "http://h.example/a/b?q=1" + } + }, + { + "base": "http://h.example/a/b", + "ref": "?", + "result": { + "input": "?", + "err": false, + "string": "http://h.example/a/b?", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "local_path": "a/b", + "key": "http://h.example/a/b?" + } + }, + { + "base": "http://h.example/a/b", + "ref": "#frag", + "result": { + "input": "#frag", + "err": false, + "string": "http://h.example/a/b#frag", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "fragment": "frag", + "local_path": "a/b", + "key": "http://h.example/a/b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a#f", + "result": { + "input": "a#f", + "err": false, + "string": "http://h.example/a/a#f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a", + "fragment": "f", + "local_path": "a/a", + "key": "http://h.example/a/a" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a?x=1\u0026y=2#f", + "result": { + "input": "a?x=1\u0026y=2#f", + "err": false, + "string": "http://h.example/a/a?x=1\u0026y=2#f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a", + "raw_query": "x=1\u0026y=2", + "fragment": "f", + "local_path": "a/a_x%3D1%26y%3D2", + "key": "http://h.example/a/a?x=1\u0026y=2" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/sp ace", + "result": { + "input": "/sp ace", + "err": false, + "string": "http://h.example/sp%20ace", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/sp ace", + "local_path": "sp ace", + "key": "http://h.example/sp%20ace" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/%41%2fb", + "result": { + "input": "/%41%2fb", + "err": false, + "string": "http://h.example/%41%2fb", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/A/b", + "local_path": "A/b", + "key": "http://h.example/%41%2fb" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/%zz", + "result": { + "input": "/%zz", + "err": true + } + }, + { + "base": "http://h.example/a/b", + "ref": "/caf%C3%A9", + "result": { + "input": "/caf%C3%A9", + "err": false, + "string": "http://h.example/caf%C3%A9", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example/caf%C3%A9" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/café", + "result": { + "input": "/café", + "err": false, + "string": "http://h.example/caf%C3%A9", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/café", + "local_path": "café", + "key": "http://h.example/caf%C3%A9" + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://H.Example:80/A", + "result": { + "input": "http://H.Example:80/A", + "err": false, + "string": "http://H.Example:80/A", + "scheme": "http", + "host": "H.Example:80", + "hostname": "H.Example", + "path": "/A", + "local_path": "A", + "key": "http://H.Example:80/A" + } + }, + { + "base": "http://h.example/a/b", + "ref": "https://x/a/b/../../..", + "result": { + "input": "https://x/a/b/../../..", + "err": false, + "string": "https://x/", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "https://x/" + } + }, + { + "base": "http://h.example/a/b", + "ref": "HTTP://x", + "result": { + "input": "HTTP://x", + "err": false, + "string": "http://x", + "scheme": "http", + "host": "x", + "hostname": "x", + "local_path": "index.html", + "key": "http://x" + } + }, + { + "base": "http://h.example/a/b", + "ref": "mailto:a@b", + "result": { + "input": "mailto:a@b", + "err": false, + "string": "mailto:a@b", + "scheme": "mailto", + "local_path": "index.html", + "key": "mailto:a@b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "javascript:alert(1)", + "result": { + "input": "javascript:alert(1)", + "err": false, + "string": "javascript:alert(1)", + "scheme": "javascript", + "local_path": "index.html", + "key": "javascript:alert(1)" + } + }, + { + "base": "http://h.example/a/b", + "ref": "ftp://x/y", + "result": { + "input": "ftp://x/y", + "err": false, + "string": "ftp://x/y", + "scheme": "ftp", + "host": "x", + "hostname": "x", + "path": "/y", + "local_path": "y", + "key": "ftp://x/y" + } + }, + { + "base": "http://h.example/a/b", + "ref": "x:y", + "result": { + "input": "x:y", + "err": false, + "string": "x:y", + "scheme": "x", + "local_path": "index.html", + "key": "x:y" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a:b/c", + "result": { + "input": "a:b/c", + "err": false, + "string": "a:b/c", + "scheme": "a", + "local_path": "index.html", + "key": "a:b/c" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/a?b?c", + "result": { + "input": "/a?b?c", + "err": false, + "string": "http://h.example/a?b?c", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a", + "raw_query": "b?c", + "local_path": "a_b%3Fc", + "key": "http://h.example/a?b?c" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a/..", + "result": { + "input": "a/..", + "err": false, + "string": "http://h.example/a/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/", + "local_path": "a/index.html", + "key": "http://h.example/a/" + } + }, + { + "base": "http://h.example/a/b", + "ref": "a/.", + "result": { + "input": "a/.", + "err": false, + "string": "http://h.example/a/a/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/a/", + "local_path": "a/a/index.html", + "key": "http://h.example/a/a/" + } + }, + { + "base": "http://h.example/a/b", + "ref": ".", + "result": { + "input": ".", + "err": false, + "string": "http://h.example/a/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/", + "local_path": "a/index.html", + "key": "http://h.example/a/" + } + }, + { + "base": "http://h.example/a/b", + "ref": "..", + "result": { + "input": "..", + "err": false, + "string": "http://h.example/", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/", + "local_path": "index.html", + "key": "http://h.example/" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/a//b", + "result": { + "input": "/a//b", + "err": false, + "string": "http://h.example/a//b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a//b", + "local_path": "a//b", + "key": "http://h.example/a//b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "https://user:pw@x/p", + "result": { + "input": "https://user:pw@x/p", + "err": false, + "string": "https://user:pw@x/p", + "scheme": "https", + "host": "x", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "https://user:pw@x/p" + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://[::1]:8080/v6", + "result": { + "input": "http://[::1]:8080/v6", + "err": false, + "string": "http://[::1]:8080/v6", + "scheme": "http", + "host": "[::1]:8080", + "hostname": "::1", + "path": "/v6", + "local_path": "v6", + "key": "http://[::1]:8080/v6" + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://x:port/", + "result": { + "input": "http://x:port/", + "err": true + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://x:/p", + "result": { + "input": "http://x:/p", + "err": false, + "string": "http://x:/p", + "scheme": "http", + "host": "x:", + "hostname": "x", + "path": "/p", + "local_path": "p", + "key": "http://x:/p" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/a%20b?c%20d#e%20f", + "result": { + "input": "/a%20b?c%20d#e%20f", + "err": false, + "string": "http://h.example/a%20b?c%20d#e%20f", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a b", + "raw_query": "c%20d", + "fragment": "e f", + "local_path": "a b_c%2520d", + "key": "http://h.example/a%20b?c%20d" + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://x/\u0001", + "result": { + "input": "http://x/\u0001", + "err": true + } + }, + { + "base": "http://h.example/a/b", + "ref": "/a+b", + "result": { + "input": "/a+b", + "err": false, + "string": "http://h.example/a+b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a+b", + "local_path": "a+b", + "key": "http://h.example/a+b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/q?a+b=c d", + "result": { + "input": "/q?a+b=c d", + "err": false, + "string": "http://h.example/q?a+b=c d", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/q", + "raw_query": "a+b=c d", + "local_path": "q_a%2Bb%3Dc+d", + "key": "http://h.example/q?a+b=c d" + } + }, + { + "base": "http://h.example/a/b", + "ref": "#", + "result": { + "input": "#", + "err": false, + "string": "http://h.example/a/b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "local_path": "a/b", + "key": "http://h.example/a/b" + } + }, + { + "base": "http://h.example/a/b", + "ref": "?#", + "result": { + "input": "?#", + "err": false, + "string": "http://h.example/a/b?", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/a/b", + "local_path": "a/b", + "key": "http://h.example/a/b?" + } + }, + { + "base": "http://h.example/a/b", + "ref": "/p#a%23b", + "result": { + "input": "/p#a%23b", + "err": false, + "string": "http://h.example/p#a%23b", + "scheme": "http", + "host": "h.example", + "hostname": "h.example", + "path": "/p", + "fragment": "a#b", + "local_path": "p", + "key": "http://h.example/p" + } + }, + { + "base": "http://h.example/a/b", + "ref": "//x/?", + "result": { + "input": "//x/?", + "err": false, + "string": "http://x/?", + "scheme": "http", + "host": "x", + "hostname": "x", + "path": "/", + "local_path": "index.html", + "key": "http://x/?" + } + }, + { + "base": "http://h.example/a/b", + "ref": "http://x?only", + "result": { + "input": "http://x?only", + "err": false, + "string": "http://x?only", + "scheme": "http", + "host": "x", + "hostname": "x", + "raw_query": "only", + "local_path": "index.html_only", + "key": "http://x?only" + } + } + ] +}