Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(url): preserve file URL hosts and leading path segments
ldm04c31485151b1
diff --git a/Cargo.lock b/Cargo.lockindex c57bb3b37..857ff3e13 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -193,7 +193,7 @@ dependencies = ["axum-core","base64 0.22.1","bytes",- "form_urlencoded",+ "form_urlencoded 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)","futures-util","http","http-body",@@ -204,7 +204,7 @@ dependencies = ["matchit","memchr","mime",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","pin-project-lite","serde_core","serde_json",@@ -1434,7 +1434,15 @@ version = "1.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"dependencies = [- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)",+]++[[package]]+name = "form_urlencoded"+version = "1.2.2"+source = "git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161#ca9fcad2b99f20c30c28a618ad78410f6fd31161"+dependencies = [+ "percent-encoding 2.3.2 (git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161)",][[package]]@@ -1988,6 +1996,16 @@ dependencies = ["utf8_iter",]+[[package]]+name = "idna"+version = "1.1.0"+source = "git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161#ca9fcad2b99f20c30c28a618ad78410f6fd31161"+dependencies = [+ "idna_adapter",+ "smallvec",+ "utf8_iter",+]+[[package]]name = "idna_adapter"version = "1.2.1"@@ -2038,7 +2056,7 @@ dependencies = ["boxed_error","deno_error","indexmap",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","serde","serde_json","thiserror",@@ -2514,7 +2532,7 @@ version = "0.22.1"dependencies = ["cookie","document-features",- "idna",+ "idna 1.1.0 (registry+https://github.com/rust-lang/crates.io-index)","indexmap","psl-types","time",@@ -2616,7 +2634,7 @@ dependencies = ["moli-html-input-temporal","moli-html-input-type","moli-html2md",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","selectors","serde","serde_json",@@ -2975,7 +2993,7 @@ dependencies = ["moli-v8-platform","moli-web-mime","parking_lot",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","sanitize-filename","serde","serde_json",@@ -3022,7 +3040,7 @@ dependencies = ["moli-protocol-webdriver-classic","moli-trace","parking_lot",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","png","serde_json","socket2",@@ -3141,7 +3159,7 @@ dependencies = ["moli-window-features","moli-xpath","parking_lot",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","png","precomputed-hash","selectors",@@ -3193,7 +3211,7 @@ dependencies = ["moli-css-parse","moli-dom","parking_lot",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","precomputed-hash","selectors","stylo",@@ -3245,7 +3263,7 @@ dependencies = ["moli-opfs","moli-storage-key","parking_lot",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)","serde","serde_json",]@@ -3834,7 +3852,7 @@ checksum = "1d7ccb0e8e7c9f1fb75e0700b2c75d9d854e534a7a356b13d2936893651f2b98"dependencies = ["cow-utils","oxc-miette",- "percent-encoding",+ "percent-encoding 2.3.2 (registry+https://github.com/rust-lang/crates.io-index)",][[package]]@@ -4065,6 +4083,11 @@ version = "2.3.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"+[[package]]+name = "percent-encoding"+version = "2.3.2"+source = "git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161#ca9fcad2b99f20c30c28a618ad78410f6fd31161"+[[package]]name = "phf"version = "0.13.1"@@ -4758,7 +4781,7 @@ version = "0.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"dependencies = [- "form_urlencoded",+ "form_urlencoded 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)","itoa","ryu","serde",@@ -5809,12 +5832,11 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"[[package]]name = "url"version = "2.5.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed"+source = "git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161#ca9fcad2b99f20c30c28a618ad78410f6fd31161"dependencies = [- "form_urlencoded",- "idna",- "percent-encoding",+ "form_urlencoded 1.2.2 (git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161)",+ "idna 1.1.0 (git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161)",+ "percent-encoding 2.3.2 (git+https://github.com/lexmount/rust-url?rev=ca9fcad2b99f20c30c28a618ad78410f6fd31161)","serde","serde_derive",]diff --git a/Cargo.toml b/Cargo.tomlindex df72c53ed..63ec86ac4 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -100,6 +100,7 @@ license = "MIT OR Apache-2.0"[patch.crates-io]parley = { git = "https://github.com/lexmount/parley", rev = "d0f5452487b872562102fde2c133d445272bcccc" }fontique = { git = "https://github.com/lexmount/parley", rev = "d0f5452487b872562102fde2c133d445272bcccc" }+url = { git = "https://github.com/lexmount/rust-url", rev = "ca9fcad2b99f20c30c28a618ad78410f6fd31161" }xml5ever = { path = "vendor/xml5ever-0.39.0" }curl = { git = "https://github.com/lexmount/curl-rust", rev = "a0ea59f5ca1b1e4bc8627a28c29463ac96aeb758" }curl-sys = { git = "https://github.com/lexmount/curl-rust", rev = "a0ea59f5ca1b1e4bc8627a28c29463ac96aeb758" }diff --git a/moli-renderer-v8/src/script_vm/tests/url_components.rs b/moli-renderer-v8/src/script_vm/tests/url_components.rsindex 8aa0ea308..90b46e583 100644--- a/moli-renderer-v8/src/script_vm/tests/url_components.rs+++ b/moli-renderer-v8/src/script_vm/tests/url_components.rs@@ -352,6 +352,64 @@ for (const tag of ['a', 'area']) {);}+#[test]+fn file_url_components_preserve_hosts_drive_letters_and_empty_path_segments() {+ assert_url_components(+ r#"+for (const [input, expected] of [+ ['file://server///?q#f', 'file://server///?q#f'],+ ['file://server/C|/path', 'file://server/C:/path'],+ ['file:///w|/m', 'file:///w:/m'],+ ['file://localhost////foo', 'file://////foo'],+]) {+ for (const [name, create] of factories) {+ const object = create(input);+ assert(object.href === expected, `${name}: parsing ${input}`);+ assert(object.origin === 'null', `${name}: file origin stays opaque`);+ object.href = object.href;+ assert(object.href === expected, `${name}: file serialization round-trips`);+ }+}+checkSetter('file://monkey/', 'pathname', '\\\\', 'file://monkey//');+checkSetter('file:///unicorn', 'pathname', '//\\/', 'file://////');+checkSetter('file:///unicorn', 'pathname', '//monkey/..//', 'file://///');+checkSetter('file://host/old?q#f', 'pathname', 'C|/new', 'file://host/C:/new?q#f');+"#,+ );+}++#[test]+fn file_url_relative_resolution_matches_document_base_and_static_url_parsing() {+ assert_url_components(+ r#"+for (const [input, base, expected] of [+ ['/', 'file://host/C:/a/b', 'file://host/C:/'],+ ['C|/new', 'file://host/D:/a/b', 'file://host/C:/new'],+ ['/..//share//file', 'file://host/path', 'file://host//share//file'],+ ['..', 'file://host/a/C:/', 'file://host/a/'],+]) {+ for (const constructor of [URL, frame.contentWindow.URL]) {+ assert(new constructor(input, base).href === expected, 'relative URL constructor');+ assert(constructor.parse(input, base).href === expected, 'relative URL.parse');+ assert(constructor.canParse(input, base), 'relative URL.canParse');+ }+ for (const owner of [document, frame.contentDocument,+ document.implementation.createHTMLDocument('file base')]) {+ const baseElement = owner.head.appendChild(owner.createElement('base'));+ baseElement.href = base;+ for (const tag of ['a', 'area']) {+ const link = owner.body.appendChild(owner.createElement(tag));+ link.href = input;+ assert(link.href === expected, `${tag}: file document base`);+ link.remove();+ }+ baseElement.remove();+ }+}+"#,+ );+}+#[test]fn location_components_empty_getters_preserve_document_urls() {assert_url_components(diff --git a/moli-url/src/file_url.rs b/moli-url/src/file_url.rsnew file mode 100644index 000000000..9d5ceeb90--- /dev/null+++ b/moli-url/src/file_url.rs@@ -0,0 +1,93 @@+use url::Url;++fn assert_file_url(input: &str, base: Option<&str>, expected: &str) {+ let base = base.map(|value| Url::parse(value).unwrap());+ let url = Url::options().base_url(base.as_ref()).parse(input).unwrap();+ assert_eq!(url.as_str(), expected, "input {input:?}, base {base:?}");+ assert_eq!(+ Url::parse(url.as_str()).unwrap(),+ url,+ "serialization must round-trip"+ );+ assert!(url.username().is_empty());+ assert_eq!(url.password(), None);+ assert_eq!(url.port(), None);+}++#[test]+fn file_urls_preserve_leading_empty_path_segments() {+ for (input, expected) in [+ ("file://server///", "file://server///"),+ ("file:////one/two", "file:////one/two"),+ ("file://localhost////foo", "file://////foo"),+ (r"file:\\\\", "file:////"),+ ("file://localhost//a//../..//foo", "file://///foo"),+ ("file:.//p", "file:////p"),+ ("file:/.//p", "file:////p"),+ ("file:////?q#f", "file:////?q#f"),+ ] {+ assert_file_url(input, None, expected);+ }+ for (input, base, expected) in [+ ("/////mouse", "file:///elephant", "file://///mouse"),+ (r"\/localhost//pig", "file://lion/", "file:////pig"),+ (+ "/..//localhost//pig",+ "file://lion/",+ "file://lion//localhost//pig",+ ),+ ("file:///.//", "file:////", "file:////"),+ ] {+ assert_file_url(input, Some(base), expected);+ }+}++#[test]+fn file_urls_normalize_drive_letters_without_removing_hosts() {+ for host in ["", "server", "127.0.0.1", "[::1]"] {+ for path in ["C:/", "C|/", "C|", "C|?q#f", "C:/dir/file"] {+ let input = format!("file://{host}/{path}");+ let expected = format!("file://{host}/{}", path.replacen('|', ":", 1));+ assert_file_url(&input, None, &expected);+ }+ }+ assert_file_url("file:///w|/m", None, "file:///w:/m");+ assert_file_url("file:///dir/C|/file", None, "file:///dir/C|/file");+ assert_file_url("file:////C|/file", None, "file:////C|/file");+}++#[test]+fn file_url_relative_paths_inherit_hosts_and_only_preserve_root_drive_letters() {+ for (input, base, expected) in [+ ("/", "file://h/C:/a/b", "file://h/C:/"),+ ("/next", "file://h/C:/a/b", "file://h/C:/next"),+ ("C|", "file://host/dir/file", "file://host/C:"),+ ("C|/", "file://host/D:/dir/file", "file://host/C:/"),+ ("C|\n/", "file://host/dir/file", "file://host/C:/"),+ (r"C|\", "file://host/dir/file", "file://host/C:/"),+ ("/c:/foo/bar", "file://host/path", "file://host/c:/foo/bar"),+ ("file:C:/", "file://host/", "file://host/C:/"),+ ("file:/C:/", "file://host/", "file://host/C:/"),+ ("..", "file://x/C:/", "file://x/C:/"),+ ("../../..", "file://x/C:/a/b", "file://x/C:/"),+ ("..", "file://x/a/C:/", "file://x/a/"),+ ] {+ assert_file_url(input, Some(base), expected);+ }+}++#[test]+fn file_url_pathname_setters_preserve_empty_segments_and_hosts() {+ for (input, value, expected) in [+ ("file://monkey/", r"\\", "file://monkey//"),+ ("file:///unicorn", r"//\/", "file://////"),+ ("file:///unicorn", "//monkey/..//", "file://///"),+ ("file://host/old?q#f", "C|/new", "file://host/C:/new?q#f"),+ ("file://host/old", "", "file://host/"),+ ] {+ let mut url = Url::parse(input).unwrap();+ crate::components::set_pathname(&mut url, value);+ assert_eq!(url.as_str(), expected, "pathname = {value:?}");+ assert_eq!(Url::parse(url.as_str()).unwrap(), url);+ }+}diff --git a/moli-url/src/lib.rs b/moli-url/src/lib.rsindex c2d3398d9..788dd94d7 100644--- a/moli-url/src/lib.rs+++ b/moli-url/src/lib.rs@@ -2,6 +2,9 @@ pub mod components;pub mod origin;pub mod search_params;+#[cfg(test)]+mod file_url;+pub use origin::is_about_srcdoc;pub use origin::{