The headless browser for AI agents and web scraping
fix(js): resolve relative URLs against the document base URL
aech98d37eb93feb
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.jsindex d7e69040..8cceccde 100644--- a/crates/obscura-js/js/bootstrap.js+++ b/crates/obscura-js/js/bootstrap.js@@ -1920,20 +1920,7 @@ class Node {get ownerDocument() { return globalThis.document; }// https://dom.spec.whatwg.org/#dom-node-baseuriget baseURI() {- try {- const doc = globalThis.document;- const docUrl = (doc && doc.URL) || "";- const baseEl = (doc && doc.querySelector) ? doc.querySelector("base[href]") : null;- if (baseEl) {- const href = baseEl.getAttribute("href");- if (href) {- return docUrl ? new URL(href, docUrl).href : href;- }- }- return docUrl;- } catch (e) {- return "";- }+ try { return _documentBase(); } catch (e) { return ""; }}get textContent() { return _domParse("text_content", this._nid) ?? ""; }set textContent(v) {@@ -2492,10 +2479,25 @@ function _applyDocQueryEncoding(u) {return u;}+// The base for relative URLs. <base href> overrides the document URL, so an app in a sub-path+// requests "chunk-A.js" under its current route and gets 404.+// https://html.spec.whatwg.org/multipage/urls-and-fetching.html#document-base-url+// Returns "" when there is no document, so each call site keeps its own fallback.+function _documentBase() {+ // history.pushState moves the document URL without reaching the Rust side. Then the base must+ // be built here, or every relative URL resolves against the pre-routing address.+ const virtual = globalThis.__virtualUrl;+ if (virtual) {+ const raw = _domParse("document_base_href");+ if (!raw) return virtual;+ try { return new URL(raw, virtual).href; } catch (e) { return virtual; }+ }+ return _domParse("document_base_url") || _domParse("document_url") || "";+}// HTMLHyperlinkElementUtils helpers (the <a>/<area> URL-decomposition members).// The element's href attribute is parsed against the document base URL via the// WHATWG url op; component getters read it, setters rewrite the href attribute.-function _anchorBase() { return _domParse("document_url") || "about:blank"; }+function _anchorBase() { return _documentBase() || "about:blank"; }function _elemHrefURL(el) {const raw = el.getAttribute('href');if (raw === null || raw === undefined) return null;@@ -3834,6 +3836,15 @@ class Element extends Node {const r = _urlResolveOp(raw, _anchorBase());return r !== null ? r : raw;}+ if (ln === 'base') {+ // https://html.spec.whatwg.org/multipage/semantics.html#dom-base-href+ // Against the fallback base URL, not the document base URL: a base element is not affected+ // by other base elements or itself. Applications read this to determine their own base.+ const raw = this.getAttribute('href');+ if (raw === null) return '';+ const r = _urlResolveOp(raw, _domParse("document_url") || "about:blank");+ return r !== null ? r : raw;+ }return this.getAttribute("href") || "";}set href(v) { this.setAttribute("href", v); }@@ -3881,7 +3892,7 @@ class Element extends Node {// value (issue #255). getAttribute("src") still returns the literal.const v = this.getAttribute("src");if (!v) return "";- try { return new URL(v, globalThis.location?.href || "about:blank").href; }+ try { return new URL(v, _documentBase() || "about:blank").href; }catch (e) { return v; }}set src(v) {@@ -3980,8 +3991,9 @@ class Element extends Node {return this._iframeWin;}get action() {+ // A missing action falls back to the document URL, a present one resolves against the base.const action = this.getAttribute("action") || _domParse("document_url") || "";- try { return new URL(action, _domParse("document_url") || "about:blank").href; } catch(e) { return action; }+ try { return new URL(action, _documentBase() || "about:blank").href; } catch(e) { return action; }}set action(v) { this.setAttribute("action", v); }get method() { return this.getAttribute("method") || "get"; }@@ -6108,7 +6120,7 @@ function _resolveUrl(url) {url = String(url);if (!url) return url;if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('about:')) return url;- try { return new URL(url, _domParse("document_url") || "about:blank").href; } catch(e) { return url; }+ try { return new URL(url, _documentBase() || "about:blank").href; } catch(e) { return url; }}// `__virtualUrl` is set by `history.pushState`/`replaceState` (and cleared by// any real navigation). When set, `location.href` and friends read it insteaddiff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex 7597a997..813d4005 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -172,6 +172,10 @@ pub struct ObscuraState {/// completions use this to discard bytes and lifecycle results belonging/// to a navigation that has already been replaced.pub document_generation: u64,+ /// Cached document base URL. Computing it walks the tree and runs the selector engine, and+ /// the JS layer asks for it on every relative URL, including the URL parts of `<a>`.+ /// Interior mutability so the read path keeps its shared borrow.+ pub base_url_cache: RefCell<Option<BaseUrlCache>>,/// Final image/font-aware layout shared by CSSOM geometry and screenshots./// DOM/style/viewport changes clear this value but retain resource bytes.#[cfg(feature = "render")]@@ -311,6 +315,7 @@ impl ObscuraState {page_in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),activity_generation: 0,document_generation: 0,+ base_url_cache: RefCell::new(None),#[cfg(feature = "render")]prepared_render: None,#[cfg(feature = "render")]@@ -1297,6 +1302,18 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->serde_json::to_string(&title).unwrap_or("\"\"".into())}"document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()),+ // The base for relative URLs. It differs from document_url exactly when the page carries+ // a <base href>, and that is the point: HTML resolves against the base, not the document.+ "document_base_url" => serde_json::to_string(+ &document_base_url_memoized(&gs).unwrap_or_else(|| gs.url.clone()),+ )+ .unwrap_or("\"\"".into()),+ // The unresolved attribute. After history.pushState only JS knows the URL, so only JS+ // can resolve a relative base against it.+ "document_base_href" => {+ serde_json::to_string(&document_base_href_memoized(&gs).unwrap_or_default())+ .unwrap_or("\"\"".into())+ }"document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),"document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),"document_element" => {@@ -4601,7 +4618,8 @@ fn op_waapi_control(changed}-#[cfg(feature = "render")]+// Not tied to `render`: the JS layer resolves every relative URL through here, in all build+// variants.pub(crate) fn document_base_url(state: &ObscuraState) -> Option<String> {let document_url = url::Url::parse(&state.url).ok()?;let base_href = state.dom.as_ref().and_then(|dom| {@@ -4614,11 +4632,74 @@ pub(crate) fn document_base_url(state: &ObscuraState) -> Option<String> {})});match base_href {- Some(href) => document_url.join(&href).ok().map(|url| url.to_string()),+ // https://html.spec.whatwg.org/multipage/semantics.html#set-the-frozen-base-url+ // A data: or javascript: base falls back to the document URL. Accepting it would instead+ // make every later relative resolution fail.+ Some(href) => match document_url.join(&href) {+ Ok(base) if base.scheme() != "data" && base.scheme() != "javascript" => {+ Some(base.to_string())+ }+ _ => Some(document_url.to_string()),+ },None => Some(document_url.to_string()),}}+/// The raw `href` attribute of the first `<base href>`, unresolved. The JS layer needs it after+/// `history.pushState`: the document URL has moved, only JS knows the new one, so only JS can+/// resolve a relative base against it.+fn document_base_href(state: &ObscuraState) -> Option<String> {+ state.dom.as_ref().and_then(|dom| {+ dom.query_selector("base[href]")+ .ok()+ .flatten()+ .and_then(|id| {+ dom.get_node(id)+ .and_then(|node| node.get_attribute("href").map(str::to_string))+ })+ })+}++/// What the cached values were computed from. If any of the three changes, they are recomputed.+pub struct BaseUrlCache {+ activity_generation: u64,+ document_generation: u64,+ url: String,+ resolved: Option<String>,+ raw_href: Option<String>,+}++/// Both base values behind a cache. Uncached, each one walks the tree and runs the selector+/// engine, which would make `a.href` an O(nodes) read.+fn base_values_memoized(state: &ObscuraState) -> (Option<String>, Option<String>) {+ if let Some(cached) = state.base_url_cache.borrow().as_ref() {+ if cached.activity_generation == state.activity_generation+ && cached.document_generation == state.document_generation+ && cached.url == state.url+ {+ return (cached.resolved.clone(), cached.raw_href.clone());+ }+ }+ let resolved = document_base_url(state);+ let raw_href = document_base_href(state);+ *state.base_url_cache.borrow_mut() = Some(BaseUrlCache {+ activity_generation: state.activity_generation,+ document_generation: state.document_generation,+ url: state.url.clone(),+ resolved: resolved.clone(),+ raw_href: raw_href.clone(),+ });+ (resolved, raw_href)+}++pub(crate) fn document_base_url_memoized(state: &ObscuraState) -> Option<String> {+ base_values_memoized(state).0+}++pub(crate) fn document_base_href_memoized(state: &ObscuraState) -> Option<String> {+ base_values_memoized(state).1+}+#[cfg(feature = "render")]pub(crate) fn ensure_prepared_render(state: &mut ObscuraState,diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rsindex f36f742e..80a894e9 100644--- a/crates/obscura-js/src/runtime.rs+++ b/crates/obscura-js/src/runtime.rs@@ -13962,6 +13962,320 @@ mod tests {);}+ /// Document URL two levels deep, base one level deep and the origin root on neither of the+ /// two. The three possible bases land on three different paths, and one assert keeps them+ /// apart:+ /// document base URL -> /app/data/x.json (the spec)+ /// document URL -> /deep/data/x.json (the bug)+ /// origin root -> /data/x.json (a base "/" would hide this one)+ fn setup_runtime_at_deep_url(html: &str) -> ObscuraJsRuntime {+ let dom = parse_html(html);+ let mut rt = ObscuraJsRuntime::new();+ rt.set_dom(dom);+ rt.set_url("http://example.com/deep/page");+ rt.run_page_init();+ rt+ }++ const BASE_HREF_PAGE: &str = r#"<html><head><base href="/app/"></head><body>+ <a id="link" href="data/x.json"></a>+ <form id="form" action="submit"></form>+ <script id="script" src="chunk.js"></script>+ </body></html>"#;++ #[test]+ fn base_href_governs_dom_url_reflection() {+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);++ // One evaluate, one assert: a bundle of assert_eq! aborts at the first failure and+ // reports one broken path while hiding the others.+ let seen = rt+ .evaluate(+ r#"return [+ document.baseURI,+ document.getElementById('link').href,+ document.getElementById('form').action,+ document.getElementById('script').src,+ ]"#,+ )+ .unwrap();+ assert_eq!(+ seen.as_array().unwrap(),+ &vec![+ serde_json::json!("http://example.com/app/"),+ serde_json::json!("http://example.com/app/data/x.json"),+ serde_json::json!("http://example.com/app/submit"),+ serde_json::json!("http://example.com/app/chunk.js"),+ ]+ );+ }++ #[test]+ fn a_relative_location_assignment_follows_the_base() {+ // _resolveUrl serves location.href=, assign, replace and window.location=. Of the six+ // call sites it has the largest external effect, so it gets its own test.+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);+ rt.evaluate("location.href = 'users/42'").unwrap();++ let landed = rt.evaluate("location.href").unwrap();+ assert_eq!(landed.as_str().unwrap(), "http://example.com/app/users/42");+ }++ #[test]+ fn without_base_href_resolution_stays_on_the_document_url() {+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head></head><body>+ <a id="link" href="data/x.json"></a>+ <form id="form" action="submit"></form>+ <script id="script" src="chunk.js"></script>+ </body></html>"#,+ );++ // Every call site, not just the anchor: a site that resolves to the origin root instead+ // of the document URL slips through on a page without a base only when nobody is looking.+ let seen = rt+ .evaluate(+ r#"return [+ document.baseURI,+ document.getElementById('link').href,+ document.getElementById('form').action,+ document.getElementById('script').src,+ ]"#,+ )+ .unwrap();+ assert_eq!(+ seen.as_array().unwrap(),+ &vec![+ serde_json::json!("http://example.com/deep/page"),+ serde_json::json!("http://example.com/deep/data/x.json"),+ serde_json::json!("http://example.com/deep/submit"),+ serde_json::json!("http://example.com/deep/chunk.js"),+ ]+ );+ }++ #[test]+ fn base_element_href_reflects_the_resolved_url() {+ // Resolved against the fallback base URL, i.e. the document URL and not /app/.+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);+ let href = rt.evaluate("document.querySelector('base').href").unwrap();+ assert_eq!(href.as_str().unwrap(), "http://example.com/app/");++ let mut relative = setup_runtime_at_deep_url(+ r#"<html><head><base href="assets/"></head><body></body></html>"#,+ );+ let href = relative+ .evaluate("document.querySelector('base').href")+ .unwrap();+ assert_eq!(href.as_str().unwrap(), "http://example.com/deep/assets/");+ }++ #[test]+ fn the_first_base_with_an_href_wins() {+ // Tree order, and a <base> without href does not count.+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head><base><base href="/a/"><base href="/b/"></head><body>+ <a id="link" href="x.json"></a>+ </body></html>"#,+ );+ let link = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(link.as_str().unwrap(), "http://example.com/a/x.json");+ }++ #[test]+ fn an_empty_base_href_resolves_to_the_document_url() {+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head><base href=""></head><body>+ <a id="link" href="x.json"></a>+ </body></html>"#,+ );+ let link = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(link.as_str().unwrap(), "http://example.com/deep/x.json");+ }++ #[test]+ fn a_cross_origin_base_moves_the_target_but_not_the_page_origin() {+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head><base href="https://cdn.example.net/v2/"></head><body>+ <a id="link" href="x.json"></a>+ </body></html>"#,+ );+ let seen = rt+ .evaluate("return [document.getElementById('link').href, location.origin]")+ .unwrap();+ assert_eq!(+ seen.as_array().unwrap(),+ &vec![+ serde_json::json!("https://cdn.example.net/v2/x.json"),+ serde_json::json!("http://example.com"),+ ]+ );+ }++ #[test]+ fn base_href_rejects_a_data_url_base() {+ // https://html.spec.whatwg.org/multipage/semantics.html#set-the-frozen-base-url+ // Accepting it would make every later relative resolution fail instead of falling back.+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head><base href="data:text/html,x"></head><body>+ <a id="link" href="data/x.json"></a>+ </body></html>"#,+ );++ let base_uri = rt.evaluate("document.baseURI").unwrap();+ assert_eq!(base_uri.as_str().unwrap(), "http://example.com/deep/page");++ let link = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(link.as_str().unwrap(), "http://example.com/deep/data/x.json");+ }++ #[test]+ fn base_resolution_follows_the_url_set_by_push_state() {+ // pushState changes the document URL, and without <base> that very URL is the base.+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head></head><body><a id="link" href="x.json"></a></body></html>"#,+ );+ rt.evaluate("history.pushState({}, '', '/other/route')").unwrap();++ let link = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(link.as_str().unwrap(), "http://example.com/other/x.json");+ }++ #[test]+ fn a_relative_base_href_resolves_against_the_push_state_url() {+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head><base href="assets/"></head><body>+ <a id="link" href="x.json"></a>+ </body></html>"#,+ );+ rt.evaluate("history.pushState({}, '', '/other/route')").unwrap();++ let link = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(link.as_str().unwrap(), "http://example.com/other/assets/x.json");+ }++ /// Guards the cache in `document_base_url_memoized`. Without it, each of these reads walked+ /// the tree and ran the selector engine, and `a.href` went from a field read to O(nodes).+ /// The bound is deliberately loose: it should catch the regression, not watch the allocator.+ #[test]+ fn anchor_href_reads_do_not_scale_with_document_size() {+ let mut body = String::from(r#"<html><head></head><body><a id="link" href="x.json"></a>"#);+ for i in 0..4000 {+ body.push_str(&format!("<div id=\"n{i}\"><span>text</span></div>"));+ }+ body.push_str("</body></html>");+ let mut rt = setup_runtime_at_deep_url(&body);++ let elapsed = rt+ .evaluate(+ r#"+ const link = document.getElementById('link');+ const started = Date.now();+ for (let i = 0; i < 2000; i++) { link.href; }+ return Date.now() - started;+ "#,+ )+ .unwrap();+ let ms = elapsed.as_f64().expect("elapsed ms");+ assert!(+ ms < 500.0,+ "2000 a.href reads on a document with 12000 nodes took {ms} ms, the base query is not cached"+ );+ }++ #[test]+ fn the_base_memo_still_sees_a_base_element_added_later() {+ let mut rt = setup_runtime_at_deep_url(+ r#"<html><head></head><body><a id="link" href="x.json"></a></body></html>"#,+ );+ let before = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(before.as_str().unwrap(), "http://example.com/deep/x.json");++ rt.evaluate(+ r#"+ const base = document.createElement('base');+ base.setAttribute('href', '/');+ document.head.appendChild(base);+ "#,+ )+ .unwrap();++ let after = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(after.as_str().unwrap(), "http://example.com/x.json");+ }++ #[test]+ fn the_base_memo_notices_a_changed_href_attribute() {+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);+ let before = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(before.as_str().unwrap(), "http://example.com/app/data/x.json");++ rt.evaluate("document.querySelector('base').setAttribute('href', '/other/')")+ .unwrap();++ let after = rt+ .evaluate("document.getElementById('link').href")+ .unwrap();+ assert_eq!(after.as_str().unwrap(), "http://example.com/other/data/x.json");+ }++ #[tokio::test(flavor = "current_thread")]+ async fn base_href_governs_fetch_and_xhr_targets() {+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const originalFetchOp = Deno.core.ops.op_fetch_url;+ const seen = [];+ try {+ Deno.core.ops.op_fetch_url = (url) => {+ seen.push(url);+ return JSON.stringify({ status: 200, headers: {}, body: "{}", url });+ };+ await fetch("data/x.json");+ const xhr = new XMLHttpRequest();+ xhr.open("GET", "data/y.json");+ xhr.send();+ await new Promise((r) => setTimeout(r, 0));+ return seen;+ } finally {+ Deno.core.ops.op_fetch_url = originalFetchOp;+ }+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();++ let value = result.value.expect("captured URLs");+ let seen = value.as_array().expect("captured URLs");+ // The XHR entry cannot isolate its own layer: send() passes the already absolute URL on+ // to fetch, so fetch takes over the resolution if it is removed from send, and the assert+ // still holds. It pins the result, not the layer.+ assert_eq!(seen.len(), 2, "one fetch, one XHR");+ assert_eq!(seen[0].as_str().unwrap(), "http://example.com/app/data/x.json");+ assert_eq!(seen[1].as_str().unwrap(), "http://example.com/app/data/y.json");+ }+#[tokio::test(flavor = "current_thread")]async fn test_fetch_url_input_decodes_binary_body_base64() {let mut rt = setup_runtime("<html><body></body></html>");