Skip to content

The headless browser for AI agents and web scraping

fix(js): resolve relative URLs against the document base URL

aech98d37eb93feb
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index d7e69040..8cceccde 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -1920,20 +1920,7 @@ class Node {
get ownerDocument() { return globalThis.document; }
// https://dom.spec.whatwg.org/#dom-node-baseuri
get baseURI() {
- try {
- const doc = globalThis.document;
- const docUrl = (doc && doc.URL) || "";
- const baseEl = (doc && doc.querySelector) ? doc.querySelector("base[href]") : null;
- if (baseEl) {
- const href = baseEl.getAttribute("href");
- if (href) {
- return docUrl ? new URL(href, docUrl).href : href;
- }
- }
- return docUrl;
- } catch (e) {
- return "";
- }
+ try { return _documentBase(); } catch (e) { return ""; }
}
get textContent() { return _domParse("text_content", this._nid) ?? ""; }
set textContent(v) {
@@ -2492,10 +2479,25 @@ function _applyDocQueryEncoding(u) {
return u;
}
+// The base for relative URLs. <base href> overrides the document URL, so an app in a sub-path
+// requests "chunk-A.js" under its current route and gets 404.
+// https://html.spec.whatwg.org/multipage/urls-and-fetching.html#document-base-url
+// Returns "" when there is no document, so each call site keeps its own fallback.
+function _documentBase() {
+ // history.pushState moves the document URL without reaching the Rust side. Then the base must
+ // be built here, or every relative URL resolves against the pre-routing address.
+ const virtual = globalThis.__virtualUrl;
+ if (virtual) {
+ const raw = _domParse("document_base_href");
+ if (!raw) return virtual;
+ try { return new URL(raw, virtual).href; } catch (e) { return virtual; }
+ }
+ return _domParse("document_base_url") || _domParse("document_url") || "";
+}
// HTMLHyperlinkElementUtils helpers (the <a>/<area> URL-decomposition members).
// The element's href attribute is parsed against the document base URL via the
// WHATWG url op; component getters read it, setters rewrite the href attribute.
-function _anchorBase() { return _domParse("document_url") || "about:blank"; }
+function _anchorBase() { return _documentBase() || "about:blank"; }
function _elemHrefURL(el) {
const raw = el.getAttribute('href');
if (raw === null || raw === undefined) return null;
@@ -3834,6 +3836,15 @@ class Element extends Node {
const r = _urlResolveOp(raw, _anchorBase());
return r !== null ? r : raw;
}
+ if (ln === 'base') {
+ // https://html.spec.whatwg.org/multipage/semantics.html#dom-base-href
+ // Against the fallback base URL, not the document base URL: a base element is not affected
+ // by other base elements or itself. Applications read this to determine their own base.
+ const raw = this.getAttribute('href');
+ if (raw === null) return '';
+ const r = _urlResolveOp(raw, _domParse("document_url") || "about:blank");
+ return r !== null ? r : raw;
+ }
return this.getAttribute("href") || "";
}
set href(v) { this.setAttribute("href", v); }
@@ -3881,7 +3892,7 @@ class Element extends Node {
// value (issue #255). getAttribute("src") still returns the literal.
const v = this.getAttribute("src");
if (!v) return "";
- try { return new URL(v, globalThis.location?.href || "about:blank").href; }
+ try { return new URL(v, _documentBase() || "about:blank").href; }
catch (e) { return v; }
}
set src(v) {
@@ -3980,8 +3991,9 @@ class Element extends Node {
return this._iframeWin;
}
get action() {
+ // A missing action falls back to the document URL, a present one resolves against the base.
const action = this.getAttribute("action") || _domParse("document_url") || "";
- try { return new URL(action, _domParse("document_url") || "about:blank").href; } catch(e) { return action; }
+ try { return new URL(action, _documentBase() || "about:blank").href; } catch(e) { return action; }
}
set action(v) { this.setAttribute("action", v); }
get method() { return this.getAttribute("method") || "get"; }
@@ -6108,7 +6120,7 @@ function _resolveUrl(url) {
url = String(url);
if (!url) return url;
if (url.startsWith('http://') || url.startsWith('https://') || url.startsWith('about:')) return url;
- try { return new URL(url, _domParse("document_url") || "about:blank").href; } catch(e) { return url; }
+ try { return new URL(url, _documentBase() || "about:blank").href; } catch(e) { return url; }
}
// `__virtualUrl` is set by `history.pushState`/`replaceState` (and cleared by
// any real navigation). When set, `location.href` and friends read it instead
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index 7597a997..813d4005 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -172,6 +172,10 @@ pub struct ObscuraState {
/// completions use this to discard bytes and lifecycle results belonging
/// to a navigation that has already been replaced.
pub document_generation: u64,
+ /// Cached document base URL. Computing it walks the tree and runs the selector engine, and
+ /// the JS layer asks for it on every relative URL, including the URL parts of `<a>`.
+ /// Interior mutability so the read path keeps its shared borrow.
+ pub base_url_cache: RefCell<Option<BaseUrlCache>>,
/// Final image/font-aware layout shared by CSSOM geometry and screenshots.
/// DOM/style/viewport changes clear this value but retain resource bytes.
#[cfg(feature = "render")]
@@ -311,6 +315,7 @@ impl ObscuraState {
page_in_flight: Arc::new(std::sync::atomic::AtomicU32::new(0)),
activity_generation: 0,
document_generation: 0,
+ base_url_cache: RefCell::new(None),
#[cfg(feature = "render")]
prepared_render: None,
#[cfg(feature = "render")]
@@ -1297,6 +1302,18 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
serde_json::to_string(&title).unwrap_or("\"\"".into())
}
"document_url" => serde_json::to_string(&gs.url).unwrap_or("\"\"".into()),
+ // The base for relative URLs. It differs from document_url exactly when the page carries
+ // a <base href>, and that is the point: HTML resolves against the base, not the document.
+ "document_base_url" => serde_json::to_string(
+ &document_base_url_memoized(&gs).unwrap_or_else(|| gs.url.clone()),
+ )
+ .unwrap_or("\"\"".into()),
+ // The unresolved attribute. After history.pushState only JS knows the URL, so only JS
+ // can resolve a relative base against it.
+ "document_base_href" => {
+ serde_json::to_string(&document_base_href_memoized(&gs).unwrap_or_default())
+ .unwrap_or("\"\"".into())
+ }
"document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),
"document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),
"document_element" => {
@@ -4601,7 +4618,8 @@ fn op_waapi_control(
changed
}
-#[cfg(feature = "render")]
+// Not tied to `render`: the JS layer resolves every relative URL through here, in all build
+// variants.
pub(crate) fn document_base_url(state: &ObscuraState) -> Option<String> {
let document_url = url::Url::parse(&state.url).ok()?;
let base_href = state.dom.as_ref().and_then(|dom| {
@@ -4614,11 +4632,74 @@ pub(crate) fn document_base_url(state: &ObscuraState) -> Option<String> {
})
});
match base_href {
- Some(href) => document_url.join(&href).ok().map(|url| url.to_string()),
+ // https://html.spec.whatwg.org/multipage/semantics.html#set-the-frozen-base-url
+ // A data: or javascript: base falls back to the document URL. Accepting it would instead
+ // make every later relative resolution fail.
+ Some(href) => match document_url.join(&href) {
+ Ok(base) if base.scheme() != "data" && base.scheme() != "javascript" => {
+ Some(base.to_string())
+ }
+ _ => Some(document_url.to_string()),
+ },
None => Some(document_url.to_string()),
}
}
+/// The raw `href` attribute of the first `<base href>`, unresolved. The JS layer needs it after
+/// `history.pushState`: the document URL has moved, only JS knows the new one, so only JS can
+/// resolve a relative base against it.
+fn document_base_href(state: &ObscuraState) -> Option<String> {
+ state.dom.as_ref().and_then(|dom| {
+ dom.query_selector("base[href]")
+ .ok()
+ .flatten()
+ .and_then(|id| {
+ dom.get_node(id)
+ .and_then(|node| node.get_attribute("href").map(str::to_string))
+ })
+ })
+}
+
+/// What the cached values were computed from. If any of the three changes, they are recomputed.
+pub struct BaseUrlCache {
+ activity_generation: u64,
+ document_generation: u64,
+ url: String,
+ resolved: Option<String>,
+ raw_href: Option<String>,
+}
+
+/// Both base values behind a cache. Uncached, each one walks the tree and runs the selector
+/// engine, which would make `a.href` an O(nodes) read.
+fn base_values_memoized(state: &ObscuraState) -> (Option<String>, Option<String>) {
+ if let Some(cached) = state.base_url_cache.borrow().as_ref() {
+ if cached.activity_generation == state.activity_generation
+ && cached.document_generation == state.document_generation
+ && cached.url == state.url
+ {
+ return (cached.resolved.clone(), cached.raw_href.clone());
+ }
+ }
+ let resolved = document_base_url(state);
+ let raw_href = document_base_href(state);
+ *state.base_url_cache.borrow_mut() = Some(BaseUrlCache {
+ activity_generation: state.activity_generation,
+ document_generation: state.document_generation,
+ url: state.url.clone(),
+ resolved: resolved.clone(),
+ raw_href: raw_href.clone(),
+ });
+ (resolved, raw_href)
+}
+
+pub(crate) fn document_base_url_memoized(state: &ObscuraState) -> Option<String> {
+ base_values_memoized(state).0
+}
+
+pub(crate) fn document_base_href_memoized(state: &ObscuraState) -> Option<String> {
+ base_values_memoized(state).1
+}
+
#[cfg(feature = "render")]
pub(crate) fn ensure_prepared_render(
state: &mut ObscuraState,
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index f36f742e..80a894e9 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -13962,6 +13962,320 @@ mod tests {
);
}
+ /// Document URL two levels deep, base one level deep and the origin root on neither of the
+ /// two. The three possible bases land on three different paths, and one assert keeps them
+ /// apart:
+ /// document base URL -> /app/data/x.json (the spec)
+ /// document URL -> /deep/data/x.json (the bug)
+ /// origin root -> /data/x.json (a base "/" would hide this one)
+ fn setup_runtime_at_deep_url(html: &str) -> ObscuraJsRuntime {
+ let dom = parse_html(html);
+ let mut rt = ObscuraJsRuntime::new();
+ rt.set_dom(dom);
+ rt.set_url("http://example.com/deep/page");
+ rt.run_page_init();
+ rt
+ }
+
+ const BASE_HREF_PAGE: &str = r#"<html><head><base href="/app/"></head><body>
+ <a id="link" href="data/x.json"></a>
+ <form id="form" action="submit"></form>
+ <script id="script" src="chunk.js"></script>
+ </body></html>"#;
+
+ #[test]
+ fn base_href_governs_dom_url_reflection() {
+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);
+
+ // One evaluate, one assert: a bundle of assert_eq! aborts at the first failure and
+ // reports one broken path while hiding the others.
+ let seen = rt
+ .evaluate(
+ r#"return [
+ document.baseURI,
+ document.getElementById('link').href,
+ document.getElementById('form').action,
+ document.getElementById('script').src,
+ ]"#,
+ )
+ .unwrap();
+ assert_eq!(
+ seen.as_array().unwrap(),
+ &vec![
+ serde_json::json!("http://example.com/app/"),
+ serde_json::json!("http://example.com/app/data/x.json"),
+ serde_json::json!("http://example.com/app/submit"),
+ serde_json::json!("http://example.com/app/chunk.js"),
+ ]
+ );
+ }
+
+ #[test]
+ fn a_relative_location_assignment_follows_the_base() {
+ // _resolveUrl serves location.href=, assign, replace and window.location=. Of the six
+ // call sites it has the largest external effect, so it gets its own test.
+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);
+ rt.evaluate("location.href = 'users/42'").unwrap();
+
+ let landed = rt.evaluate("location.href").unwrap();
+ assert_eq!(landed.as_str().unwrap(), "http://example.com/app/users/42");
+ }
+
+ #[test]
+ fn without_base_href_resolution_stays_on_the_document_url() {
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head></head><body>
+ <a id="link" href="data/x.json"></a>
+ <form id="form" action="submit"></form>
+ <script id="script" src="chunk.js"></script>
+ </body></html>"#,
+ );
+
+ // Every call site, not just the anchor: a site that resolves to the origin root instead
+ // of the document URL slips through on a page without a base only when nobody is looking.
+ let seen = rt
+ .evaluate(
+ r#"return [
+ document.baseURI,
+ document.getElementById('link').href,
+ document.getElementById('form').action,
+ document.getElementById('script').src,
+ ]"#,
+ )
+ .unwrap();
+ assert_eq!(
+ seen.as_array().unwrap(),
+ &vec![
+ serde_json::json!("http://example.com/deep/page"),
+ serde_json::json!("http://example.com/deep/data/x.json"),
+ serde_json::json!("http://example.com/deep/submit"),
+ serde_json::json!("http://example.com/deep/chunk.js"),
+ ]
+ );
+ }
+
+ #[test]
+ fn base_element_href_reflects_the_resolved_url() {
+ // Resolved against the fallback base URL, i.e. the document URL and not /app/.
+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);
+ let href = rt.evaluate("document.querySelector('base').href").unwrap();
+ assert_eq!(href.as_str().unwrap(), "http://example.com/app/");
+
+ let mut relative = setup_runtime_at_deep_url(
+ r#"<html><head><base href="assets/"></head><body></body></html>"#,
+ );
+ let href = relative
+ .evaluate("document.querySelector('base').href")
+ .unwrap();
+ assert_eq!(href.as_str().unwrap(), "http://example.com/deep/assets/");
+ }
+
+ #[test]
+ fn the_first_base_with_an_href_wins() {
+ // Tree order, and a <base> without href does not count.
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head><base><base href="/a/"><base href="/b/"></head><body>
+ <a id="link" href="x.json"></a>
+ </body></html>"#,
+ );
+ let link = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(link.as_str().unwrap(), "http://example.com/a/x.json");
+ }
+
+ #[test]
+ fn an_empty_base_href_resolves_to_the_document_url() {
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head><base href=""></head><body>
+ <a id="link" href="x.json"></a>
+ </body></html>"#,
+ );
+ let link = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(link.as_str().unwrap(), "http://example.com/deep/x.json");
+ }
+
+ #[test]
+ fn a_cross_origin_base_moves_the_target_but_not_the_page_origin() {
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head><base href="https://cdn.example.net/v2/"></head><body>
+ <a id="link" href="x.json"></a>
+ </body></html>"#,
+ );
+ let seen = rt
+ .evaluate("return [document.getElementById('link').href, location.origin]")
+ .unwrap();
+ assert_eq!(
+ seen.as_array().unwrap(),
+ &vec![
+ serde_json::json!("https://cdn.example.net/v2/x.json"),
+ serde_json::json!("http://example.com"),
+ ]
+ );
+ }
+
+ #[test]
+ fn base_href_rejects_a_data_url_base() {
+ // https://html.spec.whatwg.org/multipage/semantics.html#set-the-frozen-base-url
+ // Accepting it would make every later relative resolution fail instead of falling back.
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head><base href="data:text/html,x"></head><body>
+ <a id="link" href="data/x.json"></a>
+ </body></html>"#,
+ );
+
+ let base_uri = rt.evaluate("document.baseURI").unwrap();
+ assert_eq!(base_uri.as_str().unwrap(), "http://example.com/deep/page");
+
+ let link = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(link.as_str().unwrap(), "http://example.com/deep/data/x.json");
+ }
+
+ #[test]
+ fn base_resolution_follows_the_url_set_by_push_state() {
+ // pushState changes the document URL, and without <base> that very URL is the base.
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head></head><body><a id="link" href="x.json"></a></body></html>"#,
+ );
+ rt.evaluate("history.pushState({}, '', '/other/route')").unwrap();
+
+ let link = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(link.as_str().unwrap(), "http://example.com/other/x.json");
+ }
+
+ #[test]
+ fn a_relative_base_href_resolves_against_the_push_state_url() {
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head><base href="assets/"></head><body>
+ <a id="link" href="x.json"></a>
+ </body></html>"#,
+ );
+ rt.evaluate("history.pushState({}, '', '/other/route')").unwrap();
+
+ let link = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(link.as_str().unwrap(), "http://example.com/other/assets/x.json");
+ }
+
+ /// Guards the cache in `document_base_url_memoized`. Without it, each of these reads walked
+ /// the tree and ran the selector engine, and `a.href` went from a field read to O(nodes).
+ /// The bound is deliberately loose: it should catch the regression, not watch the allocator.
+ #[test]
+ fn anchor_href_reads_do_not_scale_with_document_size() {
+ let mut body = String::from(r#"<html><head></head><body><a id="link" href="x.json"></a>"#);
+ for i in 0..4000 {
+ body.push_str(&format!("<div id=\"n{i}\"><span>text</span></div>"));
+ }
+ body.push_str("</body></html>");
+ let mut rt = setup_runtime_at_deep_url(&body);
+
+ let elapsed = rt
+ .evaluate(
+ r#"
+ const link = document.getElementById('link');
+ const started = Date.now();
+ for (let i = 0; i < 2000; i++) { link.href; }
+ return Date.now() - started;
+ "#,
+ )
+ .unwrap();
+ let ms = elapsed.as_f64().expect("elapsed ms");
+ assert!(
+ ms < 500.0,
+ "2000 a.href reads on a document with 12000 nodes took {ms} ms, the base query is not cached"
+ );
+ }
+
+ #[test]
+ fn the_base_memo_still_sees_a_base_element_added_later() {
+ let mut rt = setup_runtime_at_deep_url(
+ r#"<html><head></head><body><a id="link" href="x.json"></a></body></html>"#,
+ );
+ let before = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(before.as_str().unwrap(), "http://example.com/deep/x.json");
+
+ rt.evaluate(
+ r#"
+ const base = document.createElement('base');
+ base.setAttribute('href', '/');
+ document.head.appendChild(base);
+ "#,
+ )
+ .unwrap();
+
+ let after = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(after.as_str().unwrap(), "http://example.com/x.json");
+ }
+
+ #[test]
+ fn the_base_memo_notices_a_changed_href_attribute() {
+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);
+ let before = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(before.as_str().unwrap(), "http://example.com/app/data/x.json");
+
+ rt.evaluate("document.querySelector('base').setAttribute('href', '/other/')")
+ .unwrap();
+
+ let after = rt
+ .evaluate("document.getElementById('link').href")
+ .unwrap();
+ assert_eq!(after.as_str().unwrap(), "http://example.com/other/data/x.json");
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn base_href_governs_fetch_and_xhr_targets() {
+ let mut rt = setup_runtime_at_deep_url(BASE_HREF_PAGE);
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const originalFetchOp = Deno.core.ops.op_fetch_url;
+ const seen = [];
+ try {
+ Deno.core.ops.op_fetch_url = (url) => {
+ seen.push(url);
+ return JSON.stringify({ status: 200, headers: {}, body: "{}", url });
+ };
+ await fetch("data/x.json");
+ const xhr = new XMLHttpRequest();
+ xhr.open("GET", "data/y.json");
+ xhr.send();
+ await new Promise((r) => setTimeout(r, 0));
+ return seen;
+ } finally {
+ Deno.core.ops.op_fetch_url = originalFetchOp;
+ }
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+
+ let value = result.value.expect("captured URLs");
+ let seen = value.as_array().expect("captured URLs");
+ // The XHR entry cannot isolate its own layer: send() passes the already absolute URL on
+ // to fetch, so fetch takes over the resolution if it is removed from send, and the assert
+ // still holds. It pins the result, not the layer.
+ assert_eq!(seen.len(), 2, "one fetch, one XHR");
+ assert_eq!(seen[0].as_str().unwrap(), "http://example.com/app/data/x.json");
+ assert_eq!(seen[1].as_str().unwrap(), "http://example.com/app/data/y.json");
+ }
+
#[tokio::test(flavor = "current_thread")]
async fn test_fetch_url_input_decodes_binary_body_base64() {
let mut rt = setup_runtime("<html><body></body></html>");