The headless browser for AI agents and web scraping
fix(js): preserve document.write script order
SGavrl2b07b7605bfc
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rsindex 2fa428ee..78f4f4ec 100644--- a/crates/obscura-browser/src/page.rs+++ b/crates/obscura-browser/src/page.rs@@ -2071,16 +2071,16 @@ impl Page {self.execute_scripts_with_module_budget(None).await;}- /// Drive only dynamic script elements which participate in the current- /// document's load-event delay set. Browser script runners keep this set- /// separate from arbitrary post-load imports, timers, and enhancement- /// scripts; navigation readiness must not turn those into an implicit- /// multi-second settle.- async fn drive_load_delaying_scripts(+ /// Drive a selected script queue until it empties or the shared parser+ /// deadline expires. The caller chooses the queue so parser blockers and+ /// load-delaying dynamic scripts keep their distinct lifecycle semantics.+ async fn drive_pending_scripts(js: &mut ObscuraJsRuntime,deadline: tokio::time::Instant,+ pending: fn(&mut ObscuraJsRuntime) -> bool,+ label: &str,) -> bool {- while js.has_pending_load_delaying_scripts() {+ while pending(js) {let Some(remaining) = deadline.checked_duration_since(tokio::time::Instant::now())else {return false;@@ -2096,13 +2096,13 @@ impl Page {.await{Ok(Ok(_idle)) => {- if js.has_pending_load_delaying_scripts() {+ if pending(js) {tokio::task::yield_now().await;}}Ok(Err(error)) => {if obscura_js::runtime::is_fatal_event_loop_error(&error) {- tracing::warn!("load-delaying dynamic script event loop failed: {error}");+ tracing::warn!("{label} script event loop failed: {error}");return false;}// A load-delaying script threw or left an unhandled@@ -2110,7 +2110,7 @@ impl Page {// killing the pump would strand every still-pending script// (#699). The absolute deadline above bounds a page that// errors on every turn.- tracing::warn!("load-delaying script task error, continuing: {error}");+ tracing::warn!("{label} script task error, continuing: {error}");tokio::task::yield_now().await;}Err(_) => {@@ -2122,6 +2122,32 @@ impl Page {true}+ async fn drive_load_delaying_scripts(+ js: &mut ObscuraJsRuntime,+ deadline: tokio::time::Instant,+ ) -> bool {+ Self::drive_pending_scripts(+ js,+ deadline,+ ObscuraJsRuntime::has_pending_load_delaying_scripts,+ "load-delaying",+ )+ .await+ }++ async fn drive_parser_blocking_scripts(+ js: &mut ObscuraJsRuntime,+ deadline: tokio::time::Instant,+ ) -> bool {+ Self::drive_pending_scripts(+ js,+ deadline,+ ObscuraJsRuntime::has_pending_parser_blocking_scripts,+ "parser-blocking",+ )+ .await+ }+async fn execute_scripts_with_module_budget(&mut self, module_budget_override: Option<u64>) {let scripts_started = std::time::Instant::now();tracing::info!(@@ -2614,6 +2640,15 @@ impl Page {} else {let fetched_script = fetched.remove(&index);execute_classic(self, script, fetched_script);+ if let Some(js) = &mut self.js {+ if js.take_document_write_inserted_script()+ && !Self::drive_parser_blocking_scripts(js, script_deadline).await+ {+ tracing::warn!(+ "script deadline reached with a parser-blocking document.write script pending"+ );+ }+ }}}ScriptKind::Module => {@@ -2786,6 +2821,15 @@ impl Page {let script = &all_scripts[index];let fetched_script = fetched.remove(&index);execute_classic(self, script, fetched_script);+ if let Some(js) = &mut self.js {+ if js.take_document_write_inserted_script()+ && !Self::drive_parser_blocking_scripts(js, script_deadline).await+ {+ tracing::warn!(+ "script deadline reached with a parser-blocking document.write script pending"+ );+ }+ }}ScheduledScript::Module {prepared,@@ -5601,6 +5645,49 @@ mod tests {(format!("http://{address}"), request_rx)}+ fn spawn_written_script_order_server() -> String {+ use std::io::{Read as _, Write as _};+ use std::sync::atomic::{AtomicBool, Ordering};++ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let vendor_finished = std::sync::Arc::new(AtomicBool::new(false));+ std::thread::spawn(move || {+ for stream in listener.incoming().take(2) {+ let Ok(mut stream) = stream else {+ continue;+ };+ let vendor_finished = vendor_finished.clone();+ std::thread::spawn(move || {+ let mut request = [0u8; 2048];+ let length = stream.read(&mut request).unwrap_or(0);+ let request = String::from_utf8_lossy(&request[..length]);+ let path = request+ .lines()+ .next()+ .and_then(|line| line.split_ascii_whitespace().nth(1))+ .unwrap_or("/");+ let body = if path == "/vendor.js" {+ std::thread::sleep(std::time::Duration::from_millis(125));+ vendor_finished.store(true, Ordering::SeqCst);+ "globalThis.__vendorValue = 1;".to_string()+ } else {+ format!(+ "globalThis.__appRequestFollowedVendor = {}; globalThis.__appSawVendor = globalThis.__vendorValue;",+ vendor_finished.load(Ordering::SeqCst),+ )+ };+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: application/javascript\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len(),+ );+ stream.write_all(response.as_bytes()).unwrap();+ });+ }+ });+ format!("http://{address}")+ }+fn spawn_script_resource_cache_server(distinct: bool,) -> (String, std::sync::Arc<std::sync::atomic::AtomicUsize>) {@@ -6866,6 +6953,77 @@ mod tests {);}+ #[tokio::test(flavor = "current_thread")]+ async fn document_write_external_script_blocks_later_parser_scripts() {+ let (base, requests) = spawn_delayed_classic_script_server(+ std::time::Duration::from_millis(75),+ "globalThis.__writtenOrder.push('external'); globalThis.__writtenValue = 1;",+ );+ let html = format!(+ r#"<html><body>+ <script>globalThis.__writtenOrder = [];</script>+ <script>+ document.write('<script src="{base}/written.js"><\/script>');+ document.write('<script>globalThis.__writtenOrder.push("written:" + String(globalThis.__writtenValue));<\/script>');+ </script>+ <script>globalThis.__writtenOrder.push('after:' + String(globalThis.__writtenValue));</script>+ </body></html>"#,+ );+ let mut page = import_map_test_page(+ "document-write-parser-blocking",+ "http://127.0.0.1:9",+ &html,+ );++ page.execute_scripts().await;++ assert_eq!(+ requests+ .recv_timeout(std::time::Duration::from_secs(1))+ .unwrap(),+ "/written.js",+ );+ assert_eq!(+ page.js+ .as_mut()+ .unwrap()+ .evaluate("globalThis.__writtenOrder")+ .unwrap(),+ serde_json::json!(["external", "written:1", "after:1"]),+ );+ }++ #[tokio::test(flavor = "current_thread")]+ async fn document_write_does_not_start_later_async_script_before_blocker() {+ let base = spawn_written_script_order_server();+ let html = format!(+ r#"<html><body><script>+ document.write('<script src="{base}/vendor.js"><\/script>');+ document.write('<script async src="{base}/app.js"><\/script>');+ </script><script>+ globalThis.__afterWriteSawVendor = globalThis.__vendorValue;+ </script></body></html>"#,+ );+ let mut page = import_map_test_page(+ "document-write-async-after-blocker",+ "http://127.0.0.1:9",+ &html,+ );++ page.execute_scripts().await;++ assert_eq!(+ page.js+ .as_mut()+ .unwrap()+ .evaluate(+ "[globalThis.__afterWriteSawVendor, globalThis.__appSawVendor, globalThis.__appRequestFollowedVendor]"+ )+ .unwrap(),+ serde_json::json!([1, 1, true]),+ );+ }+#[tokio::test(flavor = "current_thread")]async fn load_delaying_script_progresses_through_continuously_ready_timer_work() {let (base, requests) = spawn_delayed_classic_script_server(diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.jsindex b82685cd..82576e07 100644--- a/crates/obscura-js/js/bootstrap.js+++ b/crates/obscura-js/js/bootstrap.js@@ -28,7 +28,7 @@ const __obscuraCore = globalThis.Deno.core;'__obscura_registerLinkedStylesheet', '__obscura_activateLabel','__obscura_isDisabled', '__obscura_labeledControl', '__obscura_interactiveHost','__markParserScripts', '__obscura_hasPendingDynamicScripts',- '__obscura_hasPendingLoadDelayingScripts',+ '__obscura_hasPendingLoadDelayingScripts', '__obscura_hasPendingParserBlockingScripts','__obscura_nextPendingTimeoutDelay','__obscura_hw', '__obscura_mem','__documentReadyState__', '__currentUrl',@@ -267,6 +267,10 @@ let __dynScriptQueue = [];let __dynScriptBusy = false;let __dynClassicPending = 0;let __dynLoadDelayingPending = 0;+let __parserBlockingScriptQueue = [];+let __parserBlockingScriptBusy = false;+let __parserBlockingScriptPending = 0;+const __documentWriteScripts = new WeakSet();Object.defineProperty(globalThis, '__obscura_hasPendingDynamicScripts', {value: function() {return __dynClassicPending > 0 || __dynScriptBusy || __dynScriptQueue.length > 0;@@ -286,6 +290,12 @@ Object.defineProperty(globalThis, '__obscura_hasPendingLoadDelayingScripts', {enumerable: false,configurable: false,});+Object.defineProperty(globalThis, '__obscura_hasPendingParserBlockingScripts', {+ value: function() { return __parserBlockingScriptPending > 0; },+ writable: false,+ enumerable: false,+ configurable: false,+});function _decodeDataScriptUrl(url) {const comma = url.indexOf(',');if (!url.startsWith('data:') || comma < 5) {@@ -371,10 +381,13 @@ async function __runDynScriptTask(task) {if (task.isModule) {await import(task.url);} else {- if (!task.fetchResult) __startDynClassicFetch(task);- const fetched = await task.fetchResult;- if (fetched.error) throw fetched.error;- const body = fetched.body;+ let body = task.inlineCode;+ if (body === undefined) {+ if (!task.fetchResult) __startDynClassicFetch(task);+ const fetched = await task.fetchResult;+ if (fetched.error) throw fetched.error;+ body = fetched.body;+ }if (body) {// A fetched async script is executed by a ScriptRunner task, not by// the fetch promise's microtask continuation. Besides matching event@@ -429,6 +442,23 @@ async function __processDynScriptQueue() {__dynScriptBusy = false;}}+async function __processParserBlockingScriptQueue() {+ if (__parserBlockingScriptBusy) return;+ __parserBlockingScriptBusy = true;+ try {+ while (__parserBlockingScriptQueue.length > 0) {+ const entry = __parserBlockingScriptQueue.shift();+ if (!entry.blocking) {+ __runAsyncClassicScript(entry.task);+ continue;+ }+ try { await __runDynScriptTask(entry.task); }+ finally { __parserBlockingScriptPending--; }+ }+ } finally {+ __parserBlockingScriptBusy = false;+ }+}// Resolve a resource URL (script src / link href) against <base href> or the// document URL, the way the inline dynamic-script path does. Guarded so a bad// base or href never throws into appendChild.@@ -1948,6 +1978,10 @@ function __prepareInsertedScript(script) {const code = src ? "" : script.textContent;if (!src && !code) return;const prevNid = globalThis.__currentScriptNid;+ const parserInserted = __documentWriteScripts.has(script);+ const parserBlocking = parserInserted+ && !isModule+ && (!src || (!script.hasAttribute('async') && !script.hasAttribute('defer')));if (src) {let baseHref;try {@@ -1982,6 +2016,13 @@ function __prepareInsertedScript(script) {// not turn already-prepared work into a post-load enhancement.task.delaysLoad = globalThis.document?.readyState !== 'complete';if (task.delaysLoad) __dynLoadDelayingPending++;+ if (parserInserted && !isModule) {+ task.prevNid = 0;+ if (parserBlocking) __parserBlockingScriptPending++;+ __parserBlockingScriptQueue.push({ task, blocking: parserBlocking });+ __processParserBlockingScriptQueue();+ return;+ }// A non-parser-inserted classic script is force-async unless script code// explicitly assigned `.async = false`. Keep that opt-out in insertion// order; default/async=true scripts fetch concurrently and execute as soon@@ -2018,6 +2059,20 @@ function __prepareInsertedScript(script) {if (task.delaysLoad) __dynLoadDelayingPending++;__dynScriptQueue.push(task);__processDynScriptQueue();+ } else if (parserBlocking) {+ __parserBlockingScriptPending++;+ __parserBlockingScriptQueue.push({+ blocking: true,+ task: {+ inlineCode: code,+ isModule: false,+ nid: script._nid,+ prevNid: 0,+ dispatchEvent: () => {},+ delaysLoad: false,+ },+ });+ __processParserBlockingScriptQueue();} else {globalThis.__currentScriptNid = script._nid;try { (0, eval)(code); }@@ -5898,6 +5953,9 @@ class Document extends Node {var parentNid = +placements[i][0];var node = _wrap(+placements[i][1]);if (!node) continue;+ if (node.nodeType === 1 && node.tagName === 'SCRIPT') {+ __documentWriteScripts.add(node);+ }if (parentNid) {var parent = _wrap(parentNid);if (parent) parent.appendChild(node);diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex 14d5226e..2c175680 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -1,4 +1,4 @@-use std::cell::RefCell;+use std::cell::{Cell, RefCell};use std::collections::{HashMap, HashSet, VecDeque};use std::rc::Rc;use std::sync::Arc;@@ -302,6 +302,9 @@ pub struct ObscuraState {/// The document's input stream for `document.write()`, created on the first call./// Why the calls share one parser is in `write_stream`.pub(crate) write_stream: RefCell<Option<crate::write_stream::DocumentWriteStream>>,+ /// Cheap native signal that the current parser script wrote another+ /// script. This avoids polling V8 after every ordinary parser script.+ pub(crate) document_write_inserted_script: Cell<bool>,}/// A frame document waiting to be given a realm.@@ -436,6 +439,7 @@ impl ObscuraState {import_map: Rc::new(RefCell::new(ImportMap::default())),already_started_scripts: RefCell::new(HashSet::new()),write_stream: RefCell::new(None),+ document_write_inserted_script: Cell::new(false),}}}@@ -2163,8 +2167,14 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->"document_write" => {let mut slot = gs.write_stream.borrow_mut();let stream = slot.get_or_insert_with(DocumentWriteStream::new);- let pairs: Vec<[i32; 2]> = stream- .write(&arg2, dom)+ let placements = stream.write(&arg2, dom);+ if placements+ .iter()+ .any(|placement| node_is_script(dom, placement.node))+ {+ gs.document_write_inserted_script.set(true);+ }+ let pairs: Vec<[i32; 2]> = placements.iter().map(|placement| {[@@ -2178,6 +2188,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->// document.open() discards what the input stream holds and starts over."document_write_reset" => {*gs.write_stream.borrow_mut() = None;+ gs.document_write_inserted_script.set(false);"true".into()}"set_text_content" => {diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rsindex 6e5fd0c8..53440fea 100644--- a/crates/obscura-js/src/runtime.rs+++ b/crates/obscura-js/src/runtime.rs@@ -3343,6 +3343,26 @@ impl ObscuraJsRuntime {.unwrap_or(false)}+ /// Whether a parser-blocking script inserted by `document.write()` still+ /// has fetch or evaluation work outstanding.+ pub fn has_pending_parser_blocking_scripts(&mut self) -> bool {+ self.evaluate(+ "globalThis.__obscura_hasPendingParserBlockingScripts?.() === true",+ )+ .ok()+ .and_then(|value| value.as_bool())+ .unwrap_or(false)+ }++ /// Consume the native signal that the current parser script called+ /// `document.write()` with a script node.+ pub fn take_document_write_inserted_script(&self) -> bool {+ self.state+ .borrow()+ .document_write_inserted_script+ .replace(false)+ }+/// Generation of observable connected-document mutations. This excludes/// detached-tree construction and no-op writes, which cannot affect a/// screenshot or DOM dump.@@ -13952,6 +13972,7 @@ mod tests {let mut rt = setup_runtime("<html><body></body></html>");assert!(!rt.has_pending_dynamic_scripts());assert!(!rt.has_pending_load_delaying_scripts());+ assert!(!rt.has_pending_parser_blocking_scripts());assert_eq!(rt.next_pending_timeout_delay_ms(), None);assert_eq!(rt.evaluate("typeof __dynScriptBusy").unwrap(),@@ -13978,6 +13999,13 @@ mod tests {.unwrap(),serde_json::json!(false));+ assert_eq!(+ rt.evaluate(+ "Reflect.ownKeys(globalThis).includes('__obscura_hasPendingParserBlockingScripts')"+ )+ .unwrap(),+ serde_json::json!(false)+ );assert_eq!(rt.evaluate("Object.getOwnPropertyNames(globalThis).includes('__obscura_nextPendingTimeoutDelay')"