Skip to content

The headless browser for AI agents and web scraping

fix(js): preserve document.write script order

SGavrl2b07b7605bfc
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs
index 2fa428ee..78f4f4ec 100644
--- a/crates/obscura-browser/src/page.rs
+++ b/crates/obscura-browser/src/page.rs
@@ -2071,16 +2071,16 @@ impl Page {
self.execute_scripts_with_module_budget(None).await;
}
- /// Drive only dynamic script elements which participate in the current
- /// document's load-event delay set. Browser script runners keep this set
- /// separate from arbitrary post-load imports, timers, and enhancement
- /// scripts; navigation readiness must not turn those into an implicit
- /// multi-second settle.
- async fn drive_load_delaying_scripts(
+ /// Drive a selected script queue until it empties or the shared parser
+ /// deadline expires. The caller chooses the queue so parser blockers and
+ /// load-delaying dynamic scripts keep their distinct lifecycle semantics.
+ async fn drive_pending_scripts(
js: &mut ObscuraJsRuntime,
deadline: tokio::time::Instant,
+ pending: fn(&mut ObscuraJsRuntime) -> bool,
+ label: &str,
) -> bool {
- while js.has_pending_load_delaying_scripts() {
+ while pending(js) {
let Some(remaining) = deadline.checked_duration_since(tokio::time::Instant::now())
else {
return false;
@@ -2096,13 +2096,13 @@ impl Page {
.await
{
Ok(Ok(_idle)) => {
- if js.has_pending_load_delaying_scripts() {
+ if pending(js) {
tokio::task::yield_now().await;
}
}
Ok(Err(error)) => {
if obscura_js::runtime::is_fatal_event_loop_error(&error) {
- tracing::warn!("load-delaying dynamic script event loop failed: {error}");
+ tracing::warn!("{label} script event loop failed: {error}");
return false;
}
// A load-delaying script threw or left an unhandled
@@ -2110,7 +2110,7 @@ impl Page {
// killing the pump would strand every still-pending script
// (#699). The absolute deadline above bounds a page that
// errors on every turn.
- tracing::warn!("load-delaying script task error, continuing: {error}");
+ tracing::warn!("{label} script task error, continuing: {error}");
tokio::task::yield_now().await;
}
Err(_) => {
@@ -2122,6 +2122,32 @@ impl Page {
true
}
+ async fn drive_load_delaying_scripts(
+ js: &mut ObscuraJsRuntime,
+ deadline: tokio::time::Instant,
+ ) -> bool {
+ Self::drive_pending_scripts(
+ js,
+ deadline,
+ ObscuraJsRuntime::has_pending_load_delaying_scripts,
+ "load-delaying",
+ )
+ .await
+ }
+
+ async fn drive_parser_blocking_scripts(
+ js: &mut ObscuraJsRuntime,
+ deadline: tokio::time::Instant,
+ ) -> bool {
+ Self::drive_pending_scripts(
+ js,
+ deadline,
+ ObscuraJsRuntime::has_pending_parser_blocking_scripts,
+ "parser-blocking",
+ )
+ .await
+ }
+
async fn execute_scripts_with_module_budget(&mut self, module_budget_override: Option<u64>) {
let scripts_started = std::time::Instant::now();
tracing::info!(
@@ -2614,6 +2640,15 @@ impl Page {
} else {
let fetched_script = fetched.remove(&index);
execute_classic(self, script, fetched_script);
+ if let Some(js) = &mut self.js {
+ if js.take_document_write_inserted_script()
+ && !Self::drive_parser_blocking_scripts(js, script_deadline).await
+ {
+ tracing::warn!(
+ "script deadline reached with a parser-blocking document.write script pending"
+ );
+ }
+ }
}
}
ScriptKind::Module => {
@@ -2786,6 +2821,15 @@ impl Page {
let script = &all_scripts[index];
let fetched_script = fetched.remove(&index);
execute_classic(self, script, fetched_script);
+ if let Some(js) = &mut self.js {
+ if js.take_document_write_inserted_script()
+ && !Self::drive_parser_blocking_scripts(js, script_deadline).await
+ {
+ tracing::warn!(
+ "script deadline reached with a parser-blocking document.write script pending"
+ );
+ }
+ }
}
ScheduledScript::Module {
prepared,
@@ -5601,6 +5645,49 @@ mod tests {
(format!("http://{address}"), request_rx)
}
+ fn spawn_written_script_order_server() -> String {
+ use std::io::{Read as _, Write as _};
+ use std::sync::atomic::{AtomicBool, Ordering};
+
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let vendor_finished = std::sync::Arc::new(AtomicBool::new(false));
+ std::thread::spawn(move || {
+ for stream in listener.incoming().take(2) {
+ let Ok(mut stream) = stream else {
+ continue;
+ };
+ let vendor_finished = vendor_finished.clone();
+ std::thread::spawn(move || {
+ let mut request = [0u8; 2048];
+ let length = stream.read(&mut request).unwrap_or(0);
+ let request = String::from_utf8_lossy(&request[..length]);
+ let path = request
+ .lines()
+ .next()
+ .and_then(|line| line.split_ascii_whitespace().nth(1))
+ .unwrap_or("/");
+ let body = if path == "/vendor.js" {
+ std::thread::sleep(std::time::Duration::from_millis(125));
+ vendor_finished.store(true, Ordering::SeqCst);
+ "globalThis.__vendorValue = 1;".to_string()
+ } else {
+ format!(
+ "globalThis.__appRequestFollowedVendor = {}; globalThis.__appSawVendor = globalThis.__vendorValue;",
+ vendor_finished.load(Ordering::SeqCst),
+ )
+ };
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: application/javascript\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len(),
+ );
+ stream.write_all(response.as_bytes()).unwrap();
+ });
+ }
+ });
+ format!("http://{address}")
+ }
+
fn spawn_script_resource_cache_server(
distinct: bool,
) -> (String, std::sync::Arc<std::sync::atomic::AtomicUsize>) {
@@ -6866,6 +6953,77 @@ mod tests {
);
}
+ #[tokio::test(flavor = "current_thread")]
+ async fn document_write_external_script_blocks_later_parser_scripts() {
+ let (base, requests) = spawn_delayed_classic_script_server(
+ std::time::Duration::from_millis(75),
+ "globalThis.__writtenOrder.push('external'); globalThis.__writtenValue = 1;",
+ );
+ let html = format!(
+ r#"<html><body>
+ <script>globalThis.__writtenOrder = [];</script>
+ <script>
+ document.write('<script src="{base}/written.js"><\/script>');
+ document.write('<script>globalThis.__writtenOrder.push("written:" + String(globalThis.__writtenValue));<\/script>');
+ </script>
+ <script>globalThis.__writtenOrder.push('after:' + String(globalThis.__writtenValue));</script>
+ </body></html>"#,
+ );
+ let mut page = import_map_test_page(
+ "document-write-parser-blocking",
+ "http://127.0.0.1:9",
+ &html,
+ );
+
+ page.execute_scripts().await;
+
+ assert_eq!(
+ requests
+ .recv_timeout(std::time::Duration::from_secs(1))
+ .unwrap(),
+ "/written.js",
+ );
+ assert_eq!(
+ page.js
+ .as_mut()
+ .unwrap()
+ .evaluate("globalThis.__writtenOrder")
+ .unwrap(),
+ serde_json::json!(["external", "written:1", "after:1"]),
+ );
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn document_write_does_not_start_later_async_script_before_blocker() {
+ let base = spawn_written_script_order_server();
+ let html = format!(
+ r#"<html><body><script>
+ document.write('<script src="{base}/vendor.js"><\/script>');
+ document.write('<script async src="{base}/app.js"><\/script>');
+ </script><script>
+ globalThis.__afterWriteSawVendor = globalThis.__vendorValue;
+ </script></body></html>"#,
+ );
+ let mut page = import_map_test_page(
+ "document-write-async-after-blocker",
+ "http://127.0.0.1:9",
+ &html,
+ );
+
+ page.execute_scripts().await;
+
+ assert_eq!(
+ page.js
+ .as_mut()
+ .unwrap()
+ .evaluate(
+ "[globalThis.__afterWriteSawVendor, globalThis.__appSawVendor, globalThis.__appRequestFollowedVendor]"
+ )
+ .unwrap(),
+ serde_json::json!([1, 1, true]),
+ );
+ }
+
#[tokio::test(flavor = "current_thread")]
async fn load_delaying_script_progresses_through_continuously_ready_timer_work() {
let (base, requests) = spawn_delayed_classic_script_server(
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index b82685cd..82576e07 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -28,7 +28,7 @@ const __obscuraCore = globalThis.Deno.core;
'__obscura_registerLinkedStylesheet', '__obscura_activateLabel',
'__obscura_isDisabled', '__obscura_labeledControl', '__obscura_interactiveHost',
'__markParserScripts', '__obscura_hasPendingDynamicScripts',
- '__obscura_hasPendingLoadDelayingScripts',
+ '__obscura_hasPendingLoadDelayingScripts', '__obscura_hasPendingParserBlockingScripts',
'__obscura_nextPendingTimeoutDelay',
'__obscura_hw', '__obscura_mem',
'__documentReadyState__', '__currentUrl',
@@ -267,6 +267,10 @@ let __dynScriptQueue = [];
let __dynScriptBusy = false;
let __dynClassicPending = 0;
let __dynLoadDelayingPending = 0;
+let __parserBlockingScriptQueue = [];
+let __parserBlockingScriptBusy = false;
+let __parserBlockingScriptPending = 0;
+const __documentWriteScripts = new WeakSet();
Object.defineProperty(globalThis, '__obscura_hasPendingDynamicScripts', {
value: function() {
return __dynClassicPending > 0 || __dynScriptBusy || __dynScriptQueue.length > 0;
@@ -286,6 +290,12 @@ Object.defineProperty(globalThis, '__obscura_hasPendingLoadDelayingScripts', {
enumerable: false,
configurable: false,
});
+Object.defineProperty(globalThis, '__obscura_hasPendingParserBlockingScripts', {
+ value: function() { return __parserBlockingScriptPending > 0; },
+ writable: false,
+ enumerable: false,
+ configurable: false,
+});
function _decodeDataScriptUrl(url) {
const comma = url.indexOf(',');
if (!url.startsWith('data:') || comma < 5) {
@@ -371,10 +381,13 @@ async function __runDynScriptTask(task) {
if (task.isModule) {
await import(task.url);
} else {
- if (!task.fetchResult) __startDynClassicFetch(task);
- const fetched = await task.fetchResult;
- if (fetched.error) throw fetched.error;
- const body = fetched.body;
+ let body = task.inlineCode;
+ if (body === undefined) {
+ if (!task.fetchResult) __startDynClassicFetch(task);
+ const fetched = await task.fetchResult;
+ if (fetched.error) throw fetched.error;
+ body = fetched.body;
+ }
if (body) {
// A fetched async script is executed by a ScriptRunner task, not by
// the fetch promise's microtask continuation. Besides matching event
@@ -429,6 +442,23 @@ async function __processDynScriptQueue() {
__dynScriptBusy = false;
}
}
+async function __processParserBlockingScriptQueue() {
+ if (__parserBlockingScriptBusy) return;
+ __parserBlockingScriptBusy = true;
+ try {
+ while (__parserBlockingScriptQueue.length > 0) {
+ const entry = __parserBlockingScriptQueue.shift();
+ if (!entry.blocking) {
+ __runAsyncClassicScript(entry.task);
+ continue;
+ }
+ try { await __runDynScriptTask(entry.task); }
+ finally { __parserBlockingScriptPending--; }
+ }
+ } finally {
+ __parserBlockingScriptBusy = false;
+ }
+}
// Resolve a resource URL (script src / link href) against <base href> or the
// document URL, the way the inline dynamic-script path does. Guarded so a bad
// base or href never throws into appendChild.
@@ -1948,6 +1978,10 @@ function __prepareInsertedScript(script) {
const code = src ? "" : script.textContent;
if (!src && !code) return;
const prevNid = globalThis.__currentScriptNid;
+ const parserInserted = __documentWriteScripts.has(script);
+ const parserBlocking = parserInserted
+ && !isModule
+ && (!src || (!script.hasAttribute('async') && !script.hasAttribute('defer')));
if (src) {
let baseHref;
try {
@@ -1982,6 +2016,13 @@ function __prepareInsertedScript(script) {
// not turn already-prepared work into a post-load enhancement.
task.delaysLoad = globalThis.document?.readyState !== 'complete';
if (task.delaysLoad) __dynLoadDelayingPending++;
+ if (parserInserted && !isModule) {
+ task.prevNid = 0;
+ if (parserBlocking) __parserBlockingScriptPending++;
+ __parserBlockingScriptQueue.push({ task, blocking: parserBlocking });
+ __processParserBlockingScriptQueue();
+ return;
+ }
// A non-parser-inserted classic script is force-async unless script code
// explicitly assigned `.async = false`. Keep that opt-out in insertion
// order; default/async=true scripts fetch concurrently and execute as soon
@@ -2018,6 +2059,20 @@ function __prepareInsertedScript(script) {
if (task.delaysLoad) __dynLoadDelayingPending++;
__dynScriptQueue.push(task);
__processDynScriptQueue();
+ } else if (parserBlocking) {
+ __parserBlockingScriptPending++;
+ __parserBlockingScriptQueue.push({
+ blocking: true,
+ task: {
+ inlineCode: code,
+ isModule: false,
+ nid: script._nid,
+ prevNid: 0,
+ dispatchEvent: () => {},
+ delaysLoad: false,
+ },
+ });
+ __processParserBlockingScriptQueue();
} else {
globalThis.__currentScriptNid = script._nid;
try { (0, eval)(code); }
@@ -5898,6 +5953,9 @@ class Document extends Node {
var parentNid = +placements[i][0];
var node = _wrap(+placements[i][1]);
if (!node) continue;
+ if (node.nodeType === 1 && node.tagName === 'SCRIPT') {
+ __documentWriteScripts.add(node);
+ }
if (parentNid) {
var parent = _wrap(parentNid);
if (parent) parent.appendChild(node);
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index 14d5226e..2c175680 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -1,4 +1,4 @@
-use std::cell::RefCell;
+use std::cell::{Cell, RefCell};
use std::collections::{HashMap, HashSet, VecDeque};
use std::rc::Rc;
use std::sync::Arc;
@@ -302,6 +302,9 @@ pub struct ObscuraState {
/// The document's input stream for `document.write()`, created on the first call.
/// Why the calls share one parser is in `write_stream`.
pub(crate) write_stream: RefCell<Option<crate::write_stream::DocumentWriteStream>>,
+ /// Cheap native signal that the current parser script wrote another
+ /// script. This avoids polling V8 after every ordinary parser script.
+ pub(crate) document_write_inserted_script: Cell<bool>,
}
/// A frame document waiting to be given a realm.
@@ -436,6 +439,7 @@ impl ObscuraState {
import_map: Rc::new(RefCell::new(ImportMap::default())),
already_started_scripts: RefCell::new(HashSet::new()),
write_stream: RefCell::new(None),
+ document_write_inserted_script: Cell::new(false),
}
}
}
@@ -2163,8 +2167,14 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
"document_write" => {
let mut slot = gs.write_stream.borrow_mut();
let stream = slot.get_or_insert_with(DocumentWriteStream::new);
- let pairs: Vec<[i32; 2]> = stream
- .write(&arg2, dom)
+ let placements = stream.write(&arg2, dom);
+ if placements
+ .iter()
+ .any(|placement| node_is_script(dom, placement.node))
+ {
+ gs.document_write_inserted_script.set(true);
+ }
+ let pairs: Vec<[i32; 2]> = placements
.iter()
.map(|placement| {
[
@@ -2178,6 +2188,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
// document.open() discards what the input stream holds and starts over.
"document_write_reset" => {
*gs.write_stream.borrow_mut() = None;
+ gs.document_write_inserted_script.set(false);
"true".into()
}
"set_text_content" => {
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index 6e5fd0c8..53440fea 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -3343,6 +3343,26 @@ impl ObscuraJsRuntime {
.unwrap_or(false)
}
+ /// Whether a parser-blocking script inserted by `document.write()` still
+ /// has fetch or evaluation work outstanding.
+ pub fn has_pending_parser_blocking_scripts(&mut self) -> bool {
+ self.evaluate(
+ "globalThis.__obscura_hasPendingParserBlockingScripts?.() === true",
+ )
+ .ok()
+ .and_then(|value| value.as_bool())
+ .unwrap_or(false)
+ }
+
+ /// Consume the native signal that the current parser script called
+ /// `document.write()` with a script node.
+ pub fn take_document_write_inserted_script(&self) -> bool {
+ self.state
+ .borrow()
+ .document_write_inserted_script
+ .replace(false)
+ }
+
/// Generation of observable connected-document mutations. This excludes
/// detached-tree construction and no-op writes, which cannot affect a
/// screenshot or DOM dump.
@@ -13952,6 +13972,7 @@ mod tests {
let mut rt = setup_runtime("<html><body></body></html>");
assert!(!rt.has_pending_dynamic_scripts());
assert!(!rt.has_pending_load_delaying_scripts());
+ assert!(!rt.has_pending_parser_blocking_scripts());
assert_eq!(rt.next_pending_timeout_delay_ms(), None);
assert_eq!(
rt.evaluate("typeof __dynScriptBusy").unwrap(),
@@ -13978,6 +13999,13 @@ mod tests {
.unwrap(),
serde_json::json!(false)
);
+ assert_eq!(
+ rt.evaluate(
+ "Reflect.ownKeys(globalThis).includes('__obscura_hasPendingParserBlockingScripts')"
+ )
+ .unwrap(),
+ serde_json::json!(false)
+ );
assert_eq!(
rt.evaluate(
"Object.getOwnPropertyNames(globalThis).includes('__obscura_nextPendingTimeoutDelay')"