The headless browser for AI agents and web scraping
fix(js): parse replacement documents and flush written input at EOF
SGavrl721e40ff4e91
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rsindex e6c64cea..098cc840 100644--- a/crates/obscura-browser/src/page.rs+++ b/crates/obscura-browser/src/page.rs@@ -3745,7 +3745,7 @@ impl Page {self.js = None;self.url = Some(Url::parse("about:blank").unwrap());self.dom = Some(parse_html(- "<!DOCTYPE html><html><head></head><body></body></html>",+ "<html><head></head><body></body></html>",));self.title = String::new();self.lifecycle = LifecycleState::Loaded;@@ -4996,6 +4996,19 @@ mod tests {use base64::Engine as _;use obscura_dom::parse_html;+ #[tokio::test(flavor = "current_thread")]+ async fn blank_navigation_has_no_doctype_and_starts_in_quirks_mode() {+ let context = std::sync::Arc::new(crate::BrowserContext::new("blank-mode".into()));+ let mut page = super::Page::new("blank-mode".into(), context);+ page.navigate("about:blank").await.unwrap();+ assert_eq!(page.evaluate(r#"(() => [+ document.compatMode, document.doctype,+ Array.from(document.childNodes, node => node.nodeType),+ document.documentElement.nodeName, document.head.nodeName, document.body.nodeName+ ])()"#), serde_json::json!(["BackCompat", null, [1], "HTML", "HEAD", "BODY"]));+ assert!(page.with_dom(|dom| dom.is_quirks()).unwrap());+ }+#[tokio::test(flavor = "current_thread")]async fn network_idle_quiet_deadline_does_not_gain_an_extra_polling_slice() {use std::future::Future as _;diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.jsindex b651b5f2..8b0a694b 100644--- a/crates/obscura-js/js/bootstrap.js+++ b/crates/obscura-js/js/bootstrap.js@@ -149,12 +149,12 @@ const _DOM_MUTATION_COMMANDS = new Set(["append_child", "insert_before", "remove_child","set_attribute", "remove_attribute","set_text_content", "set_inner_html", "set_inner_html_context",- "set_fragment_html_executable", "document_write",+ "set_fragment_html_executable", "document_write", "document_write_close",]);const _DOM_TREE_MUTATION_COMMANDS = new Set(["append_child", "insert_before", "remove_child","set_inner_html", "set_inner_html_context", "set_fragment_html_executable",- "document_write",+ "document_write", "document_write_close",]);// Which realm this bootstrap closure belongs to. Every wrapper's methods come// from its own realm's prototypes, so a DOM call names the document it belongs@@ -5801,7 +5801,11 @@ class Document extends Node {get nodeType() { return 9; }get nodeName() { return "#document"; }get ownerDocument() { return null; } // Document has no ownerDocument- get compatMode() { return "CSS1Compat"; }+ get compatMode() {+ const get = _documentRealmMember(this, 'compatMode');+ return get ? Reflect.apply(get, this, [])+ : (this._nid == null ? "CSS1Compat" : _dom("document_compat_mode"));+ }// The document's character encoding, detected from the response charset// (HTTP Content-Type -> <meta charset>). characterSet/charset/inputEncoding// are WHATWG aliases. A node-less document (DOMParser/createDocument) has no@@ -6298,50 +6302,14 @@ class Document extends Node {// one per attribute, then ">".// https://html.spec.whatwg.org/multipage/dynamic-markup-insertion.html#dom-document-writewrite(...args) {+ const method = _documentRealmMember(this, 'write');+ if (method) return Reflect.apply(method, this, args);var html = args.join('');if (!html) return;- var body = this.body;- if (!body) return;// The host parses into the input stream and returns [[parent, node], …], parents first. The// insertion stays here, because appendChild does more than append: it reports the// mutation, registers window named access, and loads a written stylesheet.- var placements = _domParse("document_write", "", html) || [];- // The insertion point is the position of the running script. What it writes belongs- // behind it, not at the end of the body. The point moves along with every node placed,- // even across calls, so that a script's second call lands behind the first instead of- // directly behind the script again.- var scriptNid = globalThis.__currentScriptNid || 0;- var after = null;- if (scriptNid) {- var anchorNid = this._writeAnchorScript === scriptNid && this._writeAnchorNid- ? this._writeAnchorNid- : scriptNid;- var anchor = _wrap(anchorNid);- if (anchor && anchor.parentNode) after = anchor;- }- for (var i = 0; i < placements.length; i++) {- var parentNid = +placements[i][0];- var node = _wrap(+placements[i][1]);- if (!node) continue;- if (node.nodeType === 1 && node.tagName === 'SCRIPT') {- __documentWriteScripts.add(node);- }- if (parentNid) {- var parent = _wrap(parentNid);- if (parent) parent.appendChild(node);- continue;- }- if (after) {- after.parentNode.insertBefore(node, after.nextSibling);- after = node;- } else {- body.appendChild(node);- }- }- if (scriptNid && after) {- this._writeAnchorScript = scriptNid;- this._writeAnchorNid = after._nid;- }+ _insertWrittenNodes(this, _domParse("document_write", "", html) || []);}writeln(...args) {this.write(args.join('') + '\n');@@ -6350,10 +6318,10 @@ class Document extends Node {const method = _documentRealmMember(this, 'open');if (method) return Reflect.apply(method, this, []);// Native algorithms use the receiver's tree, not script-overridden getters.- const head = _wrapEl(+_dom('query_selector', 'head'));- if (head) head.innerHTML = '';- const body = _wrapEl(+_dom('query_selector', 'body'));- if (body) body.innerHTML = '';+ for (const nid of _domParse('child_nodes', this._nid) || []) {+ Node.prototype.removeChild.call(this, _wrap(nid));+ }+ this._doctype = undefined;// A new parse begins. Whatever the input stream still held is gone._dom("document_write_reset");this._writeAnchorScript = 0;@@ -6369,6 +6337,7 @@ class Document extends Node {if (method) return Reflect.apply(method, this, []);if (!this._writeOpen) return;this._writeOpen = false;+ _insertWrittenNodes(this, _domParse('document_write_close') || []);const generation = this._writeGeneration;const finishParsing = () => {if (generation !== this._writeGeneration) return;@@ -6401,6 +6370,40 @@ class Document extends Node {execCommand() { return false; }}+// Shared by write and the final EOF flush. A nonnegative parent is a real node,+// including document node 0; -1 denotes the running script's insertion point.+function _insertWrittenNodes(doc, placements) {+ var scriptNid = globalThis.__currentScriptNid || 0;+ var after = null;+ if (scriptNid) {+ var anchorNid = doc._writeAnchorScript === scriptNid && doc._writeAnchorNid+ ? doc._writeAnchorNid : scriptNid;+ var anchor = _wrap(anchorNid);+ if (anchor && anchor.parentNode) after = anchor;+ }+ for (var i = 0; i < placements.length; i++) {+ var parentNid = +placements[i][0];+ var node = _wrap(+placements[i][1]);+ if (!node) continue;+ if (node.nodeType === 10) doc._doctype = undefined;+ if (node.nodeType === 1 && node.tagName === 'SCRIPT') __documentWriteScripts.add(node);+ if (parentNid >= 0) {+ var parent = _wrap(parentNid);+ if (parent) parent.appendChild(node);+ } else if (after) {+ after.parentNode.insertBefore(node, after.nextSibling);+ after = node;+ } else {+ var body = _wrapEl(+_dom('query_selector', 'body'));+ if (body) body.appendChild(node);+ }+ }+ if (scriptNid && after) {+ doc._writeAnchorScript = scriptNid;+ doc._writeAnchorNid = after._nid;+ }+}+// Preserve the receiver realm's implementations even if a membrane remaps the// document's public prototype. The ordinary own-document path makes no op call.function _documentRealmMember(receiver, name) {@@ -16797,7 +16800,7 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint)// Capture late-defined members too, before page code can replace them.const _nativeElementClick = Element.prototype.click;const _documentMembers = Object.freeze(Object.fromEntries([- ...['URL', 'defaultView', 'readyState', 'getElementById', 'querySelector', 'querySelectorAll', 'open', 'close', 'elementFromPoint'].map(name => {+ ...['URL', 'defaultView', 'readyState', 'compatMode', 'getElementById', 'querySelector', 'querySelectorAll', 'open', 'write', 'close', 'elementFromPoint'].map(name => {const descriptor = Object.getOwnPropertyDescriptor(Document.prototype, name);return [name, descriptor.value || descriptor.get];}),diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex f37d6a61..4adc3dab 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -1959,6 +1959,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->"null".to_string()}"document_node_id" => dom.document().index().to_string(),+ "document_compat_mode" => if dom.is_quirks() { "BackCompat" } else { "CSS1Compat" }.into(),"document_title" => {// The DOM is authoritative after parsing. In particular, script// changes through title.textContent must be reflected by@@ -2471,14 +2472,23 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->// document.write() feeds the document's input stream, so the calls// share one parser and one tokenizer state. Returns the nodes that// became complete with this call, for the caller to run scripts among.- // Returns [[parent, node], …], parents before children. A `parent` of 0 means the node+ // Returns [[parent, node], …], parents before children. A `parent` of -1 means the node// belongs at the insertion point, which the caller knows. Nothing is inserted here:// that must go through Node.appendChild on the JS side, because that call also reports// the mutation, registers window named access, and loads a written stylesheet.- "document_write" => {+ "document_write" | "document_write_close" => {let mut slot = gs.write_stream.borrow_mut();- let stream = slot.get_or_insert_with(DocumentWriteStream::new);- let placements = stream.write(&arg2, dom);+ let placements = if cmd == "document_write_close" {+ let Some(mut stream) = slot.take() else { return "[]".into(); };+ let placements = stream.close(dom);+ // EOF in a script's text insertion mode marks it already started.+ // Newly released scripts have no closing tag and must not execute.+ gs.already_started_scripts.borrow_mut().extend(placements.iter()+ .filter(|p| node_is_script(dom, p.node)).map(|p| p.node));+ placements+ } else {+ slot.get_or_insert_with(|| DocumentWriteStream::new(false)).write(&arg2, dom)+ };if placements.iter().any(|placement| node_is_script(dom, placement.node))@@ -2489,7 +2499,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->.iter().map(|placement| {[- placement.parent.map_or(0, |id| id.index() as i32),+ placement.parent.map_or(-1, |id| id.index() as i32),placement.node.index() as i32,]})@@ -2498,7 +2508,8 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->}// document.open() discards what the input stream holds and starts over."document_write_reset" => {- *gs.write_stream.borrow_mut() = None;+ dom.set_quirks(false);+ *gs.write_stream.borrow_mut() = Some(DocumentWriteStream::new(true));gs.document_write_inserted_script.set(false);"true".into()}@@ -5663,7 +5674,7 @@ fn op_initial_frame({let parent = parent_state.borrow();child.dom = Some(obscura_dom::parse_html(- "<!DOCTYPE html><html><head></head><body></body></html>"));+ "<html><head></head><body></body></html>"));child.frame_id = frame_id;child.about_base_url = document_base_url(&parent);child.inherited_origin = Some(parent.inherited_origin.clone().unwrap_or_else(|| {diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rsindex c3f64b0d..be4fe1d9 100644--- a/crates/obscura-js/src/runtime.rs+++ b/crates/obscura-js/src/runtime.rs@@ -4980,6 +4980,24 @@ mod tests {);}+ #[test]+ fn initial_blank_iframe_has_no_doctype_and_starts_in_quirks_mode() {+ let mut rt = setup_runtime("<!doctype html><body></body>");+ assert_eq!(rt.evaluate(r#"(() => {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ const child = iframe.contentWindow;+ const initial = JSON.parse(child.eval(`JSON.stringify([+ document.compatMode, document.doctype,+ Array.from(document.childNodes, node => node.nodeType),+ document.documentElement.nodeName, document.head.nodeName, document.body.nodeName+ ])`));+ return [initial, document.compatMode];+ })()"#).unwrap(), serde_json::json!([+ ["BackCompat", null, [1], "HTML", "HEAD", "BODY"], "CSS1Compat"+ ]));+ }+#[test]fn initial_blank_iframe_eval_uses_the_child_realm() {let mut rt = setup_runtime("<html><body></body></html>");@@ -18865,11 +18883,151 @@ mod tests {}#[test]- fn test_document_open_clears_body() {- let mut rt = setup_runtime("<html><body><p>Old content</p></body></html>");- rt.evaluate("document.open()").unwrap();- let html = rt.evaluate("document.body.innerHTML").unwrap();- assert_eq!(html.as_str().unwrap(), "");+ fn document_open_removes_old_document_tree_before_writing() {+ let mut rt = setup_runtime("<!doctype html><html><body><p>Old content</p></body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ const oldRoot = document.documentElement, oldType = document.doctype;+ const opened = document.open();+ return [opened === document, document.documentElement, document.head,+ document.body, document.doctype, document.childNodes.length,+ oldRoot.isConnected, oldType.isConnected];+ })()"#).unwrap(), serde_json::json!([true, null, null, null, null, 0, false, false]));+ }++ #[test]+ fn document_open_close_emits_the_empty_document_at_eof() {+ let mut rt = setup_runtime("<html><body>old</body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ document.open();+ const emptyBeforeClose = document.childNodes.length === 0;+ document.close();+ return [emptyBeforeClose, document.documentElement.nodeName,+ document.head.nodeName, document.body.nodeName, document.body.childNodes.length];+ })()"#).unwrap(), serde_json::json!([true, "HTML", "HEAD", "BODY", 0]));+ }++ #[test]+ fn document_open_close_retains_unclosed_script_without_running_it() {+ let mut rt = setup_runtime("<html><body></body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ globalThis.unclosedWriteRan = false;+ document.open();+ document.write('<script id="unclosed">globalThis.unclosedWriteRan = true;');+ document.close();+ return [document.getElementById('unclosed').parentNode.nodeName,+ document.getElementById('unclosed').textContent, globalThis.unclosedWriteRan];+ })()"#).unwrap(), serde_json::json!([+ "HEAD", "globalThis.unclosedWriteRan = true;", false+ ]));+ }++ #[test]+ fn document_open_write_borrowed_methods_use_the_receiver_parser() {+ let mut rt = setup_runtime("<html><body id='parent'>parent</body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ const childDocument = iframe.contentDocument;+ Document.prototype.open.call(childDocument);+ Document.prototype.write.call(childDocument, '<title>child</title><p id="written">value</p>');+ Document.prototype.close.call(childDocument);+ return [document.body.id, document.getElementById('written'),+ childDocument.title, childDocument.getElementById('written').textContent,+ childDocument.querySelector('title').parentNode.nodeName];+ })()"#).unwrap(), serde_json::json!(["parent", null, "child", "value", "HEAD"]));+ }++ #[test]+ fn document_open_write_uses_document_insertion_modes_across_calls() {+ let mut rt = setup_runtime("<html><head><meta id='old'></head><body>old</body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ document.open();+ document.write('<sty');+ document.write('le id="written-style">body{margin:0}');+ document.write('</style>\n <input id="widget">');+ document.close();+ const widget = document.getElementById('widget');+ const range = document.createRange(); range.selectNode(widget);+ return [document.getElementById('written-style').parentNode.nodeName,+ Array.from(document.head.children, n => n.nodeName),+ Array.from(document.body.childNodes, n => n.nodeName),+ range.startContainer.nodeName, range.startOffset,+ document.getElementById('old') === null];+ })()"#).unwrap(), serde_json::json!(["HEAD", ["STYLE"], ["INPUT"], "BODY", 0, true]));+ }++ #[test]+ fn document_open_write_preserves_explicit_document_structure() {+ let mut rt = setup_runtime("<html><head></head><body></body></html>");+ assert_eq!(rt.evaluate(r#"(() => {+ document.open();+ document.write('<!doctype html><html lang="fr"><head><title>Written</title></head>');+ document.write('<body id="written-body"><table><tr><td id="cell">value</td></tr></table></body></html>');+ document.close();+ return [document.querySelectorAll('html').length, document.querySelectorAll('head').length,+ document.querySelectorAll('body').length, document.documentElement.lang,+ document.body.id, document.querySelector('title').parentNode.nodeName,+ document.getElementById('cell').parentNode.parentNode.nodeName,+ Array.from(document.childNodes, n => n.nodeType)];+ })()"#).unwrap(), serde_json::json!([1, 1, 1, "fr", "written-body", "HEAD", "TBODY", [10, 1]]));+ }++ #[test]+ fn document_compat_mode_reads_the_parsed_document_mode() {+ for (html, expected) in [+ ("<p>quirks</p>", "BackCompat"),+ ("<!doctype html><p>standards</p>", "CSS1Compat"),+ ("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\"><p>limited</p>", "CSS1Compat"),+ ] {+ let mut rt = setup_runtime(html);+ assert_eq!(rt.evaluate("document.compatMode").unwrap(), serde_json::json!(expected), "{html}");+ }+ }++ #[test]+ fn document_open_write_updates_quirks_mode_without_fragment_leaks() {+ let mut rt = setup_runtime("<!doctype html><p>old</p>");+ assert_eq!(rt.evaluate(r#"(() => {+ const modes = [document.compatMode];+ document.open(); modes.push(document.compatMode);+ document.write('<p>new</p>'); modes.push(document.compatMode);+ document.close(); modes.push(document.compatMode);+ document.open(); modes.push(document.compatMode);+ document.write('<!doctype html><p>standards</p>'); modes.push(document.compatMode);+ document.close();+ document.body.innerHTML = '<p>fragment</p>';+ document.write('<span>fragment write</span>'); modes.push(document.compatMode);+ return modes;+ })()"#).unwrap(), serde_json::json!([+ "CSS1Compat", "CSS1Compat", "BackCompat", "BackCompat",+ "CSS1Compat", "CSS1Compat", "CSS1Compat"+ ]));+ }++ #[test]+ fn document_open_close_sets_quirks_mode_at_eof() {+ let mut rt = setup_runtime("<!doctype html><p>old</p>");+ assert_eq!(rt.evaluate(r#"(() => {+ document.open();+ const before = document.compatMode;+ document.close();+ return [before, document.compatMode];+ })()"#).unwrap(), serde_json::json!(["CSS1Compat", "BackCompat"]));+ }++ #[test]+ fn document_compat_mode_borrowed_getter_uses_the_receiver_realm() {+ let mut rt = setup_runtime("<!doctype html><body></body>");+ assert_eq!(rt.evaluate(r#"(() => {+ const iframe = document.createElement('iframe');+ document.body.appendChild(iframe);+ const child = iframe.contentDocument;+ Document.prototype.open.call(child);+ Document.prototype.write.call(child, '<p>quirks</p>');+ Document.prototype.close.call(child);+ const getMode = Object.getOwnPropertyDescriptor(Document.prototype, 'compatMode').get;+ return [getMode.call(child), document.compatMode];+ })()"#).unwrap(), serde_json::json!(["BackCompat", "CSS1Compat"]));}#[test]diff --git a/crates/obscura-js/src/write_stream.rs b/crates/obscura-js/src/write_stream.rsindex 3f45a9bb..31280970 100644--- a/crates/obscura-js/src/write_stream.rs+++ b/crates/obscura-js/src/write_stream.rs@@ -11,7 +11,7 @@use std::cell::RefCell;use std::collections::{HashMap, HashSet};-use html5ever::driver::{parse_fragment, ParseOpts, Parser};+use html5ever::driver::{parse_document, parse_fragment, ParseOpts, Parser};use html5ever::tendril::TendrilSink;use html5ever::tree_builder::Tracer;use html5ever::{local_name, ns, QualName};@@ -52,6 +52,7 @@ fn needs_to_be_complete(source: &DomTree, node: NodeId) -> bool {pub(crate) struct DocumentWriteStream {parser: Parser<DomTree>,+ document: bool,/// Maps a node of the parser tree to its copy in the document. A node missing here has/// not been handed over yet.handed_over: HashMap<NodeId, NodeId>,@@ -60,11 +61,16 @@ pub(crate) struct DocumentWriteStream {}impl DocumentWriteStream {- pub(crate) fn new() -> Self {+ pub(crate) fn new(document: bool) -> Self {// Scripting enabled, as with the fragment parser behind innerHTML.let context = QualName::new(None, ns!(html), local_name!("body"));DocumentWriteStream {- parser: parse_fragment(DomTree::new(), ParseOpts::default(), context, vec![], true),+ parser: if document {+ parse_document(DomTree::new(), ParseOpts::default())+ } else {+ parse_fragment(DomTree::new(), ParseOpts::default(), context, vec![], true)+ },+ document,handed_over: HashMap::new(),staging: None,}@@ -74,17 +80,28 @@ impl DocumentWriteStream {/// Returns the nodes to insert, parents before children.pub(crate) fn write(&mut self, html: &str, dom: &DomTree) -> Vec<Placement> {self.parser.process(html.into());+ self.mirror(dom)+ }++ pub(crate) fn close(&mut self, dom: &DomTree) -> Vec<Placement> {+ self.parser.tokenizer.end();+ self.mirror(dom)+ }+ fn mirror(&mut self, dom: &DomTree) -> Vec<Placement> {let retained = RetainedNodes::default();self.parser.tokenizer.sink.trace_handles(&retained);let retained = retained.0.into_inner();// Separate fields: the parser tree is read, the mapping is written.let source = &self.parser.tokenizer.sink.sink;+ if self.document {+ dom.set_quirks(source.is_quirks());+ }let handed_over = &mut self.handed_over;let staging = &mut self.staging;- let root = source.fragment_root();+ let root = if self.document { source.document() } else { source.fragment_root() };let mut placements = Vec::new();// Only look at what is new. A walk over all children would cost as much per call as// the input stream written so far is long, so quadratic over a thousand calls. Measured at@@ -115,7 +132,7 @@ impl DocumentWriteStream {}let parent = match node.parent {- Some(p) if p == root => None,+ Some(p) if p == root => self.document.then(|| dom.document()),// If the parent is held back, this one waits along.Some(p) => match handed_over.get(&p) {Some(©) => Some(copy),