Skip to content

The headless browser for AI agents and web scraping

fix(js): parse replacement documents and flush written input at EOF

SGavrl721e40ff4e91
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs
index e6c64cea..098cc840 100644
--- a/crates/obscura-browser/src/page.rs
+++ b/crates/obscura-browser/src/page.rs
@@ -3745,7 +3745,7 @@ impl Page {
self.js = None;
self.url = Some(Url::parse("about:blank").unwrap());
self.dom = Some(parse_html(
- "<!DOCTYPE html><html><head></head><body></body></html>",
+ "<html><head></head><body></body></html>",
));
self.title = String::new();
self.lifecycle = LifecycleState::Loaded;
@@ -4996,6 +4996,19 @@ mod tests {
use base64::Engine as _;
use obscura_dom::parse_html;
+ #[tokio::test(flavor = "current_thread")]
+ async fn blank_navigation_has_no_doctype_and_starts_in_quirks_mode() {
+ let context = std::sync::Arc::new(crate::BrowserContext::new("blank-mode".into()));
+ let mut page = super::Page::new("blank-mode".into(), context);
+ page.navigate("about:blank").await.unwrap();
+ assert_eq!(page.evaluate(r#"(() => [
+ document.compatMode, document.doctype,
+ Array.from(document.childNodes, node => node.nodeType),
+ document.documentElement.nodeName, document.head.nodeName, document.body.nodeName
+ ])()"#), serde_json::json!(["BackCompat", null, [1], "HTML", "HEAD", "BODY"]));
+ assert!(page.with_dom(|dom| dom.is_quirks()).unwrap());
+ }
+
#[tokio::test(flavor = "current_thread")]
async fn network_idle_quiet_deadline_does_not_gain_an_extra_polling_slice() {
use std::future::Future as _;
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index b651b5f2..8b0a694b 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -149,12 +149,12 @@ const _DOM_MUTATION_COMMANDS = new Set([
"append_child", "insert_before", "remove_child",
"set_attribute", "remove_attribute",
"set_text_content", "set_inner_html", "set_inner_html_context",
- "set_fragment_html_executable", "document_write",
+ "set_fragment_html_executable", "document_write", "document_write_close",
]);
const _DOM_TREE_MUTATION_COMMANDS = new Set([
"append_child", "insert_before", "remove_child",
"set_inner_html", "set_inner_html_context", "set_fragment_html_executable",
- "document_write",
+ "document_write", "document_write_close",
]);
// Which realm this bootstrap closure belongs to. Every wrapper's methods come
// from its own realm's prototypes, so a DOM call names the document it belongs
@@ -5801,7 +5801,11 @@ class Document extends Node {
get nodeType() { return 9; }
get nodeName() { return "#document"; }
get ownerDocument() { return null; } // Document has no ownerDocument
- get compatMode() { return "CSS1Compat"; }
+ get compatMode() {
+ const get = _documentRealmMember(this, 'compatMode');
+ return get ? Reflect.apply(get, this, [])
+ : (this._nid == null ? "CSS1Compat" : _dom("document_compat_mode"));
+ }
// The document's character encoding, detected from the response charset
// (HTTP Content-Type -> <meta charset>). characterSet/charset/inputEncoding
// are WHATWG aliases. A node-less document (DOMParser/createDocument) has no
@@ -6298,50 +6302,14 @@ class Document extends Node {
// one per attribute, then ">".
// https://html.spec.whatwg.org/multipage/dynamic-markup-insertion.html#dom-document-write
write(...args) {
+ const method = _documentRealmMember(this, 'write');
+ if (method) return Reflect.apply(method, this, args);
var html = args.join('');
if (!html) return;
- var body = this.body;
- if (!body) return;
// The host parses into the input stream and returns [[parent, node], …], parents first. The
// insertion stays here, because appendChild does more than append: it reports the
// mutation, registers window named access, and loads a written stylesheet.
- var placements = _domParse("document_write", "", html) || [];
- // The insertion point is the position of the running script. What it writes belongs
- // behind it, not at the end of the body. The point moves along with every node placed,
- // even across calls, so that a script's second call lands behind the first instead of
- // directly behind the script again.
- var scriptNid = globalThis.__currentScriptNid || 0;
- var after = null;
- if (scriptNid) {
- var anchorNid = this._writeAnchorScript === scriptNid && this._writeAnchorNid
- ? this._writeAnchorNid
- : scriptNid;
- var anchor = _wrap(anchorNid);
- if (anchor && anchor.parentNode) after = anchor;
- }
- for (var i = 0; i < placements.length; i++) {
- var parentNid = +placements[i][0];
- var node = _wrap(+placements[i][1]);
- if (!node) continue;
- if (node.nodeType === 1 && node.tagName === 'SCRIPT') {
- __documentWriteScripts.add(node);
- }
- if (parentNid) {
- var parent = _wrap(parentNid);
- if (parent) parent.appendChild(node);
- continue;
- }
- if (after) {
- after.parentNode.insertBefore(node, after.nextSibling);
- after = node;
- } else {
- body.appendChild(node);
- }
- }
- if (scriptNid && after) {
- this._writeAnchorScript = scriptNid;
- this._writeAnchorNid = after._nid;
- }
+ _insertWrittenNodes(this, _domParse("document_write", "", html) || []);
}
writeln(...args) {
this.write(args.join('') + '\n');
@@ -6350,10 +6318,10 @@ class Document extends Node {
const method = _documentRealmMember(this, 'open');
if (method) return Reflect.apply(method, this, []);
// Native algorithms use the receiver's tree, not script-overridden getters.
- const head = _wrapEl(+_dom('query_selector', 'head'));
- if (head) head.innerHTML = '';
- const body = _wrapEl(+_dom('query_selector', 'body'));
- if (body) body.innerHTML = '';
+ for (const nid of _domParse('child_nodes', this._nid) || []) {
+ Node.prototype.removeChild.call(this, _wrap(nid));
+ }
+ this._doctype = undefined;
// A new parse begins. Whatever the input stream still held is gone.
_dom("document_write_reset");
this._writeAnchorScript = 0;
@@ -6369,6 +6337,7 @@ class Document extends Node {
if (method) return Reflect.apply(method, this, []);
if (!this._writeOpen) return;
this._writeOpen = false;
+ _insertWrittenNodes(this, _domParse('document_write_close') || []);
const generation = this._writeGeneration;
const finishParsing = () => {
if (generation !== this._writeGeneration) return;
@@ -6401,6 +6370,40 @@ class Document extends Node {
execCommand() { return false; }
}
+// Shared by write and the final EOF flush. A nonnegative parent is a real node,
+// including document node 0; -1 denotes the running script's insertion point.
+function _insertWrittenNodes(doc, placements) {
+ var scriptNid = globalThis.__currentScriptNid || 0;
+ var after = null;
+ if (scriptNid) {
+ var anchorNid = doc._writeAnchorScript === scriptNid && doc._writeAnchorNid
+ ? doc._writeAnchorNid : scriptNid;
+ var anchor = _wrap(anchorNid);
+ if (anchor && anchor.parentNode) after = anchor;
+ }
+ for (var i = 0; i < placements.length; i++) {
+ var parentNid = +placements[i][0];
+ var node = _wrap(+placements[i][1]);
+ if (!node) continue;
+ if (node.nodeType === 10) doc._doctype = undefined;
+ if (node.nodeType === 1 && node.tagName === 'SCRIPT') __documentWriteScripts.add(node);
+ if (parentNid >= 0) {
+ var parent = _wrap(parentNid);
+ if (parent) parent.appendChild(node);
+ } else if (after) {
+ after.parentNode.insertBefore(node, after.nextSibling);
+ after = node;
+ } else {
+ var body = _wrapEl(+_dom('query_selector', 'body'));
+ if (body) body.appendChild(node);
+ }
+ }
+ if (scriptNid && after) {
+ doc._writeAnchorScript = scriptNid;
+ doc._writeAnchorNid = after._nid;
+ }
+}
+
// Preserve the receiver realm's implementations even if a membrane remaps the
// document's public prototype. The ordinary own-document path makes no op call.
function _documentRealmMember(receiver, name) {
@@ -16797,7 +16800,7 @@ if (typeof ShadowRoot !== 'undefined' && !ShadowRoot.prototype.elementFromPoint)
// Capture late-defined members too, before page code can replace them.
const _nativeElementClick = Element.prototype.click;
const _documentMembers = Object.freeze(Object.fromEntries([
- ...['URL', 'defaultView', 'readyState', 'getElementById', 'querySelector', 'querySelectorAll', 'open', 'close', 'elementFromPoint'].map(name => {
+ ...['URL', 'defaultView', 'readyState', 'compatMode', 'getElementById', 'querySelector', 'querySelectorAll', 'open', 'write', 'close', 'elementFromPoint'].map(name => {
const descriptor = Object.getOwnPropertyDescriptor(Document.prototype, name);
return [name, descriptor.value || descriptor.get];
}),
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index f37d6a61..4adc3dab 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -1959,6 +1959,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
"null".to_string()
}
"document_node_id" => dom.document().index().to_string(),
+ "document_compat_mode" => if dom.is_quirks() { "BackCompat" } else { "CSS1Compat" }.into(),
"document_title" => {
// The DOM is authoritative after parsing. In particular, script
// changes through title.textContent must be reflected by
@@ -2471,14 +2472,23 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
// document.write() feeds the document's input stream, so the calls
// share one parser and one tokenizer state. Returns the nodes that
// became complete with this call, for the caller to run scripts among.
- // Returns [[parent, node], …], parents before children. A `parent` of 0 means the node
+ // Returns [[parent, node], …], parents before children. A `parent` of -1 means the node
// belongs at the insertion point, which the caller knows. Nothing is inserted here:
// that must go through Node.appendChild on the JS side, because that call also reports
// the mutation, registers window named access, and loads a written stylesheet.
- "document_write" => {
+ "document_write" | "document_write_close" => {
let mut slot = gs.write_stream.borrow_mut();
- let stream = slot.get_or_insert_with(DocumentWriteStream::new);
- let placements = stream.write(&arg2, dom);
+ let placements = if cmd == "document_write_close" {
+ let Some(mut stream) = slot.take() else { return "[]".into(); };
+ let placements = stream.close(dom);
+ // EOF in a script's text insertion mode marks it already started.
+ // Newly released scripts have no closing tag and must not execute.
+ gs.already_started_scripts.borrow_mut().extend(placements.iter()
+ .filter(|p| node_is_script(dom, p.node)).map(|p| p.node));
+ placements
+ } else {
+ slot.get_or_insert_with(|| DocumentWriteStream::new(false)).write(&arg2, dom)
+ };
if placements
.iter()
.any(|placement| node_is_script(dom, placement.node))
@@ -2489,7 +2499,7 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
.iter()
.map(|placement| {
[
- placement.parent.map_or(0, |id| id.index() as i32),
+ placement.parent.map_or(-1, |id| id.index() as i32),
placement.node.index() as i32,
]
})
@@ -2498,7 +2508,8 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
}
// document.open() discards what the input stream holds and starts over.
"document_write_reset" => {
- *gs.write_stream.borrow_mut() = None;
+ dom.set_quirks(false);
+ *gs.write_stream.borrow_mut() = Some(DocumentWriteStream::new(true));
gs.document_write_inserted_script.set(false);
"true".into()
}
@@ -5663,7 +5674,7 @@ fn op_initial_frame(
{
let parent = parent_state.borrow();
child.dom = Some(obscura_dom::parse_html(
- "<!DOCTYPE html><html><head></head><body></body></html>"));
+ "<html><head></head><body></body></html>"));
child.frame_id = frame_id;
child.about_base_url = document_base_url(&parent);
child.inherited_origin = Some(parent.inherited_origin.clone().unwrap_or_else(|| {
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index c3f64b0d..be4fe1d9 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -4980,6 +4980,24 @@ mod tests {
);
}
+ #[test]
+ fn initial_blank_iframe_has_no_doctype_and_starts_in_quirks_mode() {
+ let mut rt = setup_runtime("<!doctype html><body></body>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ const iframe = document.createElement('iframe');
+ document.body.appendChild(iframe);
+ const child = iframe.contentWindow;
+ const initial = JSON.parse(child.eval(`JSON.stringify([
+ document.compatMode, document.doctype,
+ Array.from(document.childNodes, node => node.nodeType),
+ document.documentElement.nodeName, document.head.nodeName, document.body.nodeName
+ ])`));
+ return [initial, document.compatMode];
+ })()"#).unwrap(), serde_json::json!([
+ ["BackCompat", null, [1], "HTML", "HEAD", "BODY"], "CSS1Compat"
+ ]));
+ }
+
#[test]
fn initial_blank_iframe_eval_uses_the_child_realm() {
let mut rt = setup_runtime("<html><body></body></html>");
@@ -18865,11 +18883,151 @@ mod tests {
}
#[test]
- fn test_document_open_clears_body() {
- let mut rt = setup_runtime("<html><body><p>Old content</p></body></html>");
- rt.evaluate("document.open()").unwrap();
- let html = rt.evaluate("document.body.innerHTML").unwrap();
- assert_eq!(html.as_str().unwrap(), "");
+ fn document_open_removes_old_document_tree_before_writing() {
+ let mut rt = setup_runtime("<!doctype html><html><body><p>Old content</p></body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ const oldRoot = document.documentElement, oldType = document.doctype;
+ const opened = document.open();
+ return [opened === document, document.documentElement, document.head,
+ document.body, document.doctype, document.childNodes.length,
+ oldRoot.isConnected, oldType.isConnected];
+ })()"#).unwrap(), serde_json::json!([true, null, null, null, null, 0, false, false]));
+ }
+
+ #[test]
+ fn document_open_close_emits_the_empty_document_at_eof() {
+ let mut rt = setup_runtime("<html><body>old</body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ document.open();
+ const emptyBeforeClose = document.childNodes.length === 0;
+ document.close();
+ return [emptyBeforeClose, document.documentElement.nodeName,
+ document.head.nodeName, document.body.nodeName, document.body.childNodes.length];
+ })()"#).unwrap(), serde_json::json!([true, "HTML", "HEAD", "BODY", 0]));
+ }
+
+ #[test]
+ fn document_open_close_retains_unclosed_script_without_running_it() {
+ let mut rt = setup_runtime("<html><body></body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ globalThis.unclosedWriteRan = false;
+ document.open();
+ document.write('<script id="unclosed">globalThis.unclosedWriteRan = true;');
+ document.close();
+ return [document.getElementById('unclosed').parentNode.nodeName,
+ document.getElementById('unclosed').textContent, globalThis.unclosedWriteRan];
+ })()"#).unwrap(), serde_json::json!([
+ "HEAD", "globalThis.unclosedWriteRan = true;", false
+ ]));
+ }
+
+ #[test]
+ fn document_open_write_borrowed_methods_use_the_receiver_parser() {
+ let mut rt = setup_runtime("<html><body id='parent'>parent</body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ const iframe = document.createElement('iframe');
+ document.body.appendChild(iframe);
+ const childDocument = iframe.contentDocument;
+ Document.prototype.open.call(childDocument);
+ Document.prototype.write.call(childDocument, '<title>child</title><p id="written">value</p>');
+ Document.prototype.close.call(childDocument);
+ return [document.body.id, document.getElementById('written'),
+ childDocument.title, childDocument.getElementById('written').textContent,
+ childDocument.querySelector('title').parentNode.nodeName];
+ })()"#).unwrap(), serde_json::json!(["parent", null, "child", "value", "HEAD"]));
+ }
+
+ #[test]
+ fn document_open_write_uses_document_insertion_modes_across_calls() {
+ let mut rt = setup_runtime("<html><head><meta id='old'></head><body>old</body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ document.open();
+ document.write('<sty');
+ document.write('le id="written-style">body{margin:0}');
+ document.write('</style>\n <input id="widget">');
+ document.close();
+ const widget = document.getElementById('widget');
+ const range = document.createRange(); range.selectNode(widget);
+ return [document.getElementById('written-style').parentNode.nodeName,
+ Array.from(document.head.children, n => n.nodeName),
+ Array.from(document.body.childNodes, n => n.nodeName),
+ range.startContainer.nodeName, range.startOffset,
+ document.getElementById('old') === null];
+ })()"#).unwrap(), serde_json::json!(["HEAD", ["STYLE"], ["INPUT"], "BODY", 0, true]));
+ }
+
+ #[test]
+ fn document_open_write_preserves_explicit_document_structure() {
+ let mut rt = setup_runtime("<html><head></head><body></body></html>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ document.open();
+ document.write('<!doctype html><html lang="fr"><head><title>Written</title></head>');
+ document.write('<body id="written-body"><table><tr><td id="cell">value</td></tr></table></body></html>');
+ document.close();
+ return [document.querySelectorAll('html').length, document.querySelectorAll('head').length,
+ document.querySelectorAll('body').length, document.documentElement.lang,
+ document.body.id, document.querySelector('title').parentNode.nodeName,
+ document.getElementById('cell').parentNode.parentNode.nodeName,
+ Array.from(document.childNodes, n => n.nodeType)];
+ })()"#).unwrap(), serde_json::json!([1, 1, 1, "fr", "written-body", "HEAD", "TBODY", [10, 1]]));
+ }
+
+ #[test]
+ fn document_compat_mode_reads_the_parsed_document_mode() {
+ for (html, expected) in [
+ ("<p>quirks</p>", "BackCompat"),
+ ("<!doctype html><p>standards</p>", "CSS1Compat"),
+ ("<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 4.01 Transitional//EN\" \"http://www.w3.org/TR/html4/loose.dtd\"><p>limited</p>", "CSS1Compat"),
+ ] {
+ let mut rt = setup_runtime(html);
+ assert_eq!(rt.evaluate("document.compatMode").unwrap(), serde_json::json!(expected), "{html}");
+ }
+ }
+
+ #[test]
+ fn document_open_write_updates_quirks_mode_without_fragment_leaks() {
+ let mut rt = setup_runtime("<!doctype html><p>old</p>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ const modes = [document.compatMode];
+ document.open(); modes.push(document.compatMode);
+ document.write('<p>new</p>'); modes.push(document.compatMode);
+ document.close(); modes.push(document.compatMode);
+ document.open(); modes.push(document.compatMode);
+ document.write('<!doctype html><p>standards</p>'); modes.push(document.compatMode);
+ document.close();
+ document.body.innerHTML = '<p>fragment</p>';
+ document.write('<span>fragment write</span>'); modes.push(document.compatMode);
+ return modes;
+ })()"#).unwrap(), serde_json::json!([
+ "CSS1Compat", "CSS1Compat", "BackCompat", "BackCompat",
+ "CSS1Compat", "CSS1Compat", "CSS1Compat"
+ ]));
+ }
+
+ #[test]
+ fn document_open_close_sets_quirks_mode_at_eof() {
+ let mut rt = setup_runtime("<!doctype html><p>old</p>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ document.open();
+ const before = document.compatMode;
+ document.close();
+ return [before, document.compatMode];
+ })()"#).unwrap(), serde_json::json!(["CSS1Compat", "BackCompat"]));
+ }
+
+ #[test]
+ fn document_compat_mode_borrowed_getter_uses_the_receiver_realm() {
+ let mut rt = setup_runtime("<!doctype html><body></body>");
+ assert_eq!(rt.evaluate(r#"(() => {
+ const iframe = document.createElement('iframe');
+ document.body.appendChild(iframe);
+ const child = iframe.contentDocument;
+ Document.prototype.open.call(child);
+ Document.prototype.write.call(child, '<p>quirks</p>');
+ Document.prototype.close.call(child);
+ const getMode = Object.getOwnPropertyDescriptor(Document.prototype, 'compatMode').get;
+ return [getMode.call(child), document.compatMode];
+ })()"#).unwrap(), serde_json::json!(["BackCompat", "CSS1Compat"]));
}
#[test]
diff --git a/crates/obscura-js/src/write_stream.rs b/crates/obscura-js/src/write_stream.rs
index 3f45a9bb..31280970 100644
--- a/crates/obscura-js/src/write_stream.rs
+++ b/crates/obscura-js/src/write_stream.rs
@@ -11,7 +11,7 @@
use std::cell::RefCell;
use std::collections::{HashMap, HashSet};
-use html5ever::driver::{parse_fragment, ParseOpts, Parser};
+use html5ever::driver::{parse_document, parse_fragment, ParseOpts, Parser};
use html5ever::tendril::TendrilSink;
use html5ever::tree_builder::Tracer;
use html5ever::{local_name, ns, QualName};
@@ -52,6 +52,7 @@ fn needs_to_be_complete(source: &DomTree, node: NodeId) -> bool {
pub(crate) struct DocumentWriteStream {
parser: Parser<DomTree>,
+ document: bool,
/// Maps a node of the parser tree to its copy in the document. A node missing here has
/// not been handed over yet.
handed_over: HashMap<NodeId, NodeId>,
@@ -60,11 +61,16 @@ pub(crate) struct DocumentWriteStream {
}
impl DocumentWriteStream {
- pub(crate) fn new() -> Self {
+ pub(crate) fn new(document: bool) -> Self {
// Scripting enabled, as with the fragment parser behind innerHTML.
let context = QualName::new(None, ns!(html), local_name!("body"));
DocumentWriteStream {
- parser: parse_fragment(DomTree::new(), ParseOpts::default(), context, vec![], true),
+ parser: if document {
+ parse_document(DomTree::new(), ParseOpts::default())
+ } else {
+ parse_fragment(DomTree::new(), ParseOpts::default(), context, vec![], true)
+ },
+ document,
handed_over: HashMap::new(),
staging: None,
}
@@ -74,17 +80,28 @@ impl DocumentWriteStream {
/// Returns the nodes to insert, parents before children.
pub(crate) fn write(&mut self, html: &str, dom: &DomTree) -> Vec<Placement> {
self.parser.process(html.into());
+ self.mirror(dom)
+ }
+
+ pub(crate) fn close(&mut self, dom: &DomTree) -> Vec<Placement> {
+ self.parser.tokenizer.end();
+ self.mirror(dom)
+ }
+ fn mirror(&mut self, dom: &DomTree) -> Vec<Placement> {
let retained = RetainedNodes::default();
self.parser.tokenizer.sink.trace_handles(&retained);
let retained = retained.0.into_inner();
// Separate fields: the parser tree is read, the mapping is written.
let source = &self.parser.tokenizer.sink.sink;
+ if self.document {
+ dom.set_quirks(source.is_quirks());
+ }
let handed_over = &mut self.handed_over;
let staging = &mut self.staging;
- let root = source.fragment_root();
+ let root = if self.document { source.document() } else { source.fragment_root() };
let mut placements = Vec::new();
// Only look at what is new. A walk over all children would cost as much per call as
// the input stream written so far is long, so quadratic over a thousand calls. Measured at
@@ -115,7 +132,7 @@ impl DocumentWriteStream {
}
let parent = match node.parent {
- Some(p) if p == root => None,
+ Some(p) if p == root => self.document.then(|| dom.document()),
// If the parent is held back, this one waits along.
Some(p) => match handed_over.get(&p) {
Some(&copy) => Some(copy),