The headless browser for AI agents and web scraping
fix(browser): build a plain-text document for non-HTML text responses
Yi-111-a22eb6b42388e
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rsindex fbdc129b..d85af845 100644--- a/crates/obscura-browser/src/page.rs+++ b/crates/obscura-browser/src/page.rs@@ -279,6 +279,9 @@ pub struct Page {/// Exposed to JS as `document.characterSet` and used for the URL query/// encoding override on `<a>`/`<area>` hrefs in legacy-charset documents.pub encoding: String,+ /// MIME type of the current document's main resource, lowercased and+ /// without parameters. Exposed to JS as `document.contentType`.+ pub content_type: String,/// Monotonic origin for the current document's CSS animation timeline./// It is reset once author styles are installed, so stylesheet download/// latency does not incorrectly advance newly-created animations.@@ -1113,6 +1116,7 @@ impl Page {device_scale_factor: 1.0,default_background_color_override: None,encoding: "UTF-8".to_string(),+ content_type: "text/html".to_string(),document_timeline_origin: std::time::Instant::now(),navigation_timing: NavigationTiming::default(),navigation_timeout: None,@@ -1778,6 +1782,7 @@ impl Page {);rt.set_url(&self.url_string());rt.set_encoding(&self.encoding);+ rt.set_content_type(&self.content_type);rt.set_title(&self.title);rt.set_referrer(&self.referrer);let (session_history, session_index) = self.predicted_session_history();@@ -3499,7 +3504,17 @@ impl Page {obscura_net::decode_response_with_name(&response.body, response.content_type());self.encoding = encoding_name.to_string();self.navigation_timing.record("domLoading");- let dom = parse_html(&body_text);+ // A non-HTML text response is a plain-text document, not an HTML one+ // that happens to hold text: every `<word>` in a .txt or .md body+ // would otherwise be parsed as an element and its contents dropped+ // from textContent (#1231). Chrome wraps the decoded text in a+ // single <pre> instead.+ let (dom, document_content_type) = match plain_text_document_type(response.content_type())+ {+ Some(mime) => (parse_plain_text_document(&body_text), mime),+ None => (parse_html(&body_text), document_content_type_or_html(response.content_type())),+ };+ self.content_type = document_content_type;self.title = dom.query_selector("title")@@ -5684,6 +5699,145 @@ mod tests {);}+ // #1231: a text/plain or text/markdown response is a plain-text document.+ // Parsing it as HTML turned every <word> in the body into an element and+ // dropped its text, and document.contentType reported text/html.+ #[tokio::test(flavor = "current_thread")]+ async fn plain_text_response_builds_a_pre_document() {+ let body = "RFC 9110 <URI-reference>: see <https://example.com/>\n2nd line";+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};+ let (mut stream, _) = listener.accept().unwrap();+ let mut request = [0u8; 2048];+ let _ = stream.read(&mut request).unwrap();+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len(),+ );+ stream.write_all(response.as_bytes()).unwrap();+ });++ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(+ "plain-text".to_string(),+ None,+ false,+ None,+ None,+ true,+ ));+ let mut page = super::Page::new("plain-text".to_string(), context);+ page.navigate(&format!("http://{address}/rfc9110.txt")).await.unwrap();++ let observed = page+ .js+ .as_mut()+ .unwrap()+ .evaluate(+ "(function(){return [document.contentType, \+ document.body.firstElementChild.tagName, \+ document.body.childNodes.length, \+ document.body.textContent];})()",+ )+ .unwrap();+ assert_eq!(+ observed,+ serde_json::json!(["text/plain", "PRE", 1, body])+ );+ }++ #[tokio::test(flavor = "current_thread")]+ async fn text_markdown_response_keeps_markup_as_text() {+ let body = "# Title\n\nSee <Component prop={1} /> and 3 < 4.";+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};+ let (mut stream, _) = listener.accept().unwrap();+ let mut request = [0u8; 2048];+ let _ = stream.read(&mut request).unwrap();+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: text/markdown\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len(),+ );+ stream.write_all(response.as_bytes()).unwrap();+ });++ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(+ "markdown-text".to_string(),+ None,+ false,+ None,+ None,+ true,+ ));+ let mut page = super::Page::new("markdown-text".to_string(), context);+ page.navigate(&format!("http://{address}/settings-reference")).await.unwrap();++ let observed = page+ .js+ .as_mut()+ .unwrap()+ .evaluate(+ "(function(){return [document.contentType, \+ document.body.textContent, \+ document.body.querySelectorAll('component').length];})()",+ )+ .unwrap();+ assert_eq!(+ observed,+ serde_json::json!(["text/markdown", body, 0])+ );+ }++ // The HTML parse path is unchanged, including the URL-derived contentType+ // sniffing for documents served without a usable Content-Type.+ #[tokio::test(flavor = "current_thread")]+ async fn html_response_still_builds_an_html_document() {+ let body = "<!doctype html><title>page</title><p>hello <b>world</b></p>";+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};+ let (mut stream, _) = listener.accept().unwrap();+ let mut request = [0u8; 2048];+ let _ = stream.read(&mut request).unwrap();+ let response = format!(+ "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",+ body.len(),+ );+ stream.write_all(response.as_bytes()).unwrap();+ });++ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(+ "html-doc".to_string(),+ None,+ false,+ None,+ None,+ true,+ ));+ let mut page = super::Page::new("html-doc".to_string(), context);+ page.navigate(&format!("http://{address}/index")).await.unwrap();++ let observed = page+ .js+ .as_mut()+ .unwrap()+ .evaluate(+ "(function(){return [document.contentType, \+ document.body.firstElementChild.tagName, \+ document.body.querySelectorAll('b').length, \+ document.body.textContent];})()",+ )+ .unwrap();+ assert_eq!(+ observed,+ serde_json::json!(["text/html", "P", 1, "hello world"])+ );+ }+// #971: a tab switch suspends other tabs (drops their runtime). Console// messages logged before suspension must be preserved, not lost.#[tokio::test(flavor = "current_thread")]@@ -10212,6 +10366,68 @@ fn is_text_like_content_type(content_type: Option<&str>) -> bool {|| ct.ends_with("+xml")}+/// Lowercased MIME type of a Content-Type header, without parameters.+fn base_content_type(content_type: Option<&str>) -> Option<String> {+ let ct = content_type?.split(';').next().unwrap_or("").trim().to_ascii_lowercase();+ if ct.is_empty() { None } else { Some(ct) }+}++/// The MIME type `document.contentType` reports for a main resource. Missing+/// or empty Content-Type headers stay unset so `contentType` keeps falling+/// back to URL-derived sniffing.+fn document_content_type_or_html(content_type: Option<&str>) -> String {+ base_content_type(content_type).unwrap_or_else(|| "text/html".to_string())+}++/// A response whose body is text but not markup gets a plain-text document+/// rather than an HTML parse of its content (`text/plain`, `text/markdown`).+/// Returns the MIME type to report as `document.contentType`.+fn plain_text_document_type(content_type: Option<&str>) -> Option<String> {+ let ct = base_content_type(content_type)?;+ // XML and HTML families keep their existing parse path, including the+ // `.xhtml`/`.xml` URL sniffing that `contentType` still applies.+ if ct == "text/html"+ || ct == "application/xhtml+xml"+ || ct == "text/xml"+ || ct == "application/xml"+ || ct.ends_with("+xml")+ {+ return None;+ }+ matches!(ct.as_str(), "text/plain" | "text/markdown" | "text/x-markdown")+ .then_some(ct)+}++/// Escape text for insertion as HTML character data, then build the document+/// Chrome builds for a plain-text response: `<body>` holding one `<pre>` with+/// the whole text. Serializing through the HTML parser keeps one code path for+/// document construction instead of a second native tree builder.+fn parse_plain_text_document(text: &str) -> DomTree {+ let mut html = String::with_capacity(text.len() + 64);+ html.push_str("<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title></title></head><body><pre>");+ html.push_str(&escape_html_text(text));+ // A newline straight after <pre> is dropped by the HTML parser; an extra+ // one keeps text that itself starts with a newline.+ if text.starts_with('\n') {+ html.push('\n');+ }+ html.push_str("</pre></body></html>");+ parse_html(&html)+}++fn escape_html_text(text: &str) -> String {+ let mut out = String::with_capacity(text.len());+ for ch in text.chars() {+ match ch {+ '&' => out.push_str("&"),+ '<' => out.push_str("<"),+ '>' => out.push_str(">"),+ _ => out.push(ch),+ }+ }+ out+}+fn response_body_entry_limit() -> usize {std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES").ok()diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.jsindex be457daf..b798f0f3 100644--- a/crates/obscura-js/js/bootstrap.js+++ b/crates/obscura-js/js/bootstrap.js@@ -42,7 +42,7 @@ const __obscuraCore = globalThis.Deno.core;// internal helpers (var-declared throughout the file)'__processDynScriptQueue', '_decodeDataScriptUrl', '_markNative', '_fpRand', '_fpNoise','_fpCache', '_getFp', '_fp', '_splitAsciiWhitespace',- '_getElementsByClassName', '_docEncoding', '_docIsUtf8',+ '_getElementsByClassName', '_docEncoding', '_docIsUtf8', '_docContentType','_isSpecialScheme', '_applyDocQueryEncoding', '_anchorBase','_elemHrefURL', '_setElemHrefPart', '_pad', '_daysInMonth','_isoWeek1Monday', '_inputParseNumber', '_inputFormatNumber',@@ -2919,6 +2919,18 @@ function _docEncoding() {return __docEncoding;}function _docIsUtf8() { if (__docIsUtf8 === undefined) _docEncoding(); return __docIsUtf8; }+// The main resource's MIME type. Cached per runtime for the same reason as+// the encoding: it is fixed for a document's lifetime and contentType is read+// on every _isXMLDocument() call. Empty when the navigation had no usable+// Content-Type header, so the caller can fall back to URL-derived sniffing.+let __docContentType;+function _docContentType() {+ if (__docContentType === undefined) {+ const t = _domParse("document_content_type");+ __docContentType = (typeof t === 'string') ? t : '';+ }+ return __docContentType;+}// WHATWG "special scheme" check (these get the special-query percent-encode set).function _isSpecialScheme(protocol) {const s = (protocol || '').replace(/:$/, '').toLowerCase();@@ -5839,6 +5851,11 @@ class Document extends Node {// XML document, so createCDATASection/etc. must not throw. Live documents// wrapped from the tree carry a real nid and fall through to URL-derived.if (this._nid === undefined || this._nid === null) return "application/xml";+ // The main resource's own MIME type wins over the URL: a text/plain or+ // text/markdown response must not report text/html just because its URL+ // has no extension.+ const main = _docContentType();+ if (main) return main;const url = this.URL || "";// data: URLs carry their MIME type explicitly.const dm = /^data:([^,;]+)/i.exec(url);@@ -13374,10 +13391,13 @@ globalThis.Comment = Comment;globalThis.CDATASection = CDATASection;globalThis.ProcessingInstruction = ProcessingInstruction;// True when the document was loaded from an XML/XHTML source. Obscura has no-// native XML tree, so this is inferred from contentType (derived from the URL).+// native XML tree, so this is inferred from contentType. Only the XML types+// count: a plain-text document (text/plain, text/markdown) is still an HTML+// document for createCDATASection/createProcessingInstruction purposes.function _isXMLDocument(doc) {const ct = (doc && doc.contentType) || "text/html";- return ct !== "text/html";+ return ct === "application/xhtml+xml" || ct === "text/xml"+ || ct === "application/xml" || ct.endsWith("+xml");}// XML Name production, sufficient for createProcessingInstruction targets.const _piNameStart = "A-Za-z_:\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD";diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex d1863686..4b44ee7e 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -159,6 +159,10 @@ pub struct ObscuraState {/// "UTF-8", "EUC-JP"). Backs `document.characterSet` and the URL query/// encoding override for `<a>`/`<area>` hrefs in legacy-charset documents.pub encoding: String,+ /// MIME type of the document's main resource, lowercased and without+ /// parameters. Backs `document.contentType`; a `text/plain` navigation+ /// must not report `text/html` just because the URL has no extension.+ pub content_type: String,pub title: String,/// URL of the document that initiated this document's navigation. Direct/// browser/API navigations leave this empty; document-initiated@@ -423,6 +427,7 @@ impl ObscuraState {about_base_url: None,inherited_origin: None,encoding: "UTF-8".to_string(),+ content_type: "text/html".to_string(),title: String::new(),referrer: String::new(),navigation_timing: NavigationTiming::default(),@@ -2174,6 +2179,9 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->}"document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),"document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),+ "document_content_type" => {+ serde_json::to_string(&gs.content_type).unwrap_or("\"text/html\"".into())+ }"document_element" => {for cid in dom.children(dom.document()) {if let Some(n) = dom.get_node(cid) {diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rsindex 462153a3..3370a85d 100644--- a/crates/obscura-js/src/runtime.rs+++ b/crates/obscura-js/src/runtime.rs@@ -1260,6 +1260,13 @@ impl ObscuraJsRuntime {self.state.borrow_mut().encoding = encoding.to_string();}+ /// Set the document's MIME type. Backs `document.contentType`, which+ /// reports the type of the main resource rather than one guessed from+ /// the URL.+ pub fn set_content_type(&self, content_type: &str) {+ self.state.borrow_mut().content_type = content_type.to_string();+ }+pub fn set_title(&self, title: &str) {self.state.borrow_mut().title = title.to_string();}