Skip to content

The headless browser for AI agents and web scraping

fix(browser): build a plain-text document for non-HTML text responses

Yi-111-a22eb6b42388e
diff --git a/crates/obscura-browser/src/page.rs b/crates/obscura-browser/src/page.rs
index fbdc129b..d85af845 100644
--- a/crates/obscura-browser/src/page.rs
+++ b/crates/obscura-browser/src/page.rs
@@ -279,6 +279,9 @@ pub struct Page {
/// Exposed to JS as `document.characterSet` and used for the URL query
/// encoding override on `<a>`/`<area>` hrefs in legacy-charset documents.
pub encoding: String,
+ /// MIME type of the current document's main resource, lowercased and
+ /// without parameters. Exposed to JS as `document.contentType`.
+ pub content_type: String,
/// Monotonic origin for the current document's CSS animation timeline.
/// It is reset once author styles are installed, so stylesheet download
/// latency does not incorrectly advance newly-created animations.
@@ -1113,6 +1116,7 @@ impl Page {
device_scale_factor: 1.0,
default_background_color_override: None,
encoding: "UTF-8".to_string(),
+ content_type: "text/html".to_string(),
document_timeline_origin: std::time::Instant::now(),
navigation_timing: NavigationTiming::default(),
navigation_timeout: None,
@@ -1778,6 +1782,7 @@ impl Page {
);
rt.set_url(&self.url_string());
rt.set_encoding(&self.encoding);
+ rt.set_content_type(&self.content_type);
rt.set_title(&self.title);
rt.set_referrer(&self.referrer);
let (session_history, session_index) = self.predicted_session_history();
@@ -3499,7 +3504,17 @@ impl Page {
obscura_net::decode_response_with_name(&response.body, response.content_type());
self.encoding = encoding_name.to_string();
self.navigation_timing.record("domLoading");
- let dom = parse_html(&body_text);
+ // A non-HTML text response is a plain-text document, not an HTML one
+ // that happens to hold text: every `<word>` in a .txt or .md body
+ // would otherwise be parsed as an element and its contents dropped
+ // from textContent (#1231). Chrome wraps the decoded text in a
+ // single <pre> instead.
+ let (dom, document_content_type) = match plain_text_document_type(response.content_type())
+ {
+ Some(mime) => (parse_plain_text_document(&body_text), mime),
+ None => (parse_html(&body_text), document_content_type_or_html(response.content_type())),
+ };
+ self.content_type = document_content_type;
self.title = dom
.query_selector("title")
@@ -5684,6 +5699,145 @@ mod tests {
);
}
+ // #1231: a text/plain or text/markdown response is a plain-text document.
+ // Parsing it as HTML turned every <word> in the body into an element and
+ // dropped its text, and document.contentType reported text/html.
+ #[tokio::test(flavor = "current_thread")]
+ async fn plain_text_response_builds_a_pre_document() {
+ let body = "RFC 9110 <URI-reference>: see <https://example.com/>\n2nd line";
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut request = [0u8; 2048];
+ let _ = stream.read(&mut request).unwrap();
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: text/plain; charset=utf-8\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len(),
+ );
+ stream.write_all(response.as_bytes()).unwrap();
+ });
+
+ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
+ "plain-text".to_string(),
+ None,
+ false,
+ None,
+ None,
+ true,
+ ));
+ let mut page = super::Page::new("plain-text".to_string(), context);
+ page.navigate(&format!("http://{address}/rfc9110.txt")).await.unwrap();
+
+ let observed = page
+ .js
+ .as_mut()
+ .unwrap()
+ .evaluate(
+ "(function(){return [document.contentType, \
+ document.body.firstElementChild.tagName, \
+ document.body.childNodes.length, \
+ document.body.textContent];})()",
+ )
+ .unwrap();
+ assert_eq!(
+ observed,
+ serde_json::json!(["text/plain", "PRE", 1, body])
+ );
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn text_markdown_response_keeps_markup_as_text() {
+ let body = "# Title\n\nSee <Component prop={1} /> and 3 < 4.";
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut request = [0u8; 2048];
+ let _ = stream.read(&mut request).unwrap();
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: text/markdown\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len(),
+ );
+ stream.write_all(response.as_bytes()).unwrap();
+ });
+
+ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
+ "markdown-text".to_string(),
+ None,
+ false,
+ None,
+ None,
+ true,
+ ));
+ let mut page = super::Page::new("markdown-text".to_string(), context);
+ page.navigate(&format!("http://{address}/settings-reference")).await.unwrap();
+
+ let observed = page
+ .js
+ .as_mut()
+ .unwrap()
+ .evaluate(
+ "(function(){return [document.contentType, \
+ document.body.textContent, \
+ document.body.querySelectorAll('component').length];})()",
+ )
+ .unwrap();
+ assert_eq!(
+ observed,
+ serde_json::json!(["text/markdown", body, 0])
+ );
+ }
+
+ // The HTML parse path is unchanged, including the URL-derived contentType
+ // sniffing for documents served without a usable Content-Type.
+ #[tokio::test(flavor = "current_thread")]
+ async fn html_response_still_builds_an_html_document() {
+ let body = "<!doctype html><title>page</title><p>hello <b>world</b></p>";
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut request = [0u8; 2048];
+ let _ = stream.read(&mut request).unwrap();
+ let response = format!(
+ "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}",
+ body.len(),
+ );
+ stream.write_all(response.as_bytes()).unwrap();
+ });
+
+ let context = std::sync::Arc::new(crate::BrowserContext::with_storage_and_network(
+ "html-doc".to_string(),
+ None,
+ false,
+ None,
+ None,
+ true,
+ ));
+ let mut page = super::Page::new("html-doc".to_string(), context);
+ page.navigate(&format!("http://{address}/index")).await.unwrap();
+
+ let observed = page
+ .js
+ .as_mut()
+ .unwrap()
+ .evaluate(
+ "(function(){return [document.contentType, \
+ document.body.firstElementChild.tagName, \
+ document.body.querySelectorAll('b').length, \
+ document.body.textContent];})()",
+ )
+ .unwrap();
+ assert_eq!(
+ observed,
+ serde_json::json!(["text/html", "P", 1, "hello world"])
+ );
+ }
+
// #971: a tab switch suspends other tabs (drops their runtime). Console
// messages logged before suspension must be preserved, not lost.
#[tokio::test(flavor = "current_thread")]
@@ -10212,6 +10366,68 @@ fn is_text_like_content_type(content_type: Option<&str>) -> bool {
|| ct.ends_with("+xml")
}
+/// Lowercased MIME type of a Content-Type header, without parameters.
+fn base_content_type(content_type: Option<&str>) -> Option<String> {
+ let ct = content_type?.split(';').next().unwrap_or("").trim().to_ascii_lowercase();
+ if ct.is_empty() { None } else { Some(ct) }
+}
+
+/// The MIME type `document.contentType` reports for a main resource. Missing
+/// or empty Content-Type headers stay unset so `contentType` keeps falling
+/// back to URL-derived sniffing.
+fn document_content_type_or_html(content_type: Option<&str>) -> String {
+ base_content_type(content_type).unwrap_or_else(|| "text/html".to_string())
+}
+
+/// A response whose body is text but not markup gets a plain-text document
+/// rather than an HTML parse of its content (`text/plain`, `text/markdown`).
+/// Returns the MIME type to report as `document.contentType`.
+fn plain_text_document_type(content_type: Option<&str>) -> Option<String> {
+ let ct = base_content_type(content_type)?;
+ // XML and HTML families keep their existing parse path, including the
+ // `.xhtml`/`.xml` URL sniffing that `contentType` still applies.
+ if ct == "text/html"
+ || ct == "application/xhtml+xml"
+ || ct == "text/xml"
+ || ct == "application/xml"
+ || ct.ends_with("+xml")
+ {
+ return None;
+ }
+ matches!(ct.as_str(), "text/plain" | "text/markdown" | "text/x-markdown")
+ .then_some(ct)
+}
+
+/// Escape text for insertion as HTML character data, then build the document
+/// Chrome builds for a plain-text response: `<body>` holding one `<pre>` with
+/// the whole text. Serializing through the HTML parser keeps one code path for
+/// document construction instead of a second native tree builder.
+fn parse_plain_text_document(text: &str) -> DomTree {
+ let mut html = String::with_capacity(text.len() + 64);
+ html.push_str("<!DOCTYPE html><html><head><meta charset=\"utf-8\"><title></title></head><body><pre>");
+ html.push_str(&escape_html_text(text));
+ // A newline straight after <pre> is dropped by the HTML parser; an extra
+ // one keeps text that itself starts with a newline.
+ if text.starts_with('\n') {
+ html.push('\n');
+ }
+ html.push_str("</pre></body></html>");
+ parse_html(&html)
+}
+
+fn escape_html_text(text: &str) -> String {
+ let mut out = String::with_capacity(text.len());
+ for ch in text.chars() {
+ match ch {
+ '&' => out.push_str("&amp;"),
+ '<' => out.push_str("&lt;"),
+ '>' => out.push_str("&gt;"),
+ _ => out.push(ch),
+ }
+ }
+ out
+}
+
fn response_body_entry_limit() -> usize {
std::env::var("OBSCURA_NETWORK_BODY_BUFFER_ENTRIES")
.ok()
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index be457daf..b798f0f3 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -42,7 +42,7 @@ const __obscuraCore = globalThis.Deno.core;
// internal helpers (var-declared throughout the file)
'__processDynScriptQueue', '_decodeDataScriptUrl', '_markNative', '_fpRand', '_fpNoise',
'_fpCache', '_getFp', '_fp', '_splitAsciiWhitespace',
- '_getElementsByClassName', '_docEncoding', '_docIsUtf8',
+ '_getElementsByClassName', '_docEncoding', '_docIsUtf8', '_docContentType',
'_isSpecialScheme', '_applyDocQueryEncoding', '_anchorBase',
'_elemHrefURL', '_setElemHrefPart', '_pad', '_daysInMonth',
'_isoWeek1Monday', '_inputParseNumber', '_inputFormatNumber',
@@ -2919,6 +2919,18 @@ function _docEncoding() {
return __docEncoding;
}
function _docIsUtf8() { if (__docIsUtf8 === undefined) _docEncoding(); return __docIsUtf8; }
+// The main resource's MIME type. Cached per runtime for the same reason as
+// the encoding: it is fixed for a document's lifetime and contentType is read
+// on every _isXMLDocument() call. Empty when the navigation had no usable
+// Content-Type header, so the caller can fall back to URL-derived sniffing.
+let __docContentType;
+function _docContentType() {
+ if (__docContentType === undefined) {
+ const t = _domParse("document_content_type");
+ __docContentType = (typeof t === 'string') ? t : '';
+ }
+ return __docContentType;
+}
// WHATWG "special scheme" check (these get the special-query percent-encode set).
function _isSpecialScheme(protocol) {
const s = (protocol || '').replace(/:$/, '').toLowerCase();
@@ -5839,6 +5851,11 @@ class Document extends Node {
// XML document, so createCDATASection/etc. must not throw. Live documents
// wrapped from the tree carry a real nid and fall through to URL-derived.
if (this._nid === undefined || this._nid === null) return "application/xml";
+ // The main resource's own MIME type wins over the URL: a text/plain or
+ // text/markdown response must not report text/html just because its URL
+ // has no extension.
+ const main = _docContentType();
+ if (main) return main;
const url = this.URL || "";
// data: URLs carry their MIME type explicitly.
const dm = /^data:([^,;]+)/i.exec(url);
@@ -13374,10 +13391,13 @@ globalThis.Comment = Comment;
globalThis.CDATASection = CDATASection;
globalThis.ProcessingInstruction = ProcessingInstruction;
// True when the document was loaded from an XML/XHTML source. Obscura has no
-// native XML tree, so this is inferred from contentType (derived from the URL).
+// native XML tree, so this is inferred from contentType. Only the XML types
+// count: a plain-text document (text/plain, text/markdown) is still an HTML
+// document for createCDATASection/createProcessingInstruction purposes.
function _isXMLDocument(doc) {
const ct = (doc && doc.contentType) || "text/html";
- return ct !== "text/html";
+ return ct === "application/xhtml+xml" || ct === "text/xml"
+ || ct === "application/xml" || ct.endsWith("+xml");
}
// XML Name production, sufficient for createProcessingInstruction targets.
const _piNameStart = "A-Za-z_:\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02FF\\u0370-\\u037D\\u037F-\\u1FFF\\u200C-\\u200D\\u2070-\\u218F\\u2C00-\\u2FEF\\u3001-\\uD7FF\\uF900-\\uFDCF\\uFDF0-\\uFFFD";
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index d1863686..4b44ee7e 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -159,6 +159,10 @@ pub struct ObscuraState {
/// "UTF-8", "EUC-JP"). Backs `document.characterSet` and the URL query
/// encoding override for `<a>`/`<area>` hrefs in legacy-charset documents.
pub encoding: String,
+ /// MIME type of the document's main resource, lowercased and without
+ /// parameters. Backs `document.contentType`; a `text/plain` navigation
+ /// must not report `text/html` just because the URL has no extension.
+ pub content_type: String,
pub title: String,
/// URL of the document that initiated this document's navigation. Direct
/// browser/API navigations leave this empty; document-initiated
@@ -423,6 +427,7 @@ impl ObscuraState {
about_base_url: None,
inherited_origin: None,
encoding: "UTF-8".to_string(),
+ content_type: "text/html".to_string(),
title: String::new(),
referrer: String::new(),
navigation_timing: NavigationTiming::default(),
@@ -2174,6 +2179,9 @@ fn op_dom_inner(shared: SharedState, cmd: String, arg1: String, arg2: String) ->
}
"document_referrer" => serde_json::to_string(&gs.referrer).unwrap_or("\"\"".into()),
"document_encoding" => serde_json::to_string(&gs.encoding).unwrap_or("\"UTF-8\"".into()),
+ "document_content_type" => {
+ serde_json::to_string(&gs.content_type).unwrap_or("\"text/html\"".into())
+ }
"document_element" => {
for cid in dom.children(dom.document()) {
if let Some(n) = dom.get_node(cid) {
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index 462153a3..3370a85d 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -1260,6 +1260,13 @@ impl ObscuraJsRuntime {
self.state.borrow_mut().encoding = encoding.to_string();
}
+ /// Set the document's MIME type. Backs `document.contentType`, which
+ /// reports the type of the main resource rather than one guessed from
+ /// the URL.
+ pub fn set_content_type(&self, content_type: &str) {
+ self.state.borrow_mut().content_type = content_type.to_string();
+ }
+
pub fn set_title(&self, title: &str) {
self.state.borrow_mut().title = title.to_string();
}