Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(document): parse text responses with consistent decoding (#740)
mini llkcba79021aa4b
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex 88df60445..3c14f5109 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder {url_hint: Option<String>,decoder: Option<Decoder>,selected_encoding: Option<&'static Encoding>,+ sniff_html_declarations: bool,}impl HtmlDocumentStreamingDecoder {+ /// Decode a text document without interpreting literal HTML/XML declarations.+ /// JSON defaults to UTF-8; other text retains the configured legacy detector.+ pub fn new_text_document(+ headers: &[(String, String)],+ url_hint: &str,+ detector: LegacyEncodingDetector,+ json: bool,+ ) -> Self {+ let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));+ decoder.sniff_html_declarations = false;+ if json && decoder.transport_encoding.is_none() {+ decoder.transport_encoding = Some(encoding_rs::UTF_8);+ }+ decoder+ }+pub fn new(headers: &[(String, String)]) -> Self {Self::new_with_options(headers, None, None, None)}@@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder {url_hint: url_hint.map(str::to_owned),decoder: None,selected_encoding: None,+ sniff_html_declarations: true,}}@@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder {pub fn document_encoding_name(&self) -> &'static str {self.selected_encoding_name()+ .or_else(|| self.transport_encoding.map(Encoding::name)).unwrap_or(self.fallback_encoding.name())}@@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder {if let Some(encoding) = self.transport_encoding {return Some(encoding);}+ if !self.sniff_html_declarations {+ return (finishing || self.sniff_buffer.len() >= 1024).then(|| {+ self.detected_legacy_content_encoding()+ .unwrap_or(self.fallback_encoding)+ });+ }if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {return Some(encoding);}diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 2b4863a84..d9717f32d 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector(panic!("declared encoding must take precedence over heuristic detection")}+#[test]+fn json_text_document_defaults_to_utf8_across_every_chunk_split() {+ let input = "{\"name\":\"Gülçek\"}";+ for split in 0..=input.len() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://example.test/",+ unexpected_legacy_encoding_detector,+ true,+ );+ let mut output = decoder.push(&input.as_bytes()[..split]).join("");+ output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, input, "split {split}");+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));+ }+}++#[test]+fn plain_text_document_honors_transport_charset() {+ let headers = [(+ "Content-Type".to_owned(),+ "text/plain; charset=gbk".to_owned(),+ )];+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &headers,+ "https://example.test/",+ unexpected_legacy_encoding_detector,+ false,+ );+ let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, "<b>家居</b>");+ assert_eq!(decoder.selected_encoding_name(), Some("GBK"));+}++#[test]+fn text_document_bom_precedes_transport_charset() {+ let headers = [(+ "Content-Type".to_owned(),+ "text/plain; charset=gbk".to_owned(),+ )];+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &headers,+ "https://example.test/",+ unexpected_legacy_encoding_detector,+ false,+ );+ assert_eq!(decoder.document_encoding_name(), "GBK");+ assert_eq!(decoder.selected_encoding_name(), None);+ assert!(decoder.push(&[0xef]).is_empty());+ assert!(decoder.push(&[0xbb]).is_empty());+ let mut tail = vec![0xbf];+ tail.extend_from_slice("Gülçek".as_bytes());+ let mut output = decoder.push(&tail).join("");+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, "Gülçek");+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));+ assert_eq!(decoder.document_encoding_name(), "UTF-8");+}++#[test]+fn plain_text_document_retains_legacy_detection_without_a_charset() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://legacy.example/",+ test_legacy_encoding_detector,+ false,+ );+ let mut output = decoder.push(&[0x80]).join("");+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, "А");+ assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));+}++#[test]+fn plain_text_document_ignores_literal_encoding_declarations() {+ for prefix in [+ "<meta charset=utf-8>",+ "<?xml version='1.0' encoding='utf-8'?>",+ ] {+ let mut input = prefix.as_bytes().to_vec();+ input.push(0xe9);+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://example.test/",+ test_legacy_encoding_detector,+ false,+ );+ let mut output = decoder.push(&input).join("");+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, format!("{prefix}é"));+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));+ }+}+#[test]fn content_type_charset_is_selected() {let headers = vec![(diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rsindex ce351cf5e..17369a9b5 100644--- a/moli-parser/src/html.rs+++ b/moli-parser/src/html.rs@@ -27,7 +27,7 @@ use super::live_target::{ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer,ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks,ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream,- new_parser_stream_html_tree_sink_stream,+ new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target,};use super::{ParserSourcePosition, html_chunks,@@ -438,6 +438,47 @@ impl HtmlParser {DocumentStream::new_parser_stream(final_url, self.scripting_enabled)}+ /// Start a text document using the HTML tokenizer's plaintext state.+ /// Subsequent chunks remain literal text inside the browser-owned `pre`.+ pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream {+ let stream =+ DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url));+ stream.inner.initialize_text_document();+ let mut host = stream.inner.take_parser_stream_dom_host();+ let document_handle = host.document_handle();+ host.set_document_content_type_for_handle(document_handle, content_type);+ stream.inner.restore_parser_stream_dom_host(host);+ stream+ }++ /// Initialize the same literal-text shell in an existing document. The+ /// embedder owns that document's MIME metadata and live DOM callbacks.+ pub fn start_live_text_document_root<T>(+ &self,+ final_url: Url,+ document_handle: NativeNodeId,+ consumer: &mut T,+ ) -> DocumentStream+ where+ T: ParserDomReadConsumer+ + ParserDomMutationConsumer+ + ParserMutationEffectConsumer+ + ParserElementCreationConsumer,+ {+ let stream = DocumentStream::new_text_document(+ ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle),+ );+ // SAFETY: the consumer is exclusively borrowed until the step guard+ // removes the erased callbacks, including when initialization unwinds.+ let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) };+ stream.inner.enter_runtime_dom_sinks_parse_step(sinks);+ {+ let step = RuntimeDomSinksParserStep { stream: &stream };+ step.stream.inner.initialize_text_document();+ }+ stream+ }+pub fn start_live_document_root(&self,final_url: Url,@@ -551,6 +592,17 @@ impl Default for ParserInputQueue {}impl DocumentStream {+ fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self {+ let mut options = html_parse_opts_with_scripting(false);+ // The byte decoder has already consumed the transport BOM. In plaintext+ // every remaining U+FEFF is content, including at a chunk boundary.+ options.tokenizer.discard_bom = false;+ Self {+ inner: HtmlTreeSinkStream::from_target_with_options(target, options),+ input: RefCell::default(),+ }+ }+fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self {Self {inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled),@@ -1843,6 +1895,83 @@ mod tests {const HTML_NS: &str = "http://www.w3.org/1999/xhtml";const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML";+ #[test]+ fn text_document_stream_preserves_literal_markup_and_leading_newline() {+ let payload =+ "\u{feff}\n<b>Gülçek</b>&<script>window.executed=1</script></pre>\u{feff}";+ for content_type in ["text/plain", "application/json", "application/problem+json"] {+ let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document(+ Url::parse("https://example.test/data").unwrap(),+ content_type,+ );+ for character in payload.chars() {+ stream.feed(&character.to_string());+ }+ let document = stream.finish();+ let pre = first_element_by_ns(&document, HTML_NS, "pre");+ assert_eq!(document.text_content(pre).as_deref(), Some(payload));+ assert_eq!(document.document().unwrap().content_type(), content_type);+ for tag in ["b", "script"] {+ assert!(+ document+ .elements_by_tag_name_ns(+ document.document_node_id(),+ Some(HTML_NS),+ tag,+ true+ )+ .is_empty()+ );+ }+ }+ }++ #[test]+ fn text_document_shell_has_no_doctype_and_preserves_normalized_literal_text() {+ for mime in [+ "text/plain",+ "application/json",+ "application/problem+json",+ "text/javascript",+ ] {+ let stream = HtmlParser::SCRIPTING_ENABLED+ .start_text_document(Url::parse("https://example.test/data").unwrap(), mime);+ for chunk in [+ "\n<&",+ "\r",+ "\nbeta\rgamma\0",+ "</pre><script>bad()</script>",+ ] {+ stream.feed(chunk);+ }+ let document = stream.finish_dom_host();+ let root = document.document_handle();+ let children = document.child_handles(root).collect::<Vec<_>>();+ assert_eq!(children.len(), 1, "no synthetic doctype: {mime}");+ assert!(document.is_html_element_named(children[0], "html"));+ let expected_mode = HtmlParser::SCRIPTING_ENABLED.parse_dom_host(+ Url::parse("https://example.test/control").unwrap(),+ "<!doctype html><p>control".to_owned(),+ );+ assert_eq!(+ document.document_quirks_mode_for_handle(root),+ expected_mode.document_quirks_mode_for_handle(expected_mode.document_handle())+ );+ assert_eq!(+ document.text_content(root).as_deref(),+ Some("\n<&\nbeta\ngamma\u{fffd}</pre><script>bad()</script>")+ );+ }+ }++ #[test]+ fn html_document_still_parses_markup_and_entities() {+ let document = parse_test_document("<b>Gülçek&</b>");+ let bold = first_element_by_ns(&document, HTML_NS, "b");+ assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&"));+ assert_eq!(document.document().unwrap().content_type(), "text/html");+ }+fn parse_test_document(html: &str) -> NativeDom {HtmlParser::SCRIPTING_ENABLED.parse(Url::parse("https://example.test/").expect("test url"),diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rsindex 3025906a4..2b4dfef9a 100644--- a/moli-parser/src/session.rs+++ b/moli-parser/src/session.rs@@ -167,6 +167,19 @@ impl TokenSink for EmbedderPausingTreeBuilder {}impl HtmlParserSession {+ pub(super) fn initialize_text_document(&self) {+ // Parse only the browser-owned shell. Response bytes enter the tokenizer+ // after it has switched to plaintext, so tags and entities stay literal.+ self.process(StrTendril::from(concat!(+ "<html><head></head><body>",+ "<pre style=\"word-wrap: break-word; white-space: pre-wrap;\">\n"+ )));+ // Text documents have no doctype but always use no-quirks mode.+ self.sink()+ .set_quirks_mode(html5ever::tree_builder::QuirksMode::NoQuirks);+ self.tokenizer.set_plaintext_state();+ }+fn new(sink: DocumentSink, opts: ParseOpts) -> Self {let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);Self {@@ -352,10 +365,10 @@ fn feed_with_definitive_encoding(pub(super) fn new_html_tree_sink_session(target: ParserStreamHtmlTreeSinkTarget,- scripting_enabled: bool,+ options: ParseOpts,) -> HtmlTreeSinkSession {let sink = DocumentSink::new(target);- let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));+ let parser = HtmlParserSession::new(sink, options);let script_input = ParserInputQueue::default();HtmlTreeSinkSession {diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rsindex fa14f3b82..70d3a31d3 100644--- a/moli-parser/src/stream.rs+++ b/moli-parser/src/stream.rs@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(}impl HtmlTreeSinkStream {+ pub(super) fn initialize_text_document(&self) {+ self.parser.initialize_text_document();+ }+pub(super) fn from_target_with_scripting(target: ParserStreamHtmlTreeSinkTarget,scripting_enabled: bool,) -> Self {- let session = new_html_tree_sink_session(target, scripting_enabled);+ Self::from_target_with_options(+ target,+ super::session::html_parse_opts_with_scripting(scripting_enabled),+ )+ }++ pub(super) fn from_target_with_options(+ target: ParserStreamHtmlTreeSinkTarget,+ options: html5ever::ParseOpts,+ ) -> Self {+ let session = new_html_tree_sink_session(target, options);Self {parser: session.parser,script_input: session.script_input,diff --git a/moli-protocol-server/src/protocol_server/tests/classic.rs b/moli-protocol-server/src/protocol_server/tests/classic.rsindex 69beb69ef..68d1c92ac 100644--- a/moli-protocol-server/src/protocol_server/tests/classic.rs+++ b/moli-protocol-server/src/protocol_server/tests/classic.rs@@ -3,6 +3,129 @@ use axum::http::{HeaderMap, header};use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;use serde_json::{Map, Value};+#[tokio::test]+async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {+ const PAYLOAD: &str =+ "<meta charset='gbk'><script>window.executed=42;</script><b>literal&Gülçek</b>";+ let cases = [+ (+ "plain",+ "text/plain; charset=utf-8",+ "text/plain",+ true,+ PAYLOAD,+ ),+ (+ "plain.xml",+ "text/plain; charset=utf-8",+ "text/plain",+ true,+ "\n<&\r\nbeta\rgamma\0</pre><script>window.executed=42</script>",+ ),+ (+ "json.xml",+ "application/problem+json",+ "application/problem+json",+ true,+ PAYLOAD,+ ),+ (+ "plain-bom",+ "text/plain; charset=utf-8",+ "text/plain",+ true,+ "\u{feff}\u{feff}",+ ),+ (+ "json",+ "application/json",+ "application/json",+ true,+ PAYLOAD,+ ),+ (+ "javascript",+ "text/javascript; charset=utf-8",+ "text/javascript",+ true,+ PAYLOAD,+ ),+ (+ "html",+ "text/html; charset=utf-8",+ "text/html",+ false,+ PAYLOAD,+ ),+ ];+ let mut fixture = axum::Router::new();+ for (name, mime, _, _, payload) in cases {+ fixture = fixture.route(+ &format!("/{name}"),+ axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),+ );+ }+ fixture = fixture.route(+ "/frame/{kind}",+ axum::routing::get(+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {+ (+ [(header::CONTENT_TYPE, "text/html")],+ format!("<iframe src='/{kind}'></iframe>"),+ )+ },+ ),+ );+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");+ let app = build_router(test_state());+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;+ let session_id = session["value"]["sessionId"].as_str().unwrap();+ for (name, _, content_type, literal, payload) in cases {+ for prefix in ["", "frame/"] {+ let navigated = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/url"),+ json!({"url":format!("http://{addr}/{prefix}{name}")}),+ )+ .await;+ assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");+ let observed = classic_request_json_with_body(+ app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),+ json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),+ ).await;+ let expected = if literal {+ json!([+ payload+ .strip_prefix('\u{feff}')+ .unwrap_or(payload)+ .replace("\r\n", "\n")+ .replace('\r', "\n")+ .replace('\0', "\u{fffd}"),+ null,+ 0,+ content_type+ ])+ } else {+ json!(["literal&Gülçek", 42, 1, content_type])+ };+ assert_eq!(observed["value"], expected, "{prefix}{name}");+ if literal {+ let structure = classic_request_json_with_body(+ app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),+ json!({"script": "const d=(document.querySelector('iframe')?.contentWindow ?? window).document; return [d.doctype===null,d.compatMode,d.body.children.length,d.body.firstElementChild.localName,d.querySelectorAll('pre').length];", "args": []}),+ ).await;+ assert_eq!(+ structure["value"],+ json!([true, "CSS1Compat", 1, "pre", 1]),+ "{prefix}{name}"+ );+ }+ }+ }+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;+}+#[tokio::test]async fn webdriver_classic_status_session_and_delete_routes_use_value_envelope() {let app = build_router(test_state());diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rsindex d8a83d17c..76ac1936a 100644--- a/moli-protocol/src/domains/emulation/tests.rs+++ b/moli-protocol/src/domains/emulation/tests.rs@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()listener,Router::new().route("/",- get(|| async { "<!doctype html><title>idle navigation</title>" }),+ get(|| async {+ axum::response::Html("<!doctype html><title>idle navigation</title>")+ }),),).await@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {listener,Router::new().route("/",- get(|| async { "<!doctype html><title>idle detector</title>" }),+ get(|| async {+ axum::response::Html("<!doctype html><title>idle detector</title>")+ }),),).await@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()#[tokio::test(flavor = "multi_thread")]async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {async fn page_handler() -> impl IntoResponse {- "<!doctype html><html><body>ok</body></html>"+ axum::response::Html("<!doctype html><html><body>ok</body></html>")}async fn xhr_handler(@@ -2711,9 +2715,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {.get(axum::http::header::ACCEPT_LANGUAGE).and_then(|value| value.to_str().ok()).unwrap_or("");- format!(+ axum::response::Html(format!("<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"- )+ ))}let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();@@ -2767,9 +2771,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa.get(axum::http::header::ACCEPT_LANGUAGE).and_then(|value| value.to_str().ok()).unwrap_or("");- format!(+ axum::response::Html(format!("<!doctype html><html><body data-user-agent=\"{user_agent}\" data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.userAgent, Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"- )+ ))}let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rsindex 450912689..ea7c311b9 100644--- a/moli-protocol/src/domains/page/tests/lifecycle.rs+++ b/moli-protocol/src/domains/page/tests/lifecycle.rs@@ -2391,11 +2391,15 @@ async fn network_navigations_use_unique_document_loader_ids() {axum::Router::new().route("/first",- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),+ axum::routing::get(|| async {+ axum::response::Html("<!doctype html><main>first</main>")+ }),).route("/second",- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),+ axum::routing::get(|| async {+ axum::response::Html("<!doctype html><main>second</main>")+ }),),).await@@ -2474,11 +2478,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() {axum::Router::new().route("/first",- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),+ axum::routing::get(|| async {+ axum::response::Html("<!doctype html><main>first</main>")+ }),).route("/second",- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),+ axum::routing::get(|| async {+ axum::response::Html("<!doctype html><main>second</main>")+ }),),).awaitdiff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rsindex ab651a209..1a1e48497 100644--- a/moli-protocol/src/domains/page/tests/runtime.rs+++ b/moli-protocol/src/domains/page/tests/runtime.rs@@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li.route("/main",axum::routing::get(|| async {- "<iframe id='child' name='child-frame' src='/initial'></iframe>"+ axum::response::Html(+ "<iframe id='child' name='child-frame' src='/initial'></iframe>",+ )}),).route("/initial",- axum::routing::get(|| async { "<body>initial</body>" }),+ axum::routing::get(|| async { axum::response::Html("<body>initial</body>") }),).route("/history-a",- axum::routing::get(|| async { "<body>history-a</body>" }),+ axum::routing::get(|| async { axum::response::Html("<body>history-a</body>") }),).route("/history-b",- axum::routing::get(|| async { "<body>history-b</body>" }),+ axum::routing::get(|| async { axum::response::Html("<body>history-b</body>") }),),).awaitdiff --git a/moli-protocol/src/domains/target/tests/tests_background_staging.rs b/moli-protocol/src/domains/target/tests/tests_background_staging.rsindex e52b86f54..49bebc255 100644--- a/moli-protocol/src/domains/target/tests/tests_background_staging.rs+++ b/moli-protocol/src/domains/target/tests/tests_background_staging.rs@@ -3037,9 +3037,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat.get(axum::http::header::ACCEPT_LANGUAGE).and_then(|value| value.to_str().ok()).unwrap_or("");- format!(+ axum::response::Html(format!("<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.language, document.body.dataset.acceptLanguage].join('|');</script></body></html>"- )+ ))}let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();@@ -3204,7 +3204,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat#[tokio::test(flavor = "multi_thread")]async fn same_context_background_session_can_clear_its_own_timezone_before_activation() {async fn handler() -> impl IntoResponse {- "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>"+ axum::response::Html(+ "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>",+ )}let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rsindex f1703887c..9c50732d2 100644--- a/moli-renderer-v8/src/dom_parser.rs+++ b/moli-renderer-v8/src/dom_parser.rs@@ -1,5 +1,4 @@use crate::web_api_interfaces;-use html5ever::tree_builder::QuirksMode;use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime};use moli_webapi_declare::WebApiFunctionTemplate;use url::Url;@@ -367,7 +366,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>(source: &'a str,content_type: Option<&str>,) -> std::borrow::Cow<'a, str> {- if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) {+ if source == "\u{feff}"+ && !content_type.is_some_and(|mime| {+ is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)+ })+ {std::borrow::Cow::Borrowed("<body>\u{feff}</body>")} else {std::borrow::Cow::Borrowed(source)@@ -381,7 +384,7 @@ fn parse_browsing_context_document_snapshot(html_parser: HtmlParser,) -> (DomHost, DetachedDocumentKind) {if content_type.is_some_and(is_dom_parser_xml_mime)- || child_document_url_is_xml_like(&document_url)+ || (content_type.is_none() && child_document_url_is_xml_like(&document_url)){let parser = XmlParser;return (@@ -389,13 +392,13 @@ fn parse_browsing_context_document_snapshot(DetachedDocumentKind::Xml,);}- if content_type.is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) {- let mut document =- html_parser.parse_dom_host(document_url, plain_text_document_parser_input(source));- // Text documents are HTML Documents whose mode is explicitly no-quirks,- // despite having no doctype that would select that mode through parsing.- document.set_html_quirks_mode_for_parser(QuirksMode::NoQuirks);- return (document, DetachedDocumentKind::Html);+ if let Some(content_type) =+ content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime))+ {+ let stream = HtmlParser::with_scripting_enabled(false)+ .start_text_document(document_url, content_type);+ stream.feed(source);+ return (stream.finish_dom_host(), DetachedDocumentKind::Html);}(html_parser.parse_dom_host(document_url, source.to_owned()),@@ -403,22 +406,6 @@ fn parse_browsing_context_document_snapshot()}-pub(crate) fn plain_text_document_parser_input(source: &str) -> String {- let mut input = String::with_capacity(source.len().saturating_add(64));- // The HTML parser discards the first LF after <pre>; preserve any source LF.- input.push_str("<html><head></head><body><pre>\n");- for character in source.chars() {- match character {- '&' => input.push_str("&"),- '<' => input.push_str("<"),- '\0' => input.push('\u{fffd}'),- _ => input.push(character),- }- }- input.push_str("</pre></body></html>");- input-}-fn child_document_url_is_xml_like(url: &Url) -> bool {let path = url.path().to_ascii_lowercase();path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg")@@ -599,6 +586,50 @@ mod tests {assert!(disabled.element_handle_by_id("fallback").is_some());}+ #[test]+ fn child_projection_explicit_mime_overrides_xml_extension() {+ let source = "\n<&</pre><script>window.executed=1</script>";+ for mime in [+ "text/plain",+ "application/json",+ "application/problem+json",+ "text/javascript",+ ] {+ let (document, kind) = parse_browsing_context_document_snapshot(+ Url::parse("https://example.test/data.xml").unwrap(),+ source,+ Some(mime),+ HtmlParser::SCRIPTING_ENABLED,+ );+ let root = document.document_handle();+ assert_eq!(kind, DetachedDocumentKind::Html);+ assert_eq!(+ document.text_content(root).as_deref(),+ Some(source),+ "{mime}"+ );+ let children = document.child_handles(root).collect::<Vec<_>>();+ assert_eq!(children.len(), 1);+ assert!(document.is_html_element_named(children[0], "html"));+ assert_eq!(+ document.document_quirks_mode_for_handle(root),+ Some(selectors::matching::QuirksMode::NoQuirks)+ );+ }+ let (document, kind) = parse_browsing_context_document_snapshot(+ Url::parse("https://example.test/page.svg").unwrap(),+ "<!doctype html><b id='parsed'>&</b>",+ Some("text/html"),+ HtmlParser::SCRIPTING_ENABLED,+ );+ assert_eq!(kind, DetachedDocumentKind::Html);+ assert!(document.element_handle_by_id("parsed").is_some());+ assert_eq!(+ document.text_content(document.document_handle()).as_deref(),+ Some("&")+ );+ }+#[test]fn child_plain_text_document_uses_pre_and_no_quirks_mode() {let (document, kind) = parse_browsing_context_document_snapshot(diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rsindex f8af39bc9..b67274975 100644--- a/moli-renderer-v8/src/live_document_parser.rs+++ b/moli-renderer-v8/src/live_document_parser.rs@@ -717,6 +717,14 @@ impl DocumentParserSession {)}+ pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self {+ Self::new_html(+ HtmlParser::with_scripting_enabled(false)+ .start_text_document(document_url, content_type),+ DocumentParserLifetime::Finite,+ )+ }+pub(crate) fn start_finite_live_document(document_url: Url,document_handle: NativeNodeId,@@ -729,6 +737,21 @@ impl DocumentParserSession {)}+ pub(crate) fn start_finite_live_text_document(+ document_url: Url,+ document_handle: NativeNodeId,+ owner: &mut impl LiveDocumentParserOwner,+ ) -> Self {+ Self::new_html(+ HtmlParser::with_scripting_enabled(false).start_live_text_document_root(+ document_url,+ document_handle,+ owner,+ ),+ DocumentParserLifetime::Finite,+ )+ }+pub(crate) fn start_finite_live_xml_document(document_url: Url,document_handle: NativeNodeId,diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rsindex 67f2387ab..38d747418 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs@@ -170,16 +170,8 @@ impl JsContextHost {navigation_loader: Option<crate::network::navigation::NavigationResourceLoader>,is_xml_document: bool,) -> Option<ChildDocumentInstallResult> {- let is_plain_text_document = snapshot- .content_type- .as_deref()- .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain"));let source = if is_xml_document {std::borrow::Cow::Borrowed(snapshot.markup.as_str())- } else if is_plain_text_document {- std::borrow::Cow::Owned(crate::dom_parser::plain_text_document_parser_input(- &snapshot.markup,- ))} else {crate::dom_parser::preserve_decoded_bom_only_browsing_context_body(&snapshot.markup,@@ -239,6 +231,7 @@ impl JsContextHost {parser_base_url,source.as_ref(),is_xml_document,+ snapshot.content_type.as_deref(),);Some(ChildDocumentInstallResult {initial_classic_ready_work: parser_start.initial_classic_ready_work,diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rsindex 42ad8071a..bc842f8c7 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs@@ -1002,14 +1002,6 @@ impl JsContextHost {let mut owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);parser.finish(&mut owner)};- if self- .dom_host()- .document_content_type_for_handle(document_handle)- .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain"))- {- self.dom_host_mut()- .set_html_quirks_mode_for_parser_document(document_handle, QuirksMode::NoQuirks);- }self.queue_live_child_parser_discovery_signals(child_handle,document_handle,@@ -1695,6 +1687,7 @@ impl JsContextHost {document_base_url: Url,markup: &str,is_xml_document: bool,+ content_type: Option<&str>,) -> ChildLiveDocumentParserStartResult {let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id);self.child_document_parsers.clear(owner);@@ -1703,6 +1696,13 @@ impl JsContextHost {document_base_url,document_handle,)+ } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) {+ let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);+ DocumentParserSession::start_finite_live_text_document(+ document_base_url,+ document_handle,+ &mut parser_owner,+ )} else {DocumentParserSession::start_finite_live_document(document_base_url,diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rsindex fdd143742..35acc157d 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs@@ -21,7 +21,8 @@ use crate::{LoadedChildDocument, SubresourceResponseBody,},};-use moli_encoding::decode_html_document_with_fallback;+use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};+use moli_encoding_detector::detect_legacy_html_encoding;pub(crate) struct AppliedChildDocumentLoadCompletion {/// Initial parser-classic work produced by the committed child document.@@ -762,8 +763,24 @@ fn child_document_load_outcome_from_response(let body_bytes = response_body.try_bytes().map_err(|error| format!("failed to read child document response body: {error}"))?;- let (markup, character_set) =- decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback));+ let (markup, character_set) = if let Some(mime) = content_type+ .as_deref()+ .filter(|mime| moli_web_mime::is_text_document_mime(mime))+ {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &head.headers,+ head.final_url.as_str(),+ detect_legacy_html_encoding,+ moli_web_mime::is_json_module_mime(mime) || mime == "text/json",+ );+ let mut markup = decoder.push(&body_bytes).concat();+ if let Some(tail) = decoder.finish() {+ markup.push_str(&tail);+ }+ (markup, decoder.document_encoding_name())+ } else {+ decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))+ };(markup, character_set.to_owned())};let policy_container =diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rsindex 5af62b821..454433e6c 100644--- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs+++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs@@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime {let (page_vm, triggered_navigation) = {let buffered_document_preloads = &mut state.buffered_document_preloads;let service_worker_preload_context = state.service_worker_preload_context.as_ref();+ let document_character_set = &state.document_character_set;PageVm::new_from_parser_stream_and_run_document_start(page_id,local_executor,@@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime {&mut state.parser_session,started,|page_vm| {+ page_vm.set_document_character_set(document_character_set.clone());admit_pending_preloads(page_vm,buffered_document_preloads,diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rsindex 1a4986fc8..8866cb1aa 100644--- a/moli-renderer-v8/src/runtime/phase_one/state.rs+++ b/moli-renderer-v8/src/runtime/phase_one/state.rs@@ -11,6 +11,7 @@ pub(super) struct ParseTimeDriverState {pub(super) buffered_document_preloads: Box<BufferedDocumentPreloadState>,pub(super) service_worker_preload_context: Option<ServiceWorkerScriptPreloadContext>,pub(super) input_closed: bool,+ pub(super) is_text_document: bool,}impl ParseTimeDriverState {@@ -32,6 +33,24 @@ impl ParseTimeDriverState {buffered_document_preloads: Box::default(),service_worker_preload_context: None,input_closed: false,+ is_text_document: false,+ }+ }++ pub(super) fn new_text(final_url: Url, mime: &str) -> Self {+ Self {+ parser_session: DocumentParserSession::start_main_text_document(+ final_url.clone(),+ mime,+ ),+ final_url,+ document_character_set: "UTF-8".to_owned(),+ scheduler: DocumentScriptScheduler::new(),+ pending_parsing_blocking_script: PendingParsingBlockingClassicScriptRunner::empty(),+ buffered_document_preloads: Box::default(),+ service_worker_preload_context: None,+ input_closed: false,+ is_text_document: true,}}@@ -45,6 +64,7 @@ impl ParseTimeDriverState {buffered_document_preloads: Box::default(),service_worker_preload_context: None,input_closed: false,+ is_text_document: false,}}diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rsindex 15a5d6dfd..28159400d 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs@@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime {env.apply_navigation_response_headers(&response_final_url, &response_headers);let mut body_source = RawDocumentBodySource::fetch_response(response);- let mut state = ParseTimeDriverState::new_with_scripting_enabled(- response_final_url,- main_document_parser_scripting_enabled(&env),- );+ let (mut state, mut decoder) =+ response_document_parser(response_final_url, &response_headers, &env);state.buffered_document_preloads.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(@@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime {)});state.service_worker_preload_context = service_worker_preload_context.clone();- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(- &response_headers,- state.final_url.as_str(),- detect_legacy_html_encoding,- );// Raw navigation bodies are decoded during prebootstrap scan. The decoder// is carried forward so split multibyte sequences are not decoded twice or// lost between the scan and parser handoff.@@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime {},)));}- let mut state = ParseTimeDriverState::new_with_scripting_enabled(- final_url,- main_document_parser_scripting_enabled(&env),- );+ let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env);state.buffered_document_preloads.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(@@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime {)});state.service_worker_preload_context = service_worker_preload_context.clone();- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(- &response_headers,- state.final_url.as_str(),- detect_legacy_html_encoding,- );// External raw bodies replay captured chunks. Pre-scan only what is already// buffered before bootstrap so the producer's backpressure boundary still// controls how far ahead the parser can get.@@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime {chunk: String,service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,) {+ if self.state.is_text_document {+ self.state.parser_session.queue_arrived_chunk(chunk);+ return;+ }self.state.buffered_document_preloads.append_to_main_document_scan_with_service_worker_context(@@ -628,6 +617,9 @@ fn scan_prebootstrap_html_chunk_into_state(chunk: &str,service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,) {+ if state.is_text_document {+ return;+ }state.buffered_document_preloads.append_to_main_document_prebootstrap_scan_with_service_worker_context(@@ -638,6 +630,41 @@ fn scan_prebootstrap_html_chunk_into_state();}+fn response_document_parser(+ final_url: Url,+ headers: &[(String, String)],+ env: &PageVmEnvConfig,+) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {+ let content_type = moli_web_mime::response_document_content_type(headers);+ let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));+ let mut state = if let Some(mime) = &text_type {+ ParseTimeDriverState::new_text(final_url, mime)+ } else {+ ParseTimeDriverState::new_with_scripting_enabled(+ final_url,+ main_document_parser_scripting_enabled(env),+ )+ };+ let decoder = if let Some(mime) = text_type {+ HtmlDocumentStreamingDecoder::new_text_document(+ headers,+ state.final_url.as_str(),+ detect_legacy_html_encoding,+ moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",+ )+ } else {+ HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ headers,+ state.final_url.as_str(),+ detect_legacy_html_encoding,+ )+ };+ // Headers/defaults are observable even when no body chunk is ready yet.+ // A later BOM or decoder decision can still replace this tentative value.+ sync_state_document_character_set_from_decoder(&mut state, &decoder);+ (state, decoder)+}+pub(super) fn enqueue_streaming_raw_chunk(runtime: &mut ConcurrentParseTimeRuntime,decoder: &mut HtmlDocumentStreamingDecoder,@@ -929,6 +956,105 @@ mod tests {use crate::parser::ScriptSource;use tokio::io::{AsyncReadExt, AsyncWriteExt};+ #[tokio::test]+ async fn text_document_charset_is_visible_to_document_start_before_body_arrives() {+ let _js_runtime_owner = crate::JsRuntime::initialize();+ tokio::task::LocalSet::new().run_until(async {+ for (mime, chunk, expected) in [+ ("text/plain; charset=gbk", None, "GBK"),+ ("application/json", None, "UTF-8"),+ ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"),+ ] {+ let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap();+ let loader = loader_owner.handle();+ let mut env = default_test_page_vm_env_config();+ env.document_start_scripts.push(crate::DocumentStartScript {+ registry_key: None,+ devtools_session: None,+ source: "document.title = document.characterSet".to_owned(),+ world_name: None,+ has_bidi_channel_argument: false,+ bidi_channel_handoffs: Vec::new(),+ });+ let (mut state, mut decoder) = response_document_parser(+ Url::parse("https://example.test/data").unwrap(),+ &[("Content-Type".to_owned(), mime.to_owned())], &env,+ );+ if let Some(chunk) = chunk {+ let chunks = decoder.push(chunk);+ sync_state_document_character_set_from_decoder(&mut state, &decoder);+ for chunk in chunks {+ state.parser_session.queue_arrived_chunk(chunk);+ }+ }+ assert_eq!(state.document_character_set, expected);+ let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task(+ PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env,+ PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state,+ Instant::now(), "charset test bootstrap closed",+ ).await.unwrap();+ assert!(!navigated);+ assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}");+ }+ }).await;+ }++ #[test]+ fn response_document_dispatch_distinguishes_text_from_html_and_xml() {+ for (mime, text) in [+ ("application/json", true),+ ("application/problem+json", true),+ ("text/plain; charset=gbk", true),+ ("text/javascript", true),+ ("text/html", false),+ ("application/xhtml+xml", false),+ ("application/xml", false),+ ("text/xml", false),+ ] {+ let (state, _) = response_document_parser(+ Url::parse("https://example.test/data").unwrap(),+ &[("Content-Type".to_owned(), mime.to_owned())],+ &default_test_page_vm_env_config(),+ );+ assert_eq!(state.is_text_document, text, "{mime}");+ }+ }++ #[tokio::test]+ async fn text_document_does_not_preload_literal_script_tags() {+ let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default())+ .expect("default loader");+ let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test();+ for (mime, expected_preload) in [+ ("text/plain", false),+ ("application/json", false),+ ("text/html", true),+ ] {+ let (mut state, _) = response_document_parser(+ Url::parse("https://example.test/data").unwrap(),+ &[("Content-Type".to_owned(), mime.to_owned())],+ &default_test_page_vm_env_config(),+ );+ state.buffered_document_preloads.bind_resource_runtime(+ runtime_hooks.owner_wake(),+ runtime_hooks.resource_task_runner(),+ );+ scan_prebootstrap_html_chunk_into_state(+ &mut state,+ &loader,+ "<script defer src='/literal.js'></script>",+ None,+ );+ assert_eq!(+ state.buffered_document_preloads.entries.contains_key(+ &classic_preload_key_for_streaming_test("https://example.test/literal.js")+ ),+ expected_preload,+ "{mime}",+ );+ }+ }+#[tokio::test]async fn external_raw_document_body_source_collects_chunks_and_completion() {let (completion_tx, completion_rx) = oneshot::channel();diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rsindex 3ff3a0dd5..35fca10b5 100644--- a/moli-web-mime/src/classification.rs+++ b/moli-web-mime/src/classification.rs@@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool {input.starts_with("text/")}+/// Navigation responses rendered as literal text rather than HTML or XML.+/// This also includes JSON and JavaScript resources opened as documents.+pub fn is_text_document_mime(input: &str) -> bool {+ !is_html_document_mime(input)+ && !is_dom_parser_xml_mime(input)+ && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))+}+/// Whether a `style` element's raw `type` attribute selects classic CSS.////// HTML's update-a-style-block algorithm compares the untrimmed attributediff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rsindex 615e882d2..ec09da8be 100644--- a/moli-web-mime/src/lib.rs+++ b/moli-web-mime/src/lib.rs@@ -14,9 +14,9 @@ pub use classification::{is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,- is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime,- is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime,- multipart_form_data_boundary,+ is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,+ is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,+ is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,};pub use data_url::{data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rsindex e475520a9..a21ab2a60 100644--- a/moli-web-mime/src/tests.rs+++ b/moli-web-mime/src/tests.rs@@ -648,3 +648,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() {));assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[]));}+#[test]+fn document_text_classification_excludes_html_and_xml() {+ for mime in [+ "text/plain",+ "Text/Css; charset=utf-8",+ "application/json",+ "application/ld+json",+ "application/javascript",+ ] {+ assert!(crate::is_text_document_mime(mime), "{mime}");+ }+ for mime in [+ "text/html",+ "text/xml",+ "application/xhtml+xml",+ "image/svg+xml",+ "image/png",+ "application/octet-stream",+ ] {+ assert!(!crate::is_text_document_mime(mime), "{mime}");+ }+}diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rsindex a1599c989..e9ed136c6 100644--- a/moli/src/fetch_dump/tests.rs+++ b/moli/src/fetch_dump/tests.rs@@ -1,7 +1,7 @@use std::sync::Arc;use anyhow::Result;-use axum::{Router, routing::get};+use axum::{Router, response::Html, routing::get};use moli_core::{LayoutPolicy,page::Page,@@ -36,7 +36,7 @@ async fn load_page_with_config("/",get(move || {let body = Arc::clone(&server_body);- async move { (*body).clone() }+ async move { Html((*body).clone()) }}),);axum::serve(listener, app).await.unwrap();