Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(document): preserve inherited encoding and XML MIME routing
ldm0a3af75845dbc
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rsindex 8b3a70398..db8e77016 100644--- a/moli-encoding/src/document.rs+++ b/moli-encoding/src/document.rs@@ -52,14 +52,23 @@ pub struct HtmlDocumentStreamingDecoder {impl HtmlDocumentStreamingDecoder {/// Decode a text document without interpreting literal HTML/XML declarations.- /// JSON defaults to UTF-8; other text retains the configured legacy detector.+ /// BOMs and transport charsets take precedence. JSON then defaults to UTF-8;+ /// other text uses a valid inherited encoding before heuristic detection.pub fn new_text_document(headers: &[(String, Vec<u8>)],url_hint: &str,detector: LegacyEncodingDetector,json: bool,+ inherited_encoding: Option<&str>,) -> Self {- let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));+ let inherited_encoding =+ inherited_encoding.filter(|label| encoding_for_label(label).is_some());+ let mut decoder = Self::new_with_options(+ headers,+ inherited_encoding,+ Some(url_hint),+ inherited_encoding.is_none().then_some(detector),+ );decoder.sniff_html_declarations = false;if json && decoder.transport_encoding.is_none() {decoder.transport_encoding = Some(encoding_rs::UTF_8);@@ -169,6 +178,9 @@ impl HtmlDocumentStreamingDecoder {return Some(encoding);}if !self.sniff_html_declarations {+ if self.legacy_encoding_detector.is_none() {+ return Some(self.fallback_encoding);+ }return (finishing || self.sniff_buffer.len() >= 1024).then(|| {self.detected_legacy_content_encoding().unwrap_or(self.fallback_encoding)diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rsindex 2217e0521..1ce5be37b 100644--- a/moli-encoding/src/tests.rs+++ b/moli-encoding/src/tests.rs@@ -33,6 +33,7 @@ fn json_text_document_defaults_to_utf8_across_every_chunk_split() {"https://example.test/",unexpected_legacy_encoding_detector,true,+ Some("GBK"),);let mut output = decoder.push(&input.as_bytes()[..split]).join("");output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));@@ -53,6 +54,7 @@ fn plain_text_document_honors_transport_charset() {"https://example.test/",unexpected_legacy_encoding_detector,false,+ Some("UTF-8"),);let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");output.push_str(&decoder.finish().unwrap_or_default());@@ -71,6 +73,7 @@ fn text_document_bom_precedes_transport_charset() {"https://example.test/",unexpected_legacy_encoding_detector,false,+ Some("windows-1251"),);assert_eq!(decoder.document_encoding_name(), "GBK");assert_eq!(decoder.selected_encoding_name(), None);@@ -92,6 +95,7 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() {"https://legacy.example/",test_legacy_encoding_detector,false,+ None,);let mut output = decoder.push(&[0x80]).join("");output.push_str(&decoder.finish().unwrap_or_default());@@ -99,6 +103,47 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() {assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));}+#[test]+fn plain_text_document_inherits_encoding_before_detection_across_chunk_splits() {+ for (input, encoding) in [+ ("plain ASCII", encoding_rs::UTF_8),+ ("<meta charset=gbk>吴姓-姓氏渊源", encoding_rs::UTF_8),+ ("<?xml encoding='utf-8'?>家居", encoding_rs::GBK),+ ] {+ let bytes = encoding.encode(input).0;+ for split in 0..=bytes.len() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://example.test/",+ unexpected_legacy_encoding_detector,+ false,+ Some(encoding.name()),+ );+ assert_eq!(decoder.document_encoding_name(), encoding.name());+ let mut output = decoder.push(&bytes[..split]).concat();+ output.push_str(&decoder.push(&bytes[split..]).concat());+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, input, "{} split {split}", encoding.name());+ assert_eq!(decoder.selected_encoding_name(), Some(encoding.name()));+ }+ }+}++#[test]+fn invalid_inherited_text_encoding_does_not_disable_detection() {+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(+ &[],+ "https://legacy.example/",+ test_legacy_encoding_detector,+ false,+ Some("not-an-encoding"),+ );+ let mut output = decoder.push(&[0x80]).concat();+ output.push_str(&decoder.finish().unwrap_or_default());+ assert_eq!(output, "А");+ assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));+}+#[test]fn plain_text_document_ignores_literal_encoding_declarations() {for prefix in [@@ -112,6 +157,7 @@ fn plain_text_document_ignores_literal_encoding_declarations() {"https://example.test/",test_legacy_encoding_detector,false,+ None,);let mut output = decoder.push(&input).join("");output.push_str(&decoder.finish().unwrap_or_default());diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rsindex abf196d99..9649ec823 100644--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs@@ -1,5 +1,163 @@use super::*;+#[tokio::test]+async fn webdriver_classic_text_children_preserve_parent_encoding_without_a_charset() {+ const LITERAL: &str = "<meta charset=gbk>吴姓-姓氏渊源";+ let cases = [+ ("ascii", "plain ASCII"),+ ("utf8", "吴姓-姓氏渊源"),+ ("literal", LITERAL),+ ("data", LITERAL),+ ];+ let mut fixture = axum::Router::new();+ for (name, payload) in cases {+ fixture = fixture.route(+ &format!("/{name}"),+ axum::routing::get(+ move || async move { ([(header::CONTENT_TYPE, "text/plain")], payload) },+ ),+ );+ }+ fixture = fixture.route(+ "/parent/{kind}",+ axum::routing::get(+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {+ let target = if kind == "data" {+ format!(+ "data:text/plain;base64,{}",+ base64::Engine::encode(&BASE64_STANDARD, LITERAL)+ )+ } else {+ format!("/{kind}")+ };+ (+ [(header::CONTENT_TYPE, "text/html; charset=utf-8")],+ format!("<iframe src='{target}'></iframe>"),+ )+ },+ ),+ );+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "inherited-text-encoding");+ let app = build_router(test_state());+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;+ let session_id = session["value"]["sessionId"].as_str().unwrap();+ for (name, payload) in cases {+ classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/url"),+ json!({"url":format!("http://{addr}/parent/{name}")}),+ )+ .await;+ let parent = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/execute/sync"),+ json!({"script": "return document.characterSet;", "args": []}),+ )+ .await;+ assert_eq!(parent["value"], json!("UTF-8"), "{name}");+ classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/frame"),+ json!({"id": 0}),+ )+ .await;+ let observed = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/execute/sync"),+ json!({+ "script": "return [document.body.textContent,document.characterSet,document.contentType];",+ "args": [],+ }),+ ).await;+ assert_eq!(+ observed["value"],+ json!([payload, "UTF-8", "text/plain"]),+ "{name}"+ );+ classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/frame"),+ json!({"id": null}),+ )+ .await;+ }+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;+}++#[tokio::test]+async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_parser_types() {+ let cases = [+ ("atom.xml", "application/atom+xml"),+ ("rss.xml", "application/rss+xml"),+ ("custom.xml", "text/example+xml"),+ ];+ let mut fixture = axum::Router::new();+ for (name, mime) in cases {+ fixture = fixture.route(+ &format!("/{name}"),+ axum::routing::get(move || async move {+ (+ [(header::CONTENT_TYPE, mime)],+ "<feed xmlns='http://www.w3.org/2005/Atom'><title>吴姓</title></feed>",+ )+ }),+ );+ }+ fixture = fixture.route(+ "/parent/{kind}",+ axum::routing::get(+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {+ (+ [(header::CONTENT_TYPE, "text/html; charset=windows-1252")],+ format!("<iframe src='/{kind}'></iframe>"),+ )+ },+ ),+ );+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-suffix-navigation");+ let app = build_router(test_state());+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;+ let session_id = session["value"]["sessionId"].as_str().unwrap();+ for (name, mime) in cases {+ for prefix in ["", "parent/"] {+ classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/url"),+ json!({"url":format!("http://{addr}/{prefix}{name}")}),+ )+ .await;+ let observed = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/execute/sync"),+ json!({+ "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; let rejected=false; try { new DOMParser().parseFromString('<feed/>',d.contentType); } catch(error) { rejected=error instanceof TypeError; } const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet,rejected];",+ "args": [],+ }),+ ).await;+ assert_eq!(+ observed["value"],+ json!([+ "feed",+ "http://www.w3.org/2005/Atom",+ "吴姓",+ mime,+ "UTF-8",+ true+ ]),+ "{prefix}{name}"+ );+ }+ }+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;+}+#[tokio::test]async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {const PAYLOAD: &str =diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rsindex 0f9674b25..d44b8a737 100644--- a/moli-protocol/src/conn/runtime_load.rs+++ b/moli-protocol/src/conn/runtime_load.rs@@ -158,7 +158,7 @@ async fn prepare_network_error_page_navigation_with_engine_async(fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {moli_web_mime::response_document_content_type(headers)- .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime))+ .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))}#[derive(Clone, Debug)]diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rsnew file mode 100644index 000000000..5d291b90e--- /dev/null+++ b/moli-renderer-v8/src/document_response_decoder.rs@@ -0,0 +1,77 @@+use moli_encoding::HtmlDocumentStreamingDecoder;+use moli_encoding_detector::detect_legacy_html_encoding;+use moli_web_mime::{is_json_document_mime, is_text_document_mime, is_xml_document_mime};+use url::Url;++pub(crate) fn new_document_response_decoder(+ headers: &[(String, Vec<u8>)],+ content_type: Option<&str>,+ final_url: &Url,+ inherited_encoding: Option<&str>,+) -> HtmlDocumentStreamingDecoder {+ if let Some(mime) = content_type.filter(|mime| is_text_document_mime(mime)) {+ HtmlDocumentStreamingDecoder::new_text_document(+ headers,+ final_url.as_str(),+ detect_legacy_html_encoding,+ is_json_document_mime(mime),+ inherited_encoding,+ )+ } else if content_type.is_some_and(is_xml_document_mime) {+ // XML declarations still apply; neither parent encoding nor heuristics+ // can replace XML's UTF-8 default.+ HtmlDocumentStreamingDecoder::new_with_fallback(headers, Some("UTF-8"))+ } else if inherited_encoding.is_some() {+ HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding)+ } else {+ HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(+ headers,+ final_url.as_str(),+ detect_legacy_html_encoding,+ )+ }+}++pub(crate) fn decode_document_response(+ bytes: &[u8],+ headers: &[(String, Vec<u8>)],+ content_type: Option<&str>,+ final_url: &Url,+ inherited_encoding: Option<&str>,+) -> (String, &'static str) {+ let mut decoder =+ new_document_response_decoder(headers, content_type, final_url, inherited_encoding);+ let mut text = decoder.push(bytes).concat();+ if let Some(tail) = decoder.finish() {+ text.push_str(&tail);+ }+ (text, decoder.document_encoding_name())+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() {+ for (mime, source) in [+ ("application/json", "{\"text\":\"吴姓\"}"),+ ("Text/JSON", "{\"text\":\"吴姓\"}"),+ ("application/problem+json", "{\"text\":\"吴姓\"}"),+ ("application/atom+xml", "<feed>吴姓</feed>"),+ ("application/rss+xml", "<feed>吴姓</feed>"),+ ("text/example+xml", "<feed>吴姓</feed>"),+ ] {+ let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())];+ let (decoded, encoding) = decode_document_response(+ source.as_bytes(),+ &headers,+ Some(mime),+ &Url::parse("https://example.test/document").unwrap(),+ Some("GBK"),+ );+ assert_eq!(decoded, source, "{mime}");+ assert_eq!(encoding, "UTF-8", "{mime}");+ }+ }+}diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rsindex 0dfa341eb..d764b479d 100644--- a/moli-renderer-v8/src/dom_parser.rs+++ b/moli-renderer-v8/src/dom_parser.rs@@ -1,5 +1,5 @@use crate::web_api_interfaces;-use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime};+use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime, is_xml_document_mime};use moli_webapi_declare::WebApiFunctionTemplate;use url::Url;@@ -379,7 +379,7 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>() -> std::borrow::Cow<'a, str> {if source == "\u{feff}"&& !content_type.is_some_and(|mime| {- is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)+ is_xml_document_mime(mime) || moli_web_mime::is_text_document_mime(mime)}){std::borrow::Cow::Borrowed("<body>\u{feff}</body>")@@ -394,7 +394,7 @@ fn parse_browsing_context_document_snapshot(content_type: Option<&str>,html_parser: HtmlParser,) -> (DomHost, DetachedDocumentKind) {- if content_type.is_some_and(is_dom_parser_xml_mime)+ if content_type.is_some_and(is_xml_document_mime)|| (content_type.is_none() && child_document_url_is_xml_like(&document_url)){let parser = XmlParser;@@ -585,6 +585,34 @@ mod tests {assert!(disabled.element_handle_by_id("fallback").is_some());}+ #[test]+ fn child_projection_preserves_xml_suffix_document_structure_and_namespace() {+ let source = "<feed xmlns='http://www.w3.org/2005/Atom'><title>Example</title></feed>";+ for mime in [+ "application/atom+xml",+ "application/rss+xml",+ "text/example+xml",+ ] {+ for path in ["feed.xml", "feed.html", "feed"] {+ let (document, kind) = parse_browsing_context_document_snapshot(+ Url::parse(&format!("https://example.test/{path}")).unwrap(),+ source,+ Some(mime),+ HtmlParser::SCRIPTING_ENABLED,+ );+ let root = first_document_element(&document);+ let element = document+ .node(root)+ .and_then(|node| node.as_element())+ .unwrap();+ assert_eq!(kind, DetachedDocumentKind::Xml, "{mime} at {path}");+ assert_eq!(element.local_name(), "feed", "{mime} at {path}");+ assert_eq!(element.namespace(), "http://www.w3.org/2005/Atom");+ assert_eq!(document.text_content(root).as_deref(), Some("Example"));+ }+ }+ }+#[test]fn child_projection_explicit_mime_overrides_xml_extension() {let source = "\n<&</pre><script>window.executed=1</script>";diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rsindex 4f6db92af..40a233921 100644--- a/moli-renderer-v8/src/lib.rs+++ b/moli-renderer-v8/src/lib.rs@@ -48,6 +48,7 @@ mod document_cookie_owner;mod document_language;mod document_last_modified;mod document_module_graph;+mod document_response_decoder;mod document_runtime;mod document_script_scheduler;mod document_task_lane;diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rsindex 1d4e2d9bc..f8016efb8 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs@@ -6,7 +6,7 @@ use crate::frame_owner_model::{FrameDocumentLocalWindowTransition, FrameLocalWindowOwnerTransition,};use crate::{document_runtime::DomHandle, frame_owner_model::FrameDocumentOwnerTransition};-use moli_web_mime::is_dom_parser_xml_mime;+use moli_web_mime::is_xml_document_mime;use url::Url;pub(crate) struct ChildDocumentInstallResult {@@ -135,7 +135,7 @@ impl JsContextHost {let is_xml_document = snapshot.content_type.as_deref()- .is_some_and(is_dom_parser_xml_mime);+ .is_some_and(is_xml_document_mime);self.install_live_child_document_from_snapshot(scope,handle,diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rsindex d429de742..90fac5d67 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs@@ -10,6 +10,7 @@ use crate::{content_security_policy::{ContentSecurityPolicyViolationEventFields, send_content_security_policy_reports,},+ document_response_decoder::decode_document_response,document_runtime::{DocumentNavigationEmbeddingContext, DocumentPolicyContainer, DomHandle},document_script_scheduler::FrameDocumentClassicScriptSchedulerWork,frame_owner_model::{@@ -21,8 +22,6 @@ use crate::{LoadedChildDocument, SubresourceResponseBody,},};-use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};-use moli_encoding_detector::detect_legacy_html_encoding;pub(crate) struct AppliedChildDocumentLoadCompletion {/// Initial parser-classic work produced by the committed child document.@@ -757,17 +756,6 @@ impl JsContextHost {}}-fn child_document_fallback_character_set(- parent_character_set: &str,- content_type: Option<&str>,-) -> String {- if content_type.is_some_and(moli_web_mime::is_dom_parser_xml_mime) {- "UTF-8".to_owned()- } else {- parent_character_set.to_owned()- }-}-fn child_document_load_outcome_from_response(request_url: String,request_method: String,@@ -786,30 +774,17 @@ fn child_document_load_outcome_from_response(let encoded_data_length = response_body.len();let content_type = child_document_content_type_from_headers(&head.headers).or_else(|| child_document_content_type_for_url(&head.final_url));- let fallback =- child_document_fallback_character_set(parent_character_set, content_type.as_deref());let (markup, character_set) = {let body_bytes = response_body.try_bytes().map_err(|error| format!("failed to read child document response body: {error}"))?;- let (markup, character_set) = if let Some(mime) = content_type- .as_deref()- .filter(|mime| moli_web_mime::is_text_document_mime(mime))- {- let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(- &head.headers,- head.final_url.as_str(),- detect_legacy_html_encoding,- moli_web_mime::is_json_module_mime(mime) || mime == "text/json",- );- let mut markup = decoder.push(&body_bytes).concat();- if let Some(tail) = decoder.finish() {- markup.push_str(&tail);- }- (markup, decoder.document_encoding_name())- } else {- decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))- };+ let (markup, character_set) = decode_document_response(+ &body_bytes,+ &head.headers,+ content_type.as_deref(),+ &head.final_url,+ Some(parent_character_set),+ );(markup, character_set.to_owned())};let policy_container =diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rsindex 71d318c35..20e3c344b 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs@@ -1,11 +1,10 @@use super::super::{ChildBrowsingContextBootstrap, ChildBrowsingContextSnapshot, JsContextHost};use super::configure_child_document_navigation_request;+use crate::document_response_decoder::decode_document_response;use crate::document_runtime::{DocumentPolicyContainer, DomHandle};-use moli_encoding::decode_html_document_with_fallback;use moli_fetch::Request;use moli_web_mime::{- is_dom_parser_xml_mime, is_html_document_mime, resource_mime_essence_for_url,- response_document_content_type,+ is_html_document_mime, resource_mime_essence_for_url, response_document_content_type,};use url::Url;@@ -52,7 +51,7 @@ impl JsContextHost {if !mime_type.is_empty() && !is_html_document_mime(&mime_type) {return None;}- let character_set = self.child_document_fallback_character_set(Some("text/html"));+ let character_set = self.document_character_set().to_owned();Some(ChildBrowsingContextSnapshot::with_character_set(url.clone(),body,@@ -64,11 +63,12 @@ impl JsContextHost {"data" => crate::network_host::local_url_response(url).map(|response| {let head = response.head();let content_type = child_document_content_type_from_headers(&head.headers);- let fallback = self.child_document_fallback_character_set(content_type.as_deref());- let (body, character_set) = decode_html_document_with_fallback(+ let (body, character_set) = decode_document_response(response.body_bytes(),&head.headers,- Some(&fallback),+ content_type.as_deref(),+ &head.final_url,+ Some(self.document_character_set()),);let policy_container = DocumentPolicyContainer::from_navigation_response_headers(&head.headers,@@ -116,11 +116,12 @@ impl JsContextHost {let head = response.head();let content_type = child_document_content_type_from_headers(&head.headers).or_else(|| child_document_content_type_for_url(&head.final_url));- let fallback = self.child_document_fallback_character_set(content_type.as_deref());- let (markup, character_set) = decode_html_document_with_fallback(+ let (markup, character_set) = decode_document_response(response.body_bytes(),&head.headers,- Some(&fallback),+ content_type.as_deref(),+ &head.final_url,+ Some(self.document_character_set()),);let policy_container = DocumentPolicyContainer::from_navigation_response_headers(&head.headers,@@ -170,17 +171,6 @@ impl JsContextHost {ChildBrowsingContextBootstrap::Request(_) => None,}}-- pub(in crate::native_bridge::context_host) fn child_document_fallback_character_set(- &self,- content_type: Option<&str>,- ) -> String {- if content_type.is_some_and(is_dom_parser_xml_mime) {- "UTF-8".to_owned()- } else {- self.document_character_set().to_owned()- }- }}pub(in crate::native_bridge::context_host) fn child_document_content_type_from_headers(diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rsindex 10c829ac1..1a84f1a04 100644--- a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs@@ -238,7 +238,7 @@ impl JsContextHost {return false;}moli_web_mime::is_html_document_mime(&mime)- || moli_web_mime::is_dom_parser_xml_mime(&mime)+ || moli_web_mime::is_xml_document_mime(&mime)|| moli_web_mime::is_text_mime(&mime)|| moli_web_mime::is_json_module_mime(&mime)|| moli_web_mime::is_javascript_mime(&mime)diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rsindex 5204a4d92..4f3e187a2 100644--- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs+++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs@@ -2310,7 +2310,7 @@ impl JsContextHost {super::child_documents::child_document_content_type_from_headers(&head.headers);let fallback = if content_type.as_deref()- .is_some_and(moli_web_mime::is_dom_parser_xml_mime)+ .is_some_and(moli_web_mime::is_xml_document_mime){"UTF-8".to_owned()} else {diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rsindex fa8df43b8..f8c9351b9 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs@@ -3,8 +3,8 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_penuse super::parser_blocking_pending::main_parser_blocking_classic_script_item;use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;use super::*;+use crate::document_response_decoder::new_document_response_decoder;use moli_encoding::HtmlDocumentStreamingDecoder;-use moli_encoding_detector::detect_legacy_html_encoding;use moli_web_mime::response_headers_indicate_attachment_download;use tokio::sync::{mpsc, oneshot};@@ -471,7 +471,7 @@ impl ConcurrentParseTimeRuntime {fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {moli_web_mime::response_document_content_type(headers)- .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime))+ .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))}const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8;@@ -615,8 +615,10 @@ fn response_document_parser(env: &PageVmEnvConfig,) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {let content_type = moli_web_mime::response_document_content_type(headers);- let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));- let mut state = if let Some(mime) = &text_type {+ let text_type = content_type+ .as_deref()+ .filter(|mime| moli_web_mime::is_text_document_mime(mime));+ let mut state = if let Some(mime) = text_type {ParseTimeDriverState::new_text(final_url, mime)} else {ParseTimeDriverState::new_with_scripting_enabled(@@ -624,20 +626,8 @@ fn response_document_parser(main_document_parser_scripting_enabled(env),)};- let decoder = if let Some(mime) = text_type {- HtmlDocumentStreamingDecoder::new_text_document(- headers,- state.final_url.as_str(),- detect_legacy_html_encoding,- moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",- )- } else {- HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(- headers,- state.final_url.as_str(),- detect_legacy_html_encoding,- )- };+ let decoder =+ new_document_response_decoder(headers, content_type.as_deref(), &state.final_url, None);// Headers/defaults are observable even when no body chunk is ready yet.// A later BOM or decoder decision can still replace this tentative value.sync_state_document_character_set_from_decoder(&mut state, &decoder);diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rsindex 35fca10b5..9574cdde2 100644--- a/moli-web-mime/src/classification.rs+++ b/moli-web-mime/src/classification.rs@@ -13,6 +13,14 @@ pub fn is_dom_parser_xml_mime(input: &str) -> bool {})}+/// XML navigation responses, including registered and custom `+xml` subtypes.+/// DOMParser's Web API accepts a narrower, separately classified set of types.+pub fn is_xml_document_mime(input: &str) -> bool {+ mime_essence(input).is_some_and(|mime| {+ matches!(mime.as_str(), "text/xml" | "application/xml") || mime.ends_with("+xml")+ })+}+pub fn is_javascript_mime(input: &str) -> bool {mime_essence(input).is_some_and(|mime| is_javascript_mime_essence(&mime))}@@ -55,7 +63,7 @@ pub fn is_text_mime_essence(input: &str) -> bool {/// This also includes JSON and JavaScript resources opened as documents.pub fn is_text_document_mime(input: &str) -> bool {!is_html_document_mime(input)- && !is_dom_parser_xml_mime(input)+ && !is_xml_document_mime(input)&& (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))}@@ -143,6 +151,12 @@ pub fn is_json_module_mime(input: &str) -> bool {mime_essence(input).is_some_and(|mime| mime == "application/json" || mime.ends_with("+json"))}+pub fn is_json_document_mime(input: &str) -> bool {+ mime_essence(input).is_some_and(|mime| {+ matches!(mime.as_str(), "application/json" | "text/json") || mime.ends_with("+json")+ })+}+pub fn is_webassembly_mime(input: &str) -> bool {mime_essence(input).is_some_and(|mime| mime == "application/wasm")}diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rsindex 806eb2455..bcae3a9eb 100644--- a/moli-web-mime/src/lib.rs+++ b/moli-web-mime/src/lib.rs@@ -13,10 +13,11 @@ pub use classification::{is_css_style_element_type_attribute, is_css_stylesheet_type_hint, is_dom_parser_xml_mime,is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,- is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,- is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,- is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,- is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,+ is_json_document_mime, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime,+ is_png_image_mime_essence, is_supported_document_mime_type, is_svg_image_mime,+ is_svg_image_mime_essence, is_text_document_mime, is_text_mime, is_text_mime_essence,+ is_video_mime, is_video_mime_essence, is_webassembly_mime, is_xml_document_mime,+ multipart_form_data_boundary,};pub use data_url::{data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,diff --git a/moli-web-mime/src/sniffing.rs b/moli-web-mime/src/sniffing.rsindex 5b21532b6..28f0d6342 100644--- a/moli-web-mime/src/sniffing.rs+++ b/moli-web-mime/src/sniffing.rs@@ -1,4 +1,4 @@-use crate::classification::{is_dom_parser_xml_mime, is_html_document_mime};+use crate::classification::{is_html_document_mime, is_xml_document_mime};use crate::parse::mime_essence;pub const RESOURCE_HEADER_BYTE_LIMIT: usize = 1445;@@ -212,7 +212,7 @@ fn computed_browsing_mime_type(}fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String {- if supplied.is_some_and(is_xml_mime) {+ if supplied.is_some_and(is_xml_document_mime) {return supplied.expect("checked supplied MIME").into();}sniff_image_mime_type(header)@@ -222,7 +222,7 @@ fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String {}fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> String {- if supplied.is_some_and(is_xml_mime) {+ if supplied.is_some_and(is_xml_document_mime) {return supplied.expect("checked supplied MIME").into();}sniff_audio_video_mime_type(header)@@ -232,7 +232,7 @@ fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> Stri}fn computed_font_mime_type(supplied: Option<&str>, header: &[u8]) -> String {- if supplied.is_some_and(is_xml_mime) {+ if supplied.is_some_and(is_xml_document_mime) {return supplied.expect("checked supplied MIME").into();}sniff_font_mime_type(header)@@ -424,11 +424,7 @@ fn is_binary_data_byte(byte: u8) -> bool {}fn is_xml_or_html_mime(mime: &str) -> bool {- is_xml_mime(mime) || is_html_mime(mime)-}--fn is_xml_mime(mime: &str) -> bool {- is_dom_parser_xml_mime(mime) || mime.ends_with("+xml")+ is_xml_document_mime(mime) || is_html_mime(mime)}fn is_html_mime(mime: &str) -> bool {diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rsindex d29adb69b..d361b05e0 100644--- a/moli-web-mime/src/tests.rs+++ b/moli-web-mime/src/tests.rs@@ -886,9 +886,53 @@ fn document_text_classification_excludes_html_and_xml() {"text/xml","application/xhtml+xml","image/svg+xml",+ "application/atom+xml",+ "application/rss+xml",+ "text/example+xml; charset=utf-8","image/png","application/octet-stream",] {assert!(!crate::is_text_document_mime(mime), "{mime}");}}++#[test]+fn navigation_xml_mime_classification_keeps_dom_parser_allowlist_separate() {+ for mime in [+ "text/xml",+ "application/xml",+ "application/xhtml+xml",+ "image/svg+xml",+ ] {+ assert!(is_xml_document_mime(mime), "{mime}");+ assert!(is_dom_parser_xml_mime(mime), "{mime}");+ }+ for mime in [+ "Application/Atom+XML; charset=utf-8",+ "application/rss+xml",+ "text/example+xml",+ ] {+ assert!(is_xml_document_mime(mime), "{mime}");+ assert!(!is_dom_parser_xml_mime(mime), "{mime}");+ assert!(!is_text_document_mime(mime), "{mime}");+ }+ for mime in [+ "text/html",+ "text/plain",+ "application/xml-dtd",+ "invalid+xml",+ ] {+ assert!(!is_xml_document_mime(mime), "{mime}");+ }+}++#[test]+fn navigation_json_mime_includes_text_json_without_changing_module_mime() {+ for mime in ["application/json", "application/problem+json"] {+ assert!(is_json_document_mime(mime), "{mime}");+ assert!(is_json_module_mime(mime), "{mime}");+ }+ assert!(is_json_document_mime("Text/JSON; charset=utf-8"));+ assert!(!is_json_module_mime("Text/JSON; charset=utf-8"));+ assert!(!is_json_document_mime("text/plain"));+}