Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(document): preserve inherited encoding and XML MIME routing

ldm0a3af75845dbc
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index 8b3a70398..db8e77016 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -52,14 +52,23 @@ pub struct HtmlDocumentStreamingDecoder {
impl HtmlDocumentStreamingDecoder {
/// Decode a text document without interpreting literal HTML/XML declarations.
- /// JSON defaults to UTF-8; other text retains the configured legacy detector.
+ /// BOMs and transport charsets take precedence. JSON then defaults to UTF-8;
+ /// other text uses a valid inherited encoding before heuristic detection.
pub fn new_text_document(
headers: &[(String, Vec<u8>)],
url_hint: &str,
detector: LegacyEncodingDetector,
json: bool,
+ inherited_encoding: Option<&str>,
) -> Self {
- let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));
+ let inherited_encoding =
+ inherited_encoding.filter(|label| encoding_for_label(label).is_some());
+ let mut decoder = Self::new_with_options(
+ headers,
+ inherited_encoding,
+ Some(url_hint),
+ inherited_encoding.is_none().then_some(detector),
+ );
decoder.sniff_html_declarations = false;
if json && decoder.transport_encoding.is_none() {
decoder.transport_encoding = Some(encoding_rs::UTF_8);
@@ -169,6 +178,9 @@ impl HtmlDocumentStreamingDecoder {
return Some(encoding);
}
if !self.sniff_html_declarations {
+ if self.legacy_encoding_detector.is_none() {
+ return Some(self.fallback_encoding);
+ }
return (finishing || self.sniff_buffer.len() >= 1024).then(|| {
self.detected_legacy_content_encoding()
.unwrap_or(self.fallback_encoding)
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 2217e0521..1ce5be37b 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -33,6 +33,7 @@ fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
"https://example.test/",
unexpected_legacy_encoding_detector,
true,
+ Some("GBK"),
);
let mut output = decoder.push(&input.as_bytes()[..split]).join("");
output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
@@ -53,6 +54,7 @@ fn plain_text_document_honors_transport_charset() {
"https://example.test/",
unexpected_legacy_encoding_detector,
false,
+ Some("UTF-8"),
);
let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
output.push_str(&decoder.finish().unwrap_or_default());
@@ -71,6 +73,7 @@ fn text_document_bom_precedes_transport_charset() {
"https://example.test/",
unexpected_legacy_encoding_detector,
false,
+ Some("windows-1251"),
);
assert_eq!(decoder.document_encoding_name(), "GBK");
assert_eq!(decoder.selected_encoding_name(), None);
@@ -92,6 +95,7 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() {
"https://legacy.example/",
test_legacy_encoding_detector,
false,
+ None,
);
let mut output = decoder.push(&[0x80]).join("");
output.push_str(&decoder.finish().unwrap_or_default());
@@ -99,6 +103,47 @@ fn plain_text_document_retains_legacy_detection_without_a_charset() {
assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
}
+#[test]
+fn plain_text_document_inherits_encoding_before_detection_across_chunk_splits() {
+ for (input, encoding) in [
+ ("plain ASCII", encoding_rs::UTF_8),
+ ("<meta charset=gbk>吴姓-姓氏渊源", encoding_rs::UTF_8),
+ ("<?xml encoding='utf-8'?>家居", encoding_rs::GBK),
+ ] {
+ let bytes = encoding.encode(input).0;
+ for split in 0..=bytes.len() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ false,
+ Some(encoding.name()),
+ );
+ assert_eq!(decoder.document_encoding_name(), encoding.name());
+ let mut output = decoder.push(&bytes[..split]).concat();
+ output.push_str(&decoder.push(&bytes[split..]).concat());
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, input, "{} split {split}", encoding.name());
+ assert_eq!(decoder.selected_encoding_name(), Some(encoding.name()));
+ }
+ }
+}
+
+#[test]
+fn invalid_inherited_text_encoding_does_not_disable_detection() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://legacy.example/",
+ test_legacy_encoding_detector,
+ false,
+ Some("not-an-encoding"),
+ );
+ let mut output = decoder.push(&[0x80]).concat();
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "А");
+ assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
+}
+
#[test]
fn plain_text_document_ignores_literal_encoding_declarations() {
for prefix in [
@@ -112,6 +157,7 @@ fn plain_text_document_ignores_literal_encoding_declarations() {
"https://example.test/",
test_legacy_encoding_detector,
false,
+ None,
);
let mut output = decoder.push(&input).join("");
output.push_str(&decoder.finish().unwrap_or_default());
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index abf196d99..9649ec823 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -1,5 +1,163 @@
use super::*;
+#[tokio::test]
+async fn webdriver_classic_text_children_preserve_parent_encoding_without_a_charset() {
+ const LITERAL: &str = "<meta charset=gbk>吴姓-姓氏渊源";
+ let cases = [
+ ("ascii", "plain ASCII"),
+ ("utf8", "吴姓-姓氏渊源"),
+ ("literal", LITERAL),
+ ("data", LITERAL),
+ ];
+ let mut fixture = axum::Router::new();
+ for (name, payload) in cases {
+ fixture = fixture.route(
+ &format!("/{name}"),
+ axum::routing::get(
+ move || async move { ([(header::CONTENT_TYPE, "text/plain")], payload) },
+ ),
+ );
+ }
+ fixture = fixture.route(
+ "/parent/{kind}",
+ axum::routing::get(
+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {
+ let target = if kind == "data" {
+ format!(
+ "data:text/plain;base64,{}",
+ base64::Engine::encode(&BASE64_STANDARD, LITERAL)
+ )
+ } else {
+ format!("/{kind}")
+ };
+ (
+ [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+ format!("<iframe src='{target}'></iframe>"),
+ )
+ },
+ ),
+ );
+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "inherited-text-encoding");
+ let app = build_router(test_state());
+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+ let session_id = session["value"]["sessionId"].as_str().unwrap();
+ for (name, payload) in cases {
+ classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/url"),
+ json!({"url":format!("http://{addr}/parent/{name}")}),
+ )
+ .await;
+ let parent = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/execute/sync"),
+ json!({"script": "return document.characterSet;", "args": []}),
+ )
+ .await;
+ assert_eq!(parent["value"], json!("UTF-8"), "{name}");
+ classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/frame"),
+ json!({"id": 0}),
+ )
+ .await;
+ let observed = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/execute/sync"),
+ json!({
+ "script": "return [document.body.textContent,document.characterSet,document.contentType];",
+ "args": [],
+ }),
+ ).await;
+ assert_eq!(
+ observed["value"],
+ json!([payload, "UTF-8", "text/plain"]),
+ "{name}"
+ );
+ classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/frame"),
+ json!({"id": null}),
+ )
+ .await;
+ }
+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
+#[tokio::test]
+async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_parser_types() {
+ let cases = [
+ ("atom.xml", "application/atom+xml"),
+ ("rss.xml", "application/rss+xml"),
+ ("custom.xml", "text/example+xml"),
+ ];
+ let mut fixture = axum::Router::new();
+ for (name, mime) in cases {
+ fixture = fixture.route(
+ &format!("/{name}"),
+ axum::routing::get(move || async move {
+ (
+ [(header::CONTENT_TYPE, mime)],
+ "<feed xmlns='http://www.w3.org/2005/Atom'><title>吴姓</title></feed>",
+ )
+ }),
+ );
+ }
+ fixture = fixture.route(
+ "/parent/{kind}",
+ axum::routing::get(
+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {
+ (
+ [(header::CONTENT_TYPE, "text/html; charset=windows-1252")],
+ format!("<iframe src='/{kind}'></iframe>"),
+ )
+ },
+ ),
+ );
+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-suffix-navigation");
+ let app = build_router(test_state());
+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+ let session_id = session["value"]["sessionId"].as_str().unwrap();
+ for (name, mime) in cases {
+ for prefix in ["", "parent/"] {
+ classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/url"),
+ json!({"url":format!("http://{addr}/{prefix}{name}")}),
+ )
+ .await;
+ let observed = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/execute/sync"),
+ json!({
+ "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; let rejected=false; try { new DOMParser().parseFromString('<feed/>',d.contentType); } catch(error) { rejected=error instanceof TypeError; } const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet,rejected];",
+ "args": [],
+ }),
+ ).await;
+ assert_eq!(
+ observed["value"],
+ json!([
+ "feed",
+ "http://www.w3.org/2005/Atom",
+ "吴姓",
+ mime,
+ "UTF-8",
+ true
+ ]),
+ "{prefix}{name}"
+ );
+ }
+ }
+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
#[tokio::test]
async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
const PAYLOAD: &str =
diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs
index 0f9674b25..d44b8a737 100644
--- a/moli-protocol/src/conn/runtime_load.rs
+++ b/moli-protocol/src/conn/runtime_load.rs
@@ -158,7 +158,7 @@ async fn prepare_network_error_page_navigation_with_engine_async(
fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {
moli_web_mime::response_document_content_type(headers)
- .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime))
+ .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))
}
#[derive(Clone, Debug)]
diff --git a/moli-renderer-v8/src/document_response_decoder.rs b/moli-renderer-v8/src/document_response_decoder.rs
new file mode 100644
index 000000000..5d291b90e
--- /dev/null
+++ b/moli-renderer-v8/src/document_response_decoder.rs
@@ -0,0 +1,77 @@
+use moli_encoding::HtmlDocumentStreamingDecoder;
+use moli_encoding_detector::detect_legacy_html_encoding;
+use moli_web_mime::{is_json_document_mime, is_text_document_mime, is_xml_document_mime};
+use url::Url;
+
+pub(crate) fn new_document_response_decoder(
+ headers: &[(String, Vec<u8>)],
+ content_type: Option<&str>,
+ final_url: &Url,
+ inherited_encoding: Option<&str>,
+) -> HtmlDocumentStreamingDecoder {
+ if let Some(mime) = content_type.filter(|mime| is_text_document_mime(mime)) {
+ HtmlDocumentStreamingDecoder::new_text_document(
+ headers,
+ final_url.as_str(),
+ detect_legacy_html_encoding,
+ is_json_document_mime(mime),
+ inherited_encoding,
+ )
+ } else if content_type.is_some_and(is_xml_document_mime) {
+ // XML declarations still apply; neither parent encoding nor heuristics
+ // can replace XML's UTF-8 default.
+ HtmlDocumentStreamingDecoder::new_with_fallback(headers, Some("UTF-8"))
+ } else if inherited_encoding.is_some() {
+ HtmlDocumentStreamingDecoder::new_with_fallback(headers, inherited_encoding)
+ } else {
+ HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ headers,
+ final_url.as_str(),
+ detect_legacy_html_encoding,
+ )
+ }
+}
+
+pub(crate) fn decode_document_response(
+ bytes: &[u8],
+ headers: &[(String, Vec<u8>)],
+ content_type: Option<&str>,
+ final_url: &Url,
+ inherited_encoding: Option<&str>,
+) -> (String, &'static str) {
+ let mut decoder =
+ new_document_response_decoder(headers, content_type, final_url, inherited_encoding);
+ let mut text = decoder.push(bytes).concat();
+ if let Some(tail) = decoder.finish() {
+ text.push_str(&tail);
+ }
+ (text, decoder.document_encoding_name())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn json_and_xml_responses_keep_utf8_defaults_with_a_legacy_parent() {
+ for (mime, source) in [
+ ("application/json", "{\"text\":\"吴姓\"}"),
+ ("Text/JSON", "{\"text\":\"吴姓\"}"),
+ ("application/problem+json", "{\"text\":\"吴姓\"}"),
+ ("application/atom+xml", "<feed>吴姓</feed>"),
+ ("application/rss+xml", "<feed>吴姓</feed>"),
+ ("text/example+xml", "<feed>吴姓</feed>"),
+ ] {
+ let headers = [("Content-Type".to_owned(), mime.as_bytes().to_vec())];
+ let (decoded, encoding) = decode_document_response(
+ source.as_bytes(),
+ &headers,
+ Some(mime),
+ &Url::parse("https://example.test/document").unwrap(),
+ Some("GBK"),
+ );
+ assert_eq!(decoded, source, "{mime}");
+ assert_eq!(encoding, "UTF-8", "{mime}");
+ }
+ }
+}
diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs
index 0dfa341eb..d764b479d 100644
--- a/moli-renderer-v8/src/dom_parser.rs
+++ b/moli-renderer-v8/src/dom_parser.rs
@@ -1,5 +1,5 @@
use crate::web_api_interfaces;
-use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime};
+use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime, is_xml_document_mime};
use moli_webapi_declare::WebApiFunctionTemplate;
use url::Url;
@@ -379,7 +379,7 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>(
) -> std::borrow::Cow<'a, str> {
if source == "\u{feff}"
&& !content_type.is_some_and(|mime| {
- is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)
+ is_xml_document_mime(mime) || moli_web_mime::is_text_document_mime(mime)
})
{
std::borrow::Cow::Borrowed("<body>\u{feff}</body>")
@@ -394,7 +394,7 @@ fn parse_browsing_context_document_snapshot(
content_type: Option<&str>,
html_parser: HtmlParser,
) -> (DomHost, DetachedDocumentKind) {
- if content_type.is_some_and(is_dom_parser_xml_mime)
+ if content_type.is_some_and(is_xml_document_mime)
|| (content_type.is_none() && child_document_url_is_xml_like(&document_url))
{
let parser = XmlParser;
@@ -585,6 +585,34 @@ mod tests {
assert!(disabled.element_handle_by_id("fallback").is_some());
}
+ #[test]
+ fn child_projection_preserves_xml_suffix_document_structure_and_namespace() {
+ let source = "<feed xmlns='http://www.w3.org/2005/Atom'><title>Example</title></feed>";
+ for mime in [
+ "application/atom+xml",
+ "application/rss+xml",
+ "text/example+xml",
+ ] {
+ for path in ["feed.xml", "feed.html", "feed"] {
+ let (document, kind) = parse_browsing_context_document_snapshot(
+ Url::parse(&format!("https://example.test/{path}")).unwrap(),
+ source,
+ Some(mime),
+ HtmlParser::SCRIPTING_ENABLED,
+ );
+ let root = first_document_element(&document);
+ let element = document
+ .node(root)
+ .and_then(|node| node.as_element())
+ .unwrap();
+ assert_eq!(kind, DetachedDocumentKind::Xml, "{mime} at {path}");
+ assert_eq!(element.local_name(), "feed", "{mime} at {path}");
+ assert_eq!(element.namespace(), "http://www.w3.org/2005/Atom");
+ assert_eq!(document.text_content(root).as_deref(), Some("Example"));
+ }
+ }
+ }
+
#[test]
fn child_projection_explicit_mime_overrides_xml_extension() {
let source = "\n<&amp;</pre><script>window.executed=1</script>";
diff --git a/moli-renderer-v8/src/lib.rs b/moli-renderer-v8/src/lib.rs
index 4f6db92af..40a233921 100644
--- a/moli-renderer-v8/src/lib.rs
+++ b/moli-renderer-v8/src/lib.rs
@@ -48,6 +48,7 @@ mod document_cookie_owner;
mod document_language;
mod document_last_modified;
mod document_module_graph;
+mod document_response_decoder;
mod document_runtime;
mod document_script_scheduler;
mod document_task_lane;
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
index 1d4e2d9bc..f8016efb8 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
@@ -6,7 +6,7 @@ use crate::frame_owner_model::{
FrameDocumentLocalWindowTransition, FrameLocalWindowOwnerTransition,
};
use crate::{document_runtime::DomHandle, frame_owner_model::FrameDocumentOwnerTransition};
-use moli_web_mime::is_dom_parser_xml_mime;
+use moli_web_mime::is_xml_document_mime;
use url::Url;
pub(crate) struct ChildDocumentInstallResult {
@@ -135,7 +135,7 @@ impl JsContextHost {
let is_xml_document = snapshot
.content_type
.as_deref()
- .is_some_and(is_dom_parser_xml_mime);
+ .is_some_and(is_xml_document_mime);
self.install_live_child_document_from_snapshot(
scope,
handle,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
index d429de742..90fac5d67 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
@@ -10,6 +10,7 @@ use crate::{
content_security_policy::{
ContentSecurityPolicyViolationEventFields, send_content_security_policy_reports,
},
+ document_response_decoder::decode_document_response,
document_runtime::{DocumentNavigationEmbeddingContext, DocumentPolicyContainer, DomHandle},
document_script_scheduler::FrameDocumentClassicScriptSchedulerWork,
frame_owner_model::{
@@ -21,8 +22,6 @@ use crate::{
LoadedChildDocument, SubresourceResponseBody,
},
};
-use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};
-use moli_encoding_detector::detect_legacy_html_encoding;
pub(crate) struct AppliedChildDocumentLoadCompletion {
/// Initial parser-classic work produced by the committed child document.
@@ -757,17 +756,6 @@ impl JsContextHost {
}
}
-fn child_document_fallback_character_set(
- parent_character_set: &str,
- content_type: Option<&str>,
-) -> String {
- if content_type.is_some_and(moli_web_mime::is_dom_parser_xml_mime) {
- "UTF-8".to_owned()
- } else {
- parent_character_set.to_owned()
- }
-}
-
fn child_document_load_outcome_from_response(
request_url: String,
request_method: String,
@@ -786,30 +774,17 @@ fn child_document_load_outcome_from_response(
let encoded_data_length = response_body.len();
let content_type = child_document_content_type_from_headers(&head.headers)
.or_else(|| child_document_content_type_for_url(&head.final_url));
- let fallback =
- child_document_fallback_character_set(parent_character_set, content_type.as_deref());
let (markup, character_set) = {
let body_bytes = response_body
.try_bytes()
.map_err(|error| format!("failed to read child document response body: {error}"))?;
- let (markup, character_set) = if let Some(mime) = content_type
- .as_deref()
- .filter(|mime| moli_web_mime::is_text_document_mime(mime))
- {
- let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
- &head.headers,
- head.final_url.as_str(),
- detect_legacy_html_encoding,
- moli_web_mime::is_json_module_mime(mime) || mime == "text/json",
- );
- let mut markup = decoder.push(&body_bytes).concat();
- if let Some(tail) = decoder.finish() {
- markup.push_str(&tail);
- }
- (markup, decoder.document_encoding_name())
- } else {
- decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))
- };
+ let (markup, character_set) = decode_document_response(
+ &body_bytes,
+ &head.headers,
+ content_type.as_deref(),
+ &head.final_url,
+ Some(parent_character_set),
+ );
(markup, character_set.to_owned())
};
let policy_container =
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs
index 71d318c35..20e3c344b 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/snapshots.rs
@@ -1,11 +1,10 @@
use super::super::{ChildBrowsingContextBootstrap, ChildBrowsingContextSnapshot, JsContextHost};
use super::configure_child_document_navigation_request;
+use crate::document_response_decoder::decode_document_response;
use crate::document_runtime::{DocumentPolicyContainer, DomHandle};
-use moli_encoding::decode_html_document_with_fallback;
use moli_fetch::Request;
use moli_web_mime::{
- is_dom_parser_xml_mime, is_html_document_mime, resource_mime_essence_for_url,
- response_document_content_type,
+ is_html_document_mime, resource_mime_essence_for_url, response_document_content_type,
};
use url::Url;
@@ -52,7 +51,7 @@ impl JsContextHost {
if !mime_type.is_empty() && !is_html_document_mime(&mime_type) {
return None;
}
- let character_set = self.child_document_fallback_character_set(Some("text/html"));
+ let character_set = self.document_character_set().to_owned();
Some(ChildBrowsingContextSnapshot::with_character_set(
url.clone(),
body,
@@ -64,11 +63,12 @@ impl JsContextHost {
"data" => crate::network_host::local_url_response(url).map(|response| {
let head = response.head();
let content_type = child_document_content_type_from_headers(&head.headers);
- let fallback = self.child_document_fallback_character_set(content_type.as_deref());
- let (body, character_set) = decode_html_document_with_fallback(
+ let (body, character_set) = decode_document_response(
response.body_bytes(),
&head.headers,
- Some(&fallback),
+ content_type.as_deref(),
+ &head.final_url,
+ Some(self.document_character_set()),
);
let policy_container = DocumentPolicyContainer::from_navigation_response_headers(
&head.headers,
@@ -116,11 +116,12 @@ impl JsContextHost {
let head = response.head();
let content_type = child_document_content_type_from_headers(&head.headers)
.or_else(|| child_document_content_type_for_url(&head.final_url));
- let fallback = self.child_document_fallback_character_set(content_type.as_deref());
- let (markup, character_set) = decode_html_document_with_fallback(
+ let (markup, character_set) = decode_document_response(
response.body_bytes(),
&head.headers,
- Some(&fallback),
+ content_type.as_deref(),
+ &head.final_url,
+ Some(self.document_character_set()),
);
let policy_container = DocumentPolicyContainer::from_navigation_response_headers(
&head.headers,
@@ -170,17 +171,6 @@ impl JsContextHost {
ChildBrowsingContextBootstrap::Request(_) => None,
}
}
-
- pub(in crate::native_bridge::context_host) fn child_document_fallback_character_set(
- &self,
- content_type: Option<&str>,
- ) -> String {
- if content_type.is_some_and(is_dom_parser_xml_mime) {
- "UTF-8".to_owned()
- } else {
- self.document_character_set().to_owned()
- }
- }
}
pub(in crate::native_bridge::context_host) fn child_document_content_type_from_headers(
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs
index 10c829ac1..1a84f1a04 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_frames/discovery.rs
@@ -238,7 +238,7 @@ impl JsContextHost {
return false;
}
moli_web_mime::is_html_document_mime(&mime)
- || moli_web_mime::is_dom_parser_xml_mime(&mime)
+ || moli_web_mime::is_xml_document_mime(&mime)
|| moli_web_mime::is_text_mime(&mime)
|| moli_web_mime::is_json_module_mime(&mime)
|| moli_web_mime::is_javascript_mime(&mime)
diff --git a/moli-renderer-v8/src/native_bridge/context_host/popups.rs b/moli-renderer-v8/src/native_bridge/context_host/popups.rs
index 5204a4d92..4f3e187a2 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/popups.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/popups.rs
@@ -2310,7 +2310,7 @@ impl JsContextHost {
super::child_documents::child_document_content_type_from_headers(&head.headers);
let fallback = if content_type
.as_deref()
- .is_some_and(moli_web_mime::is_dom_parser_xml_mime)
+ .is_some_and(moli_web_mime::is_xml_document_mime)
{
"UTF-8".to_owned()
} else {
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index fa8df43b8..f8c9351b9 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -3,8 +3,8 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_pen
use super::parser_blocking_pending::main_parser_blocking_classic_script_item;
use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;
use super::*;
+use crate::document_response_decoder::new_document_response_decoder;
use moli_encoding::HtmlDocumentStreamingDecoder;
-use moli_encoding_detector::detect_legacy_html_encoding;
use moli_web_mime::response_headers_indicate_attachment_download;
use tokio::sync::{mpsc, oneshot};
@@ -471,7 +471,7 @@ impl ConcurrentParseTimeRuntime {
fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {
moli_web_mime::response_document_content_type(headers)
- .is_some_and(|mime| moli_web_mime::is_dom_parser_xml_mime(&mime))
+ .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))
}
const EXTERNAL_RAW_DOCUMENT_BODY_BUFFERED_CHUNKS: usize = 8;
@@ -615,8 +615,10 @@ fn response_document_parser(
env: &PageVmEnvConfig,
) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {
let content_type = moli_web_mime::response_document_content_type(headers);
- let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));
- let mut state = if let Some(mime) = &text_type {
+ let text_type = content_type
+ .as_deref()
+ .filter(|mime| moli_web_mime::is_text_document_mime(mime));
+ let mut state = if let Some(mime) = text_type {
ParseTimeDriverState::new_text(final_url, mime)
} else {
ParseTimeDriverState::new_with_scripting_enabled(
@@ -624,20 +626,8 @@ fn response_document_parser(
main_document_parser_scripting_enabled(env),
)
};
- let decoder = if let Some(mime) = text_type {
- HtmlDocumentStreamingDecoder::new_text_document(
- headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",
- )
- } else {
- HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
- headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- )
- };
+ let decoder =
+ new_document_response_decoder(headers, content_type.as_deref(), &state.final_url, None);
// Headers/defaults are observable even when no body chunk is ready yet.
// A later BOM or decoder decision can still replace this tentative value.
sync_state_document_character_set_from_decoder(&mut state, &decoder);
diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs
index 35fca10b5..9574cdde2 100644
--- a/moli-web-mime/src/classification.rs
+++ b/moli-web-mime/src/classification.rs
@@ -13,6 +13,14 @@ pub fn is_dom_parser_xml_mime(input: &str) -> bool {
})
}
+/// XML navigation responses, including registered and custom `+xml` subtypes.
+/// DOMParser's Web API accepts a narrower, separately classified set of types.
+pub fn is_xml_document_mime(input: &str) -> bool {
+ mime_essence(input).is_some_and(|mime| {
+ matches!(mime.as_str(), "text/xml" | "application/xml") || mime.ends_with("+xml")
+ })
+}
+
pub fn is_javascript_mime(input: &str) -> bool {
mime_essence(input).is_some_and(|mime| is_javascript_mime_essence(&mime))
}
@@ -55,7 +63,7 @@ pub fn is_text_mime_essence(input: &str) -> bool {
/// This also includes JSON and JavaScript resources opened as documents.
pub fn is_text_document_mime(input: &str) -> bool {
!is_html_document_mime(input)
- && !is_dom_parser_xml_mime(input)
+ && !is_xml_document_mime(input)
&& (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))
}
@@ -143,6 +151,12 @@ pub fn is_json_module_mime(input: &str) -> bool {
mime_essence(input).is_some_and(|mime| mime == "application/json" || mime.ends_with("+json"))
}
+pub fn is_json_document_mime(input: &str) -> bool {
+ mime_essence(input).is_some_and(|mime| {
+ matches!(mime.as_str(), "application/json" | "text/json") || mime.ends_with("+json")
+ })
+}
+
pub fn is_webassembly_mime(input: &str) -> bool {
mime_essence(input).is_some_and(|mime| mime == "application/wasm")
}
diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs
index 806eb2455..bcae3a9eb 100644
--- a/moli-web-mime/src/lib.rs
+++ b/moli-web-mime/src/lib.rs
@@ -13,10 +13,11 @@ pub use classification::{
is_css_style_element_type_attribute, is_css_stylesheet_type_hint, is_dom_parser_xml_mime,
is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,
is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,
- is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,
- is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,
- is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,
- is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,
+ is_json_document_mime, is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime,
+ is_png_image_mime_essence, is_supported_document_mime_type, is_svg_image_mime,
+ is_svg_image_mime_essence, is_text_document_mime, is_text_mime, is_text_mime_essence,
+ is_video_mime, is_video_mime_essence, is_webassembly_mime, is_xml_document_mime,
+ multipart_form_data_boundary,
};
pub use data_url::{
data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,
diff --git a/moli-web-mime/src/sniffing.rs b/moli-web-mime/src/sniffing.rs
index 5b21532b6..28f0d6342 100644
--- a/moli-web-mime/src/sniffing.rs
+++ b/moli-web-mime/src/sniffing.rs
@@ -1,4 +1,4 @@
-use crate::classification::{is_dom_parser_xml_mime, is_html_document_mime};
+use crate::classification::{is_html_document_mime, is_xml_document_mime};
use crate::parse::mime_essence;
pub const RESOURCE_HEADER_BYTE_LIMIT: usize = 1445;
@@ -212,7 +212,7 @@ fn computed_browsing_mime_type(
}
fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String {
- if supplied.is_some_and(is_xml_mime) {
+ if supplied.is_some_and(is_xml_document_mime) {
return supplied.expect("checked supplied MIME").into();
}
sniff_image_mime_type(header)
@@ -222,7 +222,7 @@ fn computed_image_mime_type(supplied: Option<&str>, header: &[u8]) -> String {
}
fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> String {
- if supplied.is_some_and(is_xml_mime) {
+ if supplied.is_some_and(is_xml_document_mime) {
return supplied.expect("checked supplied MIME").into();
}
sniff_audio_video_mime_type(header)
@@ -232,7 +232,7 @@ fn computed_audio_video_mime_type(supplied: Option<&str>, header: &[u8]) -> Stri
}
fn computed_font_mime_type(supplied: Option<&str>, header: &[u8]) -> String {
- if supplied.is_some_and(is_xml_mime) {
+ if supplied.is_some_and(is_xml_document_mime) {
return supplied.expect("checked supplied MIME").into();
}
sniff_font_mime_type(header)
@@ -424,11 +424,7 @@ fn is_binary_data_byte(byte: u8) -> bool {
}
fn is_xml_or_html_mime(mime: &str) -> bool {
- is_xml_mime(mime) || is_html_mime(mime)
-}
-
-fn is_xml_mime(mime: &str) -> bool {
- is_dom_parser_xml_mime(mime) || mime.ends_with("+xml")
+ is_xml_document_mime(mime) || is_html_mime(mime)
}
fn is_html_mime(mime: &str) -> bool {
diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs
index d29adb69b..d361b05e0 100644
--- a/moli-web-mime/src/tests.rs
+++ b/moli-web-mime/src/tests.rs
@@ -886,9 +886,53 @@ fn document_text_classification_excludes_html_and_xml() {
"text/xml",
"application/xhtml+xml",
"image/svg+xml",
+ "application/atom+xml",
+ "application/rss+xml",
+ "text/example+xml; charset=utf-8",
"image/png",
"application/octet-stream",
] {
assert!(!crate::is_text_document_mime(mime), "{mime}");
}
}
+
+#[test]
+fn navigation_xml_mime_classification_keeps_dom_parser_allowlist_separate() {
+ for mime in [
+ "text/xml",
+ "application/xml",
+ "application/xhtml+xml",
+ "image/svg+xml",
+ ] {
+ assert!(is_xml_document_mime(mime), "{mime}");
+ assert!(is_dom_parser_xml_mime(mime), "{mime}");
+ }
+ for mime in [
+ "Application/Atom+XML; charset=utf-8",
+ "application/rss+xml",
+ "text/example+xml",
+ ] {
+ assert!(is_xml_document_mime(mime), "{mime}");
+ assert!(!is_dom_parser_xml_mime(mime), "{mime}");
+ assert!(!is_text_document_mime(mime), "{mime}");
+ }
+ for mime in [
+ "text/html",
+ "text/plain",
+ "application/xml-dtd",
+ "invalid+xml",
+ ] {
+ assert!(!is_xml_document_mime(mime), "{mime}");
+ }
+}
+
+#[test]
+fn navigation_json_mime_includes_text_json_without_changing_module_mime() {
+ for mime in ["application/json", "application/problem+json"] {
+ assert!(is_json_document_mime(mime), "{mime}");
+ assert!(is_json_module_mime(mime), "{mime}");
+ }
+ assert!(is_json_document_mime("Text/JSON; charset=utf-8"));
+ assert!(!is_json_module_mime("Text/JSON; charset=utf-8"));
+ assert!(!is_json_document_mime("text/plain"));
+}