Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(document): parse text responses with consistent decoding (#740)

mini llkcba79021aa4b
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index 88df60445..3c14f5109 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder {
url_hint: Option<String>,
decoder: Option<Decoder>,
selected_encoding: Option<&'static Encoding>,
+ sniff_html_declarations: bool,
}
impl HtmlDocumentStreamingDecoder {
+ /// Decode a text document without interpreting literal HTML/XML declarations.
+ /// JSON defaults to UTF-8; other text retains the configured legacy detector.
+ pub fn new_text_document(
+ headers: &[(String, String)],
+ url_hint: &str,
+ detector: LegacyEncodingDetector,
+ json: bool,
+ ) -> Self {
+ let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));
+ decoder.sniff_html_declarations = false;
+ if json && decoder.transport_encoding.is_none() {
+ decoder.transport_encoding = Some(encoding_rs::UTF_8);
+ }
+ decoder
+ }
+
pub fn new(headers: &[(String, String)]) -> Self {
Self::new_with_options(headers, None, None, None)
}
@@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder {
url_hint: url_hint.map(str::to_owned),
decoder: None,
selected_encoding: None,
+ sniff_html_declarations: true,
}
}
@@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder {
pub fn document_encoding_name(&self) -> &'static str {
self.selected_encoding_name()
+ .or_else(|| self.transport_encoding.map(Encoding::name))
.unwrap_or(self.fallback_encoding.name())
}
@@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder {
if let Some(encoding) = self.transport_encoding {
return Some(encoding);
}
+ if !self.sniff_html_declarations {
+ return (finishing || self.sniff_buffer.len() >= 1024).then(|| {
+ self.detected_legacy_content_encoding()
+ .unwrap_or(self.fallback_encoding)
+ });
+ }
if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {
return Some(encoding);
}
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 2b4863a84..d9717f32d 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector(
panic!("declared encoding must take precedence over heuristic detection")
}
+#[test]
+fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
+ let input = "{\"name\":\"Gülçek\"}";
+ for split in 0..=input.len() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ true,
+ );
+ let mut output = decoder.push(&input.as_bytes()[..split]).join("");
+ output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, input, "split {split}");
+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+ }
+}
+
+#[test]
+fn plain_text_document_honors_transport_charset() {
+ let headers = [(
+ "Content-Type".to_owned(),
+ "text/plain; charset=gbk".to_owned(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &headers,
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "<b>家居</b>");
+ assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
+}
+
+#[test]
+fn text_document_bom_precedes_transport_charset() {
+ let headers = [(
+ "Content-Type".to_owned(),
+ "text/plain; charset=gbk".to_owned(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &headers,
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ false,
+ );
+ assert_eq!(decoder.document_encoding_name(), "GBK");
+ assert_eq!(decoder.selected_encoding_name(), None);
+ assert!(decoder.push(&[0xef]).is_empty());
+ assert!(decoder.push(&[0xbb]).is_empty());
+ let mut tail = vec![0xbf];
+ tail.extend_from_slice("Gülçek".as_bytes());
+ let mut output = decoder.push(&tail).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "Gülçek");
+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+ assert_eq!(decoder.document_encoding_name(), "UTF-8");
+}
+
+#[test]
+fn plain_text_document_retains_legacy_detection_without_a_charset() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://legacy.example/",
+ test_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&[0x80]).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "А");
+ assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
+}
+
+#[test]
+fn plain_text_document_ignores_literal_encoding_declarations() {
+ for prefix in [
+ "<meta charset=utf-8>",
+ "<?xml version='1.0' encoding='utf-8'?>",
+ ] {
+ let mut input = prefix.as_bytes().to_vec();
+ input.push(0xe9);
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://example.test/",
+ test_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&input).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, format!("{prefix}é"));
+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
+ }
+}
+
#[test]
fn content_type_charset_is_selected() {
let headers = vec![(
diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs
index ce351cf5e..17369a9b5 100644
--- a/moli-parser/src/html.rs
+++ b/moli-parser/src/html.rs
@@ -27,7 +27,7 @@ use super::live_target::{
ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer,
ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks,
ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream,
- new_parser_stream_html_tree_sink_stream,
+ new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target,
};
use super::{
ParserSourcePosition, html_chunks,
@@ -438,6 +438,47 @@ impl HtmlParser {
DocumentStream::new_parser_stream(final_url, self.scripting_enabled)
}
+ /// Start a text document using the HTML tokenizer's plaintext state.
+ /// Subsequent chunks remain literal text inside the browser-owned `pre`.
+ pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream {
+ let stream =
+ DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url));
+ stream.inner.initialize_text_document();
+ let mut host = stream.inner.take_parser_stream_dom_host();
+ let document_handle = host.document_handle();
+ host.set_document_content_type_for_handle(document_handle, content_type);
+ stream.inner.restore_parser_stream_dom_host(host);
+ stream
+ }
+
+ /// Initialize the same literal-text shell in an existing document. The
+ /// embedder owns that document's MIME metadata and live DOM callbacks.
+ pub fn start_live_text_document_root<T>(
+ &self,
+ final_url: Url,
+ document_handle: NativeNodeId,
+ consumer: &mut T,
+ ) -> DocumentStream
+ where
+ T: ParserDomReadConsumer
+ + ParserDomMutationConsumer
+ + ParserMutationEffectConsumer
+ + ParserElementCreationConsumer,
+ {
+ let stream = DocumentStream::new_text_document(
+ ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle),
+ );
+ // SAFETY: the consumer is exclusively borrowed until the step guard
+ // removes the erased callbacks, including when initialization unwinds.
+ let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) };
+ stream.inner.enter_runtime_dom_sinks_parse_step(sinks);
+ {
+ let step = RuntimeDomSinksParserStep { stream: &stream };
+ step.stream.inner.initialize_text_document();
+ }
+ stream
+ }
+
pub fn start_live_document_root(
&self,
final_url: Url,
@@ -551,6 +592,17 @@ impl Default for ParserInputQueue {
}
impl DocumentStream {
+ fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self {
+ let mut options = html_parse_opts_with_scripting(false);
+ // The byte decoder has already consumed the transport BOM. In plaintext
+ // every remaining U+FEFF is content, including at a chunk boundary.
+ options.tokenizer.discard_bom = false;
+ Self {
+ inner: HtmlTreeSinkStream::from_target_with_options(target, options),
+ input: RefCell::default(),
+ }
+ }
+
fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self {
Self {
inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled),
@@ -1843,6 +1895,83 @@ mod tests {
const HTML_NS: &str = "http://www.w3.org/1999/xhtml";
const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML";
+ #[test]
+ fn text_document_stream_preserves_literal_markup_and_leading_newline() {
+ let payload =
+ "\u{feff}\n<b>Gülçek</b>&amp;<script>window.executed=1</script></pre>\u{feff}";
+ for content_type in ["text/plain", "application/json", "application/problem+json"] {
+ let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document(
+ Url::parse("https://example.test/data").unwrap(),
+ content_type,
+ );
+ for character in payload.chars() {
+ stream.feed(&character.to_string());
+ }
+ let document = stream.finish();
+ let pre = first_element_by_ns(&document, HTML_NS, "pre");
+ assert_eq!(document.text_content(pre).as_deref(), Some(payload));
+ assert_eq!(document.document().unwrap().content_type(), content_type);
+ for tag in ["b", "script"] {
+ assert!(
+ document
+ .elements_by_tag_name_ns(
+ document.document_node_id(),
+ Some(HTML_NS),
+ tag,
+ true
+ )
+ .is_empty()
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn text_document_shell_has_no_doctype_and_preserves_normalized_literal_text() {
+ for mime in [
+ "text/plain",
+ "application/json",
+ "application/problem+json",
+ "text/javascript",
+ ] {
+ let stream = HtmlParser::SCRIPTING_ENABLED
+ .start_text_document(Url::parse("https://example.test/data").unwrap(), mime);
+ for chunk in [
+ "\n<&amp;",
+ "\r",
+ "\nbeta\rgamma\0",
+ "</pre><script>bad()</script>",
+ ] {
+ stream.feed(chunk);
+ }
+ let document = stream.finish_dom_host();
+ let root = document.document_handle();
+ let children = document.child_handles(root).collect::<Vec<_>>();
+ assert_eq!(children.len(), 1, "no synthetic doctype: {mime}");
+ assert!(document.is_html_element_named(children[0], "html"));
+ let expected_mode = HtmlParser::SCRIPTING_ENABLED.parse_dom_host(
+ Url::parse("https://example.test/control").unwrap(),
+ "<!doctype html><p>control".to_owned(),
+ );
+ assert_eq!(
+ document.document_quirks_mode_for_handle(root),
+ expected_mode.document_quirks_mode_for_handle(expected_mode.document_handle())
+ );
+ assert_eq!(
+ document.text_content(root).as_deref(),
+ Some("\n<&amp;\nbeta\ngamma\u{fffd}</pre><script>bad()</script>")
+ );
+ }
+ }
+
+ #[test]
+ fn html_document_still_parses_markup_and_entities() {
+ let document = parse_test_document("<b>Gülçek&amp;</b>");
+ let bold = first_element_by_ns(&document, HTML_NS, "b");
+ assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&"));
+ assert_eq!(document.document().unwrap().content_type(), "text/html");
+ }
+
fn parse_test_document(html: &str) -> NativeDom {
HtmlParser::SCRIPTING_ENABLED.parse(
Url::parse("https://example.test/").expect("test url"),
diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs
index 3025906a4..2b4dfef9a 100644
--- a/moli-parser/src/session.rs
+++ b/moli-parser/src/session.rs
@@ -167,6 +167,19 @@ impl TokenSink for EmbedderPausingTreeBuilder {
}
impl HtmlParserSession {
+ pub(super) fn initialize_text_document(&self) {
+ // Parse only the browser-owned shell. Response bytes enter the tokenizer
+ // after it has switched to plaintext, so tags and entities stay literal.
+ self.process(StrTendril::from(concat!(
+ "<html><head></head><body>",
+ "<pre style=\"word-wrap: break-word; white-space: pre-wrap;\">\n"
+ )));
+ // Text documents have no doctype but always use no-quirks mode.
+ self.sink()
+ .set_quirks_mode(html5ever::tree_builder::QuirksMode::NoQuirks);
+ self.tokenizer.set_plaintext_state();
+ }
+
fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
Self {
@@ -352,10 +365,10 @@ fn feed_with_definitive_encoding(
pub(super) fn new_html_tree_sink_session(
target: ParserStreamHtmlTreeSinkTarget,
- scripting_enabled: bool,
+ options: ParseOpts,
) -> HtmlTreeSinkSession {
let sink = DocumentSink::new(target);
- let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+ let parser = HtmlParserSession::new(sink, options);
let script_input = ParserInputQueue::default();
HtmlTreeSinkSession {
diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs
index fa14f3b82..70d3a31d3 100644
--- a/moli-parser/src/stream.rs
+++ b/moli-parser/src/stream.rs
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
}
impl HtmlTreeSinkStream {
+ pub(super) fn initialize_text_document(&self) {
+ self.parser.initialize_text_document();
+ }
+
pub(super) fn from_target_with_scripting(
target: ParserStreamHtmlTreeSinkTarget,
scripting_enabled: bool,
) -> Self {
- let session = new_html_tree_sink_session(target, scripting_enabled);
+ Self::from_target_with_options(
+ target,
+ super::session::html_parse_opts_with_scripting(scripting_enabled),
+ )
+ }
+
+ pub(super) fn from_target_with_options(
+ target: ParserStreamHtmlTreeSinkTarget,
+ options: html5ever::ParseOpts,
+ ) -> Self {
+ let session = new_html_tree_sink_session(target, options);
Self {
parser: session.parser,
script_input: session.script_input,
diff --git a/moli-protocol-server/src/protocol_server/tests/classic.rs b/moli-protocol-server/src/protocol_server/tests/classic.rs
index 69beb69ef..68d1c92ac 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic.rs
@@ -3,6 +3,129 @@ use axum::http::{HeaderMap, header};
use base64::engine::general_purpose::STANDARD as BASE64_STANDARD;
use serde_json::{Map, Value};
+#[tokio::test]
+async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
+ const PAYLOAD: &str =
+ "<meta charset='gbk'><script>window.executed=42;</script><b>literal&amp;Gülçek</b>";
+ let cases = [
+ (
+ "plain",
+ "text/plain; charset=utf-8",
+ "text/plain",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "plain.xml",
+ "text/plain; charset=utf-8",
+ "text/plain",
+ true,
+ "\n<&amp;\r\nbeta\rgamma\0</pre><script>window.executed=42</script>",
+ ),
+ (
+ "json.xml",
+ "application/problem+json",
+ "application/problem+json",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "plain-bom",
+ "text/plain; charset=utf-8",
+ "text/plain",
+ true,
+ "\u{feff}\u{feff}",
+ ),
+ (
+ "json",
+ "application/json",
+ "application/json",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "javascript",
+ "text/javascript; charset=utf-8",
+ "text/javascript",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "html",
+ "text/html; charset=utf-8",
+ "text/html",
+ false,
+ PAYLOAD,
+ ),
+ ];
+ let mut fixture = axum::Router::new();
+ for (name, mime, _, _, payload) in cases {
+ fixture = fixture.route(
+ &format!("/{name}"),
+ axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),
+ );
+ }
+ fixture = fixture.route(
+ "/frame/{kind}",
+ axum::routing::get(
+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {
+ (
+ [(header::CONTENT_TYPE, "text/html")],
+ format!("<iframe src='/{kind}'></iframe>"),
+ )
+ },
+ ),
+ );
+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");
+ let app = build_router(test_state());
+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+ let session_id = session["value"]["sessionId"].as_str().unwrap();
+ for (name, _, content_type, literal, payload) in cases {
+ for prefix in ["", "frame/"] {
+ let navigated = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/url"),
+ json!({"url":format!("http://{addr}/{prefix}{name}")}),
+ )
+ .await;
+ assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");
+ let observed = classic_request_json_with_body(
+ app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
+ json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),
+ ).await;
+ let expected = if literal {
+ json!([
+ payload
+ .strip_prefix('\u{feff}')
+ .unwrap_or(payload)
+ .replace("\r\n", "\n")
+ .replace('\r', "\n")
+ .replace('\0', "\u{fffd}"),
+ null,
+ 0,
+ content_type
+ ])
+ } else {
+ json!(["literal&Gülçek", 42, 1, content_type])
+ };
+ assert_eq!(observed["value"], expected, "{prefix}{name}");
+ if literal {
+ let structure = classic_request_json_with_body(
+ app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
+ json!({"script": "const d=(document.querySelector('iframe')?.contentWindow ?? window).document; return [d.doctype===null,d.compatMode,d.body.children.length,d.body.firstElementChild.localName,d.querySelectorAll('pre').length];", "args": []}),
+ ).await;
+ assert_eq!(
+ structure["value"],
+ json!([true, "CSS1Compat", 1, "pre", 1]),
+ "{prefix}{name}"
+ );
+ }
+ }
+ }
+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
#[tokio::test]
async fn webdriver_classic_status_session_and_delete_routes_use_value_envelope() {
let app = build_router(test_state());
diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rs
index d8a83d17c..76ac1936a 100644
--- a/moli-protocol/src/domains/emulation/tests.rs
+++ b/moli-protocol/src/domains/emulation/tests.rs
@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()
listener,
Router::new().route(
"/",
- get(|| async { "<!doctype html><title>idle navigation</title>" }),
+ get(|| async {
+ axum::response::Html("<!doctype html><title>idle navigation</title>")
+ }),
),
)
.await
@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {
listener,
Router::new().route(
"/",
- get(|| async { "<!doctype html><title>idle detector</title>" }),
+ get(|| async {
+ axum::response::Html("<!doctype html><title>idle detector</title>")
+ }),
),
)
.await
@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()
#[tokio::test(flavor = "multi_thread")]
async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {
async fn page_handler() -> impl IntoResponse {
- "<!doctype html><html><body>ok</body></html>"
+ axum::response::Html("<!doctype html><html><body>ok</body></html>")
}
async fn xhr_handler(
@@ -2711,9 +2715,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2767,9 +2771,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-user-agent=\"{user_agent}\" data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.userAgent, Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rs
index 450912689..ea7c311b9 100644
--- a/moli-protocol/src/domains/page/tests/lifecycle.rs
+++ b/moli-protocol/src/domains/page/tests/lifecycle.rs
@@ -2391,11 +2391,15 @@ async fn network_navigations_use_unique_document_loader_ids() {
axum::Router::new()
.route(
"/first",
- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>first</main>")
+ }),
)
.route(
"/second",
- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>second</main>")
+ }),
),
)
.await
@@ -2474,11 +2478,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() {
axum::Router::new()
.route(
"/first",
- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>first</main>")
+ }),
)
.route(
"/second",
- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>second</main>")
+ }),
),
)
.await
diff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rs
index ab651a209..1a1e48497 100644
--- a/moli-protocol/src/domains/page/tests/runtime.rs
+++ b/moli-protocol/src/domains/page/tests/runtime.rs
@@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li
.route(
"/main",
axum::routing::get(|| async {
- "<iframe id='child' name='child-frame' src='/initial'></iframe>"
+ axum::response::Html(
+ "<iframe id='child' name='child-frame' src='/initial'></iframe>",
+ )
}),
)
.route(
"/initial",
- axum::routing::get(|| async { "<body>initial</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>initial</body>") }),
)
.route(
"/history-a",
- axum::routing::get(|| async { "<body>history-a</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>history-a</body>") }),
)
.route(
"/history-b",
- axum::routing::get(|| async { "<body>history-b</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>history-b</body>") }),
),
)
.await
diff --git a/moli-protocol/src/domains/target/tests/tests_background_staging.rs b/moli-protocol/src/domains/target/tests/tests_background_staging.rs
index e52b86f54..49bebc255 100644
--- a/moli-protocol/src/domains/target/tests/tests_background_staging.rs
+++ b/moli-protocol/src/domains/target/tests/tests_background_staging.rs
@@ -3037,9 +3037,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.language, document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -3204,7 +3204,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
#[tokio::test(flavor = "multi_thread")]
async fn same_context_background_session_can_clear_its_own_timezone_before_activation() {
async fn handler() -> impl IntoResponse {
- "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>"
+ axum::response::Html(
+ "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>",
+ )
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs
index f1703887c..9c50732d2 100644
--- a/moli-renderer-v8/src/dom_parser.rs
+++ b/moli-renderer-v8/src/dom_parser.rs
@@ -1,5 +1,4 @@
use crate::web_api_interfaces;
-use html5ever::tree_builder::QuirksMode;
use moli_web_mime::{is_dom_parser_xml_mime, is_html_document_mime};
use moli_webapi_declare::WebApiFunctionTemplate;
use url::Url;
@@ -367,7 +366,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>(
source: &'a str,
content_type: Option<&str>,
) -> std::borrow::Cow<'a, str> {
- if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) {
+ if source == "\u{feff}"
+ && !content_type.is_some_and(|mime| {
+ is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)
+ })
+ {
std::borrow::Cow::Borrowed("<body>\u{feff}</body>")
} else {
std::borrow::Cow::Borrowed(source)
@@ -381,7 +384,7 @@ fn parse_browsing_context_document_snapshot(
html_parser: HtmlParser,
) -> (DomHost, DetachedDocumentKind) {
if content_type.is_some_and(is_dom_parser_xml_mime)
- || child_document_url_is_xml_like(&document_url)
+ || (content_type.is_none() && child_document_url_is_xml_like(&document_url))
{
let parser = XmlParser;
return (
@@ -389,13 +392,13 @@ fn parse_browsing_context_document_snapshot(
DetachedDocumentKind::Xml,
);
}
- if content_type.is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain")) {
- let mut document =
- html_parser.parse_dom_host(document_url, plain_text_document_parser_input(source));
- // Text documents are HTML Documents whose mode is explicitly no-quirks,
- // despite having no doctype that would select that mode through parsing.
- document.set_html_quirks_mode_for_parser(QuirksMode::NoQuirks);
- return (document, DetachedDocumentKind::Html);
+ if let Some(content_type) =
+ content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime))
+ {
+ let stream = HtmlParser::with_scripting_enabled(false)
+ .start_text_document(document_url, content_type);
+ stream.feed(source);
+ return (stream.finish_dom_host(), DetachedDocumentKind::Html);
}
(
html_parser.parse_dom_host(document_url, source.to_owned()),
@@ -403,22 +406,6 @@ fn parse_browsing_context_document_snapshot(
)
}
-pub(crate) fn plain_text_document_parser_input(source: &str) -> String {
- let mut input = String::with_capacity(source.len().saturating_add(64));
- // The HTML parser discards the first LF after <pre>; preserve any source LF.
- input.push_str("<html><head></head><body><pre>\n");
- for character in source.chars() {
- match character {
- '&' => input.push_str("&amp;"),
- '<' => input.push_str("&lt;"),
- '\0' => input.push('\u{fffd}'),
- _ => input.push(character),
- }
- }
- input.push_str("</pre></body></html>");
- input
-}
-
fn child_document_url_is_xml_like(url: &Url) -> bool {
let path = url.path().to_ascii_lowercase();
path.ends_with(".xml") || path.ends_with(".xhtml") || path.ends_with(".svg")
@@ -599,6 +586,50 @@ mod tests {
assert!(disabled.element_handle_by_id("fallback").is_some());
}
+ #[test]
+ fn child_projection_explicit_mime_overrides_xml_extension() {
+ let source = "\n<&amp;</pre><script>window.executed=1</script>";
+ for mime in [
+ "text/plain",
+ "application/json",
+ "application/problem+json",
+ "text/javascript",
+ ] {
+ let (document, kind) = parse_browsing_context_document_snapshot(
+ Url::parse("https://example.test/data.xml").unwrap(),
+ source,
+ Some(mime),
+ HtmlParser::SCRIPTING_ENABLED,
+ );
+ let root = document.document_handle();
+ assert_eq!(kind, DetachedDocumentKind::Html);
+ assert_eq!(
+ document.text_content(root).as_deref(),
+ Some(source),
+ "{mime}"
+ );
+ let children = document.child_handles(root).collect::<Vec<_>>();
+ assert_eq!(children.len(), 1);
+ assert!(document.is_html_element_named(children[0], "html"));
+ assert_eq!(
+ document.document_quirks_mode_for_handle(root),
+ Some(selectors::matching::QuirksMode::NoQuirks)
+ );
+ }
+ let (document, kind) = parse_browsing_context_document_snapshot(
+ Url::parse("https://example.test/page.svg").unwrap(),
+ "<!doctype html><b id='parsed'>&amp;</b>",
+ Some("text/html"),
+ HtmlParser::SCRIPTING_ENABLED,
+ );
+ assert_eq!(kind, DetachedDocumentKind::Html);
+ assert!(document.element_handle_by_id("parsed").is_some());
+ assert_eq!(
+ document.text_content(document.document_handle()).as_deref(),
+ Some("&")
+ );
+ }
+
#[test]
fn child_plain_text_document_uses_pre_and_no_quirks_mode() {
let (document, kind) = parse_browsing_context_document_snapshot(
diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rs
index f8af39bc9..b67274975 100644
--- a/moli-renderer-v8/src/live_document_parser.rs
+++ b/moli-renderer-v8/src/live_document_parser.rs
@@ -717,6 +717,14 @@ impl DocumentParserSession {
)
}
+ pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self {
+ Self::new_html(
+ HtmlParser::with_scripting_enabled(false)
+ .start_text_document(document_url, content_type),
+ DocumentParserLifetime::Finite,
+ )
+ }
+
pub(crate) fn start_finite_live_document(
document_url: Url,
document_handle: NativeNodeId,
@@ -729,6 +737,21 @@ impl DocumentParserSession {
)
}
+ pub(crate) fn start_finite_live_text_document(
+ document_url: Url,
+ document_handle: NativeNodeId,
+ owner: &mut impl LiveDocumentParserOwner,
+ ) -> Self {
+ Self::new_html(
+ HtmlParser::with_scripting_enabled(false).start_live_text_document_root(
+ document_url,
+ document_handle,
+ owner,
+ ),
+ DocumentParserLifetime::Finite,
+ )
+ }
+
pub(crate) fn start_finite_live_xml_document(
document_url: Url,
document_handle: NativeNodeId,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
index 67f2387ab..38d747418 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
@@ -170,16 +170,8 @@ impl JsContextHost {
navigation_loader: Option<crate::network::navigation::NavigationResourceLoader>,
is_xml_document: bool,
) -> Option<ChildDocumentInstallResult> {
- let is_plain_text_document = snapshot
- .content_type
- .as_deref()
- .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain"));
let source = if is_xml_document {
std::borrow::Cow::Borrowed(snapshot.markup.as_str())
- } else if is_plain_text_document {
- std::borrow::Cow::Owned(crate::dom_parser::plain_text_document_parser_input(
- &snapshot.markup,
- ))
} else {
crate::dom_parser::preserve_decoded_bom_only_browsing_context_body(
&snapshot.markup,
@@ -239,6 +231,7 @@ impl JsContextHost {
parser_base_url,
source.as_ref(),
is_xml_document,
+ snapshot.content_type.as_deref(),
);
Some(ChildDocumentInstallResult {
initial_classic_ready_work: parser_start.initial_classic_ready_work,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
index 42ad8071a..bc842f8c7 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
@@ -1002,14 +1002,6 @@ impl JsContextHost {
let mut owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);
parser.finish(&mut owner)
};
- if self
- .dom_host()
- .document_content_type_for_handle(document_handle)
- .is_some_and(|mime| mime.eq_ignore_ascii_case("text/plain"))
- {
- self.dom_host_mut()
- .set_html_quirks_mode_for_parser_document(document_handle, QuirksMode::NoQuirks);
- }
self.queue_live_child_parser_discovery_signals(
child_handle,
document_handle,
@@ -1695,6 +1687,7 @@ impl JsContextHost {
document_base_url: Url,
markup: &str,
is_xml_document: bool,
+ content_type: Option<&str>,
) -> ChildLiveDocumentParserStartResult {
let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id);
self.child_document_parsers.clear(owner);
@@ -1703,6 +1696,13 @@ impl JsContextHost {
document_base_url,
document_handle,
)
+ } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) {
+ let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);
+ DocumentParserSession::start_finite_live_text_document(
+ document_base_url,
+ document_handle,
+ &mut parser_owner,
+ )
} else {
DocumentParserSession::start_finite_live_document(
document_base_url,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
index fdd143742..35acc157d 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
@@ -21,7 +21,8 @@ use crate::{
LoadedChildDocument, SubresourceResponseBody,
},
};
-use moli_encoding::decode_html_document_with_fallback;
+use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};
+use moli_encoding_detector::detect_legacy_html_encoding;
pub(crate) struct AppliedChildDocumentLoadCompletion {
/// Initial parser-classic work produced by the committed child document.
@@ -762,8 +763,24 @@ fn child_document_load_outcome_from_response(
let body_bytes = response_body
.try_bytes()
.map_err(|error| format!("failed to read child document response body: {error}"))?;
- let (markup, character_set) =
- decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback));
+ let (markup, character_set) = if let Some(mime) = content_type
+ .as_deref()
+ .filter(|mime| moli_web_mime::is_text_document_mime(mime))
+ {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &head.headers,
+ head.final_url.as_str(),
+ detect_legacy_html_encoding,
+ moli_web_mime::is_json_module_mime(mime) || mime == "text/json",
+ );
+ let mut markup = decoder.push(&body_bytes).concat();
+ if let Some(tail) = decoder.finish() {
+ markup.push_str(&tail);
+ }
+ (markup, decoder.document_encoding_name())
+ } else {
+ decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))
+ };
(markup, character_set.to_owned())
};
let policy_container =
diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
index 5af62b821..454433e6c 100644
--- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
@@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime {
let (page_vm, triggered_navigation) = {
let buffered_document_preloads = &mut state.buffered_document_preloads;
let service_worker_preload_context = state.service_worker_preload_context.as_ref();
+ let document_character_set = &state.document_character_set;
PageVm::new_from_parser_stream_and_run_document_start(
page_id,
local_executor,
@@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime {
&mut state.parser_session,
started,
|page_vm| {
+ page_vm.set_document_character_set(document_character_set.clone());
admit_pending_preloads(
page_vm,
buffered_document_preloads,
diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs
index 1a4986fc8..8866cb1aa 100644
--- a/moli-renderer-v8/src/runtime/phase_one/state.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/state.rs
@@ -11,6 +11,7 @@ pub(super) struct ParseTimeDriverState {
pub(super) buffered_document_preloads: Box<BufferedDocumentPreloadState>,
pub(super) service_worker_preload_context: Option<ServiceWorkerScriptPreloadContext>,
pub(super) input_closed: bool,
+ pub(super) is_text_document: bool,
}
impl ParseTimeDriverState {
@@ -32,6 +33,24 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
+ is_text_document: false,
+ }
+ }
+
+ pub(super) fn new_text(final_url: Url, mime: &str) -> Self {
+ Self {
+ parser_session: DocumentParserSession::start_main_text_document(
+ final_url.clone(),
+ mime,
+ ),
+ final_url,
+ document_character_set: "UTF-8".to_owned(),
+ scheduler: DocumentScriptScheduler::new(),
+ pending_parsing_blocking_script: PendingParsingBlockingClassicScriptRunner::empty(),
+ buffered_document_preloads: Box::default(),
+ service_worker_preload_context: None,
+ input_closed: false,
+ is_text_document: true,
}
}
@@ -45,6 +64,7 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
+ is_text_document: false,
}
}
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index 15a5d6dfd..28159400d 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime {
env.apply_navigation_response_headers(&response_final_url, &response_headers);
let mut body_source = RawDocumentBodySource::fetch_response(response);
- let mut state = ParseTimeDriverState::new_with_scripting_enabled(
- response_final_url,
- main_document_parser_scripting_enabled(&env),
- );
+ let (mut state, mut decoder) =
+ response_document_parser(response_final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
- &response_headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- );
// Raw navigation bodies are decoded during prebootstrap scan. The decoder
// is carried forward so split multibyte sequences are not decoded twice or
// lost between the scan and parser handoff.
@@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime {
},
)));
}
- let mut state = ParseTimeDriverState::new_with_scripting_enabled(
- final_url,
- main_document_parser_scripting_enabled(&env),
- );
+ let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
- &response_headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- );
// External raw bodies replay captured chunks. Pre-scan only what is already
// buffered before bootstrap so the producer's backpressure boundary still
// controls how far ahead the parser can get.
@@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime {
chunk: String,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
+ if self.state.is_text_document {
+ self.state.parser_session.queue_arrived_chunk(chunk);
+ return;
+ }
self.state
.buffered_document_preloads
.append_to_main_document_scan_with_service_worker_context(
@@ -628,6 +617,9 @@ fn scan_prebootstrap_html_chunk_into_state(
chunk: &str,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
+ if state.is_text_document {
+ return;
+ }
state
.buffered_document_preloads
.append_to_main_document_prebootstrap_scan_with_service_worker_context(
@@ -638,6 +630,41 @@ fn scan_prebootstrap_html_chunk_into_state(
);
}
+fn response_document_parser(
+ final_url: Url,
+ headers: &[(String, String)],
+ env: &PageVmEnvConfig,
+) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {
+ let content_type = moli_web_mime::response_document_content_type(headers);
+ let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));
+ let mut state = if let Some(mime) = &text_type {
+ ParseTimeDriverState::new_text(final_url, mime)
+ } else {
+ ParseTimeDriverState::new_with_scripting_enabled(
+ final_url,
+ main_document_parser_scripting_enabled(env),
+ )
+ };
+ let decoder = if let Some(mime) = text_type {
+ HtmlDocumentStreamingDecoder::new_text_document(
+ headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",
+ )
+ } else {
+ HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ )
+ };
+ // Headers/defaults are observable even when no body chunk is ready yet.
+ // A later BOM or decoder decision can still replace this tentative value.
+ sync_state_document_character_set_from_decoder(&mut state, &decoder);
+ (state, decoder)
+}
+
pub(super) fn enqueue_streaming_raw_chunk(
runtime: &mut ConcurrentParseTimeRuntime,
decoder: &mut HtmlDocumentStreamingDecoder,
@@ -929,6 +956,105 @@ mod tests {
use crate::parser::ScriptSource;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ #[tokio::test]
+ async fn text_document_charset_is_visible_to_document_start_before_body_arrives() {
+ let _js_runtime_owner = crate::JsRuntime::initialize();
+ tokio::task::LocalSet::new().run_until(async {
+ for (mime, chunk, expected) in [
+ ("text/plain; charset=gbk", None, "GBK"),
+ ("application/json", None, "UTF-8"),
+ ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"),
+ ] {
+ let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap();
+ let loader = loader_owner.handle();
+ let mut env = default_test_page_vm_env_config();
+ env.document_start_scripts.push(crate::DocumentStartScript {
+ registry_key: None,
+ devtools_session: None,
+ source: "document.title = document.characterSet".to_owned(),
+ world_name: None,
+ has_bidi_channel_argument: false,
+ bidi_channel_handoffs: Vec::new(),
+ });
+ let (mut state, mut decoder) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.to_owned())], &env,
+ );
+ if let Some(chunk) = chunk {
+ let chunks = decoder.push(chunk);
+ sync_state_document_character_set_from_decoder(&mut state, &decoder);
+ for chunk in chunks {
+ state.parser_session.queue_arrived_chunk(chunk);
+ }
+ }
+ assert_eq!(state.document_character_set, expected);
+ let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task(
+ PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env,
+ PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state,
+ Instant::now(), "charset test bootstrap closed",
+ ).await.unwrap();
+ assert!(!navigated);
+ assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}");
+ }
+ }).await;
+ }
+
+ #[test]
+ fn response_document_dispatch_distinguishes_text_from_html_and_xml() {
+ for (mime, text) in [
+ ("application/json", true),
+ ("application/problem+json", true),
+ ("text/plain; charset=gbk", true),
+ ("text/javascript", true),
+ ("text/html", false),
+ ("application/xhtml+xml", false),
+ ("application/xml", false),
+ ("text/xml", false),
+ ] {
+ let (state, _) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.to_owned())],
+ &default_test_page_vm_env_config(),
+ );
+ assert_eq!(state.is_text_document, text, "{mime}");
+ }
+ }
+
+ #[tokio::test]
+ async fn text_document_does_not_preload_literal_script_tags() {
+ let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default())
+ .expect("default loader");
+ let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test();
+ for (mime, expected_preload) in [
+ ("text/plain", false),
+ ("application/json", false),
+ ("text/html", true),
+ ] {
+ let (mut state, _) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.to_owned())],
+ &default_test_page_vm_env_config(),
+ );
+ state.buffered_document_preloads.bind_resource_runtime(
+ runtime_hooks.owner_wake(),
+ runtime_hooks.resource_task_runner(),
+ );
+ scan_prebootstrap_html_chunk_into_state(
+ &mut state,
+ &loader,
+ "<script defer src='/literal.js'></script>",
+ None,
+ );
+ assert_eq!(
+ state.buffered_document_preloads.entries.contains_key(
+ &classic_preload_key_for_streaming_test("https://example.test/literal.js")
+ ),
+ expected_preload,
+ "{mime}",
+ );
+ }
+ }
+
#[tokio::test]
async fn external_raw_document_body_source_collects_chunks_and_completion() {
let (completion_tx, completion_rx) = oneshot::channel();
diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs
index 3ff3a0dd5..35fca10b5 100644
--- a/moli-web-mime/src/classification.rs
+++ b/moli-web-mime/src/classification.rs
@@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool {
input.starts_with("text/")
}
+/// Navigation responses rendered as literal text rather than HTML or XML.
+/// This also includes JSON and JavaScript resources opened as documents.
+pub fn is_text_document_mime(input: &str) -> bool {
+ !is_html_document_mime(input)
+ && !is_dom_parser_xml_mime(input)
+ && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))
+}
+
/// Whether a `style` element's raw `type` attribute selects classic CSS.
///
/// HTML's update-a-style-block algorithm compares the untrimmed attribute
diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs
index 615e882d2..ec09da8be 100644
--- a/moli-web-mime/src/lib.rs
+++ b/moli-web-mime/src/lib.rs
@@ -14,9 +14,9 @@ pub use classification::{
is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,
is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,
is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,
- is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime,
- is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime,
- multipart_form_data_boundary,
+ is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,
+ is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,
+ is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,
};
pub use data_url::{
data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,
diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs
index e475520a9..a21ab2a60 100644
--- a/moli-web-mime/src/tests.rs
+++ b/moli-web-mime/src/tests.rs
@@ -648,3 +648,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() {
));
assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[]));
}
+#[test]
+fn document_text_classification_excludes_html_and_xml() {
+ for mime in [
+ "text/plain",
+ "Text/Css; charset=utf-8",
+ "application/json",
+ "application/ld+json",
+ "application/javascript",
+ ] {
+ assert!(crate::is_text_document_mime(mime), "{mime}");
+ }
+ for mime in [
+ "text/html",
+ "text/xml",
+ "application/xhtml+xml",
+ "image/svg+xml",
+ "image/png",
+ "application/octet-stream",
+ ] {
+ assert!(!crate::is_text_document_mime(mime), "{mime}");
+ }
+}
diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs
index a1599c989..e9ed136c6 100644
--- a/moli/src/fetch_dump/tests.rs
+++ b/moli/src/fetch_dump/tests.rs
@@ -1,7 +1,7 @@
use std::sync::Arc;
use anyhow::Result;
-use axum::{Router, routing::get};
+use axum::{Router, response::Html, routing::get};
use moli_core::{
LayoutPolicy,
page::Page,
@@ -36,7 +36,7 @@ async fn load_page_with_config(
"/",
get(move || {
let body = Arc::clone(&server_body);
- async move { (*body).clone() }
+ async move { Html((*body).clone()) }
}),
);
axum::serve(listener, app).await.unwrap();