Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(parser): preserve text document MIME and encoding semantics

lanyue-llk9d236a20ef09
diff --git a/moli-encoding/src/document.rs b/moli-encoding/src/document.rs
index 1ceb79b9f..8b3a70398 100644
--- a/moli-encoding/src/document.rs
+++ b/moli-encoding/src/document.rs
@@ -47,9 +47,26 @@ pub struct HtmlDocumentStreamingDecoder {
url_hint: Option<String>,
decoder: Option<Decoder>,
selected_encoding: Option<&'static Encoding>,
+ sniff_html_declarations: bool,
}
impl HtmlDocumentStreamingDecoder {
+ /// Decode a text document without interpreting literal HTML/XML declarations.
+ /// JSON defaults to UTF-8; other text retains the configured legacy detector.
+ pub fn new_text_document(
+ headers: &[(String, Vec<u8>)],
+ url_hint: &str,
+ detector: LegacyEncodingDetector,
+ json: bool,
+ ) -> Self {
+ let mut decoder = Self::new_with_options(headers, None, Some(url_hint), Some(detector));
+ decoder.sniff_html_declarations = false;
+ if json && decoder.transport_encoding.is_none() {
+ decoder.transport_encoding = Some(encoding_rs::UTF_8);
+ }
+ decoder
+ }
+
pub fn new(headers: &[(String, Vec<u8>)]) -> Self {
Self::new_with_options(headers, None, None, None)
}
@@ -88,6 +105,7 @@ impl HtmlDocumentStreamingDecoder {
url_hint: url_hint.map(str::to_owned),
decoder: None,
selected_encoding: None,
+ sniff_html_declarations: true,
}
}
@@ -97,6 +115,7 @@ impl HtmlDocumentStreamingDecoder {
pub fn document_encoding_name(&self) -> &'static str {
self.selected_encoding_name()
+ .or_else(|| self.transport_encoding.map(Encoding::name))
.unwrap_or(self.fallback_encoding.name())
}
@@ -149,6 +168,12 @@ impl HtmlDocumentStreamingDecoder {
if let Some(encoding) = self.transport_encoding {
return Some(encoding);
}
+ if !self.sniff_html_declarations {
+ return (finishing || self.sniff_buffer.len() >= 1024).then(|| {
+ self.detected_legacy_content_encoding()
+ .unwrap_or(self.fallback_encoding)
+ });
+ }
if let Some(encoding) = encoding_for_document_utf16_xml_prefix(&self.sniff_buffer) {
return Some(encoding);
}
diff --git a/moli-encoding/src/tests.rs b/moli-encoding/src/tests.rs
index 6693d5d54..2217e0521 100644
--- a/moli-encoding/src/tests.rs
+++ b/moli-encoding/src/tests.rs
@@ -24,6 +24,102 @@ fn unexpected_legacy_encoding_detector(
panic!("declared encoding must take precedence over heuristic detection")
}
+#[test]
+fn json_text_document_defaults_to_utf8_across_every_chunk_split() {
+ let input = "{\"name\":\"Gülçek\"}";
+ for split in 0..=input.len() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ true,
+ );
+ let mut output = decoder.push(&input.as_bytes()[..split]).join("");
+ output.push_str(&decoder.push(&input.as_bytes()[split..]).join(""));
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, input, "split {split}");
+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+ }
+}
+
+#[test]
+fn plain_text_document_honors_transport_charset() {
+ let headers = [(
+ "Content-Type".to_owned(),
+ b"text/plain; charset=gbk".to_vec(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &headers,
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&gbk_bytes("<b>家居</b>")).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "<b>家居</b>");
+ assert_eq!(decoder.selected_encoding_name(), Some("GBK"));
+}
+
+#[test]
+fn text_document_bom_precedes_transport_charset() {
+ let headers = [(
+ "Content-Type".to_owned(),
+ b"text/plain; charset=gbk".to_vec(),
+ )];
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &headers,
+ "https://example.test/",
+ unexpected_legacy_encoding_detector,
+ false,
+ );
+ assert_eq!(decoder.document_encoding_name(), "GBK");
+ assert_eq!(decoder.selected_encoding_name(), None);
+ assert!(decoder.push(&[0xef]).is_empty());
+ assert!(decoder.push(&[0xbb]).is_empty());
+ let mut tail = vec![0xbf];
+ tail.extend_from_slice("Gülçek".as_bytes());
+ let mut output = decoder.push(&tail).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "Gülçek");
+ assert_eq!(decoder.selected_encoding_name(), Some("UTF-8"));
+ assert_eq!(decoder.document_encoding_name(), "UTF-8");
+}
+
+#[test]
+fn plain_text_document_retains_legacy_detection_without_a_charset() {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://legacy.example/",
+ test_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&[0x80]).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, "А");
+ assert_eq!(decoder.selected_encoding_name(), Some("IBM866"));
+}
+
+#[test]
+fn plain_text_document_ignores_literal_encoding_declarations() {
+ for prefix in [
+ "<meta charset=utf-8>",
+ "<?xml version='1.0' encoding='utf-8'?>",
+ ] {
+ let mut input = prefix.as_bytes().to_vec();
+ input.push(0xe9);
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &[],
+ "https://example.test/",
+ test_legacy_encoding_detector,
+ false,
+ );
+ let mut output = decoder.push(&input).join("");
+ output.push_str(&decoder.finish().unwrap_or_default());
+ assert_eq!(output, format!("{prefix}é"));
+ assert_eq!(decoder.selected_encoding_name(), Some("windows-1252"));
+ }
+}
+
#[test]
fn content_type_charset_is_selected() {
let headers: Vec<(String, Vec<u8>)> = vec![(
diff --git a/moli-parser/src/html.rs b/moli-parser/src/html.rs
index e585f55cf..0d44d8831 100644
--- a/moli-parser/src/html.rs
+++ b/moli-parser/src/html.rs
@@ -27,7 +27,7 @@ use super::live_target::{
ParserDomMutationConsumer, ParserDomReadConsumer, ParserElementCreationConsumer,
ParserMutationEffectConsumer, ParserMutationEffectDelivery, ParserRuntimeDomSinks,
ParserStreamHtmlTreeSinkTarget, new_live_document_root_html_tree_sink_stream,
- new_parser_stream_html_tree_sink_stream,
+ new_parser_stream_html_tree_sink_stream, new_parser_stream_html_tree_sink_target,
};
use super::{
ParserSourcePosition, html_chunks,
@@ -439,6 +439,47 @@ impl HtmlParser {
DocumentStream::new_parser_stream(final_url, self.scripting_enabled)
}
+ /// Start a text document using the HTML tokenizer's plaintext state.
+ /// Subsequent chunks remain literal text inside the browser-owned `pre`.
+ pub fn start_text_document(&self, final_url: Url, content_type: &str) -> DocumentStream {
+ let stream =
+ DocumentStream::new_text_document(new_parser_stream_html_tree_sink_target(final_url));
+ stream.inner.initialize_text_document();
+ let mut host = stream.inner.take_parser_stream_dom_host();
+ let document_handle = host.document_handle();
+ host.set_document_content_type_for_handle(document_handle, content_type);
+ stream.inner.restore_parser_stream_dom_host(host);
+ stream
+ }
+
+ /// Initialize the same literal-text shell in an existing document. The
+ /// embedder owns that document's MIME metadata and live DOM callbacks.
+ pub fn start_live_text_document_root<T>(
+ &self,
+ final_url: Url,
+ document_handle: NativeNodeId,
+ consumer: &mut T,
+ ) -> DocumentStream
+ where
+ T: ParserDomReadConsumer
+ + ParserDomMutationConsumer
+ + ParserMutationEffectConsumer
+ + ParserElementCreationConsumer,
+ {
+ let stream = DocumentStream::new_text_document(
+ ParserStreamHtmlTreeSinkTarget::new_live_document_root(final_url, document_handle),
+ );
+ // SAFETY: the consumer is exclusively borrowed until the step guard
+ // removes the erased callbacks, including when initialization unwinds.
+ let sinks = unsafe { ParserRuntimeDomSinks::from_consumer(consumer) };
+ stream.inner.enter_runtime_dom_sinks_parse_step(sinks);
+ {
+ let step = RuntimeDomSinksParserStep { stream: &stream };
+ step.stream.inner.initialize_text_document();
+ }
+ stream
+ }
+
pub fn start_live_document_root(
&self,
final_url: Url,
@@ -552,6 +593,17 @@ impl Default for ParserInputQueue {
}
impl DocumentStream {
+ fn new_text_document(target: ParserStreamHtmlTreeSinkTarget) -> Self {
+ let mut options = html_parse_opts_with_scripting(false);
+ // The byte decoder has already consumed the transport BOM. In plaintext
+ // every remaining U+FEFF is content, including at a chunk boundary.
+ options.tokenizer.discard_bom = false;
+ Self {
+ inner: HtmlTreeSinkStream::from_target_with_options(target, options),
+ input: RefCell::default(),
+ }
+ }
+
fn new_parser_stream(final_url: Url, scripting_enabled: bool) -> Self {
Self {
inner: new_parser_stream_html_tree_sink_stream(final_url, scripting_enabled),
@@ -1854,6 +1906,45 @@ mod tests {
const HTML_NS: &str = "http://www.w3.org/1999/xhtml";
const MATHML_NS: &str = "http://www.w3.org/1998/Math/MathML";
+ #[test]
+ fn text_document_stream_preserves_literal_markup_and_leading_newline() {
+ let payload =
+ "\u{feff}\n<b>Gülçek</b>&amp;<script>window.executed=1</script></pre>\u{feff}";
+ for content_type in ["text/plain", "application/json", "application/problem+json"] {
+ let stream = HtmlParser::SCRIPTING_ENABLED.start_text_document(
+ Url::parse("https://example.test/data").unwrap(),
+ content_type,
+ );
+ for character in payload.chars() {
+ stream.feed(&character.to_string());
+ }
+ let document = stream.finish();
+ let pre = first_element_by_ns(&document, HTML_NS, "pre");
+ assert_eq!(document.text_content(pre).as_deref(), Some(payload));
+ assert_eq!(document.document().unwrap().content_type(), content_type);
+ for tag in ["b", "script"] {
+ assert!(
+ document
+ .elements_by_tag_name_ns(
+ document.document_node_id(),
+ Some(HTML_NS),
+ tag,
+ true
+ )
+ .is_empty()
+ );
+ }
+ }
+ }
+
+ #[test]
+ fn html_document_still_parses_markup_and_entities() {
+ let document = parse_test_document("<b>Gülçek&amp;</b>");
+ let bold = first_element_by_ns(&document, HTML_NS, "b");
+ assert_eq!(document.text_content(bold).as_deref(), Some("Gülçek&"));
+ assert_eq!(document.document().unwrap().content_type(), "text/html");
+ }
+
fn parse_test_document(html: &str) -> NativeDom {
HtmlParser::SCRIPTING_ENABLED.parse(
Url::parse("https://example.test/").expect("test url"),
diff --git a/moli-parser/src/session.rs b/moli-parser/src/session.rs
index ad019567d..a38de08e5 100644
--- a/moli-parser/src/session.rs
+++ b/moli-parser/src/session.rs
@@ -232,6 +232,16 @@ impl TokenSink for EmbedderPausingTreeBuilder {
}
impl HtmlParserSession {
+ pub(super) fn initialize_text_document(&self) {
+ // Parse only the browser-owned shell. Response bytes enter the tokenizer
+ // after it has switched to plaintext, so tags and entities stay literal.
+ self.process(StrTendril::from(concat!(
+ "<!doctype html><html><head></head><body>",
+ "<pre style=\"word-wrap: break-word; white-space: pre-wrap;\">\n"
+ )));
+ self.tokenizer.set_plaintext_state();
+ }
+
fn new(sink: DocumentSink, opts: ParseOpts) -> Self {
let tree_builder = EmbedderPausingTreeBuilder::new(sink, opts.tree_builder);
Self {
@@ -418,10 +428,10 @@ fn feed_with_definitive_encoding(
pub(super) fn new_html_tree_sink_session(
target: ParserStreamHtmlTreeSinkTarget,
- scripting_enabled: bool,
+ options: ParseOpts,
) -> HtmlTreeSinkSession {
let sink = DocumentSink::new(target);
- let parser = HtmlParserSession::new(sink, html_parse_opts_with_scripting(scripting_enabled));
+ let parser = HtmlParserSession::new(sink, options);
let script_input = ParserInputQueue::default();
HtmlTreeSinkSession {
diff --git a/moli-parser/src/stream.rs b/moli-parser/src/stream.rs
index f74879e54..667788515 100644
--- a/moli-parser/src/stream.rs
+++ b/moli-parser/src/stream.rs
@@ -429,11 +429,25 @@ pub(crate) fn prepare_parser_script_handoff_for_static_document(
}
impl HtmlTreeSinkStream {
+ pub(super) fn initialize_text_document(&self) {
+ self.parser.initialize_text_document();
+ }
+
pub(super) fn from_target_with_scripting(
target: ParserStreamHtmlTreeSinkTarget,
scripting_enabled: bool,
) -> Self {
- let session = new_html_tree_sink_session(target, scripting_enabled);
+ Self::from_target_with_options(
+ target,
+ super::session::html_parse_opts_with_scripting(scripting_enabled),
+ )
+ }
+
+ pub(super) fn from_target_with_options(
+ target: ParserStreamHtmlTreeSinkTarget,
+ options: html5ever::ParseOpts,
+ ) -> Self {
+ let session = new_html_tree_sink_session(target, options);
Self {
parser: session.parser,
script_input: session.script_input,
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index 9f891a382..57c4abc15 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -1,5 +1,98 @@
use super::*;
+#[tokio::test]
+async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
+ const PAYLOAD: &str =
+ "<meta charset='gbk'><script>window.executed=42;</script><b>literal&amp;Gülçek</b>";
+ let cases = [
+ (
+ "plain",
+ "text/plain; charset=utf-8",
+ "text/plain",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "plain-bom",
+ "text/plain; charset=utf-8",
+ "text/plain",
+ true,
+ "\u{feff}\u{feff}",
+ ),
+ (
+ "json",
+ "application/json",
+ "application/json",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "javascript",
+ "text/javascript; charset=utf-8",
+ "text/javascript",
+ true,
+ PAYLOAD,
+ ),
+ (
+ "html",
+ "text/html; charset=utf-8",
+ "text/html",
+ false,
+ PAYLOAD,
+ ),
+ ];
+ let mut fixture = axum::Router::new();
+ for (name, mime, _, _, payload) in cases {
+ fixture = fixture.route(
+ &format!("/{name}"),
+ axum::routing::get(move || async move { ([(header::CONTENT_TYPE, mime)], payload) }),
+ );
+ }
+ fixture = fixture.route(
+ "/frame/{kind}",
+ axum::routing::get(
+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {
+ (
+ [(header::CONTENT_TYPE, "text/html")],
+ format!("<iframe src='/{kind}'></iframe>"),
+ )
+ },
+ ),
+ );
+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "document-mime");
+ let app = build_router(test_state());
+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+ let session_id = session["value"]["sessionId"].as_str().unwrap();
+ for (name, _, content_type, literal, payload) in cases {
+ for prefix in ["", "frame/"] {
+ let navigated = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/url"),
+ json!({"url":format!("http://{addr}/{prefix}{name}")}),
+ )
+ .await;
+ assert_eq!(navigated, json!({"value":null}), "{prefix}{name}");
+ let observed = classic_request_json_with_body(
+ app.clone(), Method::POST, &format!("/session/{session_id}/execute/sync"),
+ json!({"script": "const w=document.querySelector('iframe')?.contentWindow ?? window; return [w.document.body.textContent,w.executed??null,w.document.querySelectorAll('script').length,w.document.contentType];", "args": []}),
+ ).await;
+ let expected = if literal {
+ json!([
+ payload.strip_prefix('\u{feff}').unwrap_or(payload),
+ null,
+ 0,
+ content_type
+ ])
+ } else {
+ json!(["literal&Gülçek", 42, 1, content_type])
+ };
+ assert_eq!(observed["value"], expected, "{prefix}{name}");
+ }
+ }
+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
#[tokio::test]
async fn webdriver_classic_page_load_strategy_maps_navigation_wait_policy() {
let app = build_router(test_state());
diff --git a/moli-protocol/src/domains/emulation/tests.rs b/moli-protocol/src/domains/emulation/tests.rs
index dd3ad74ca..2b677b6fe 100644
--- a/moli-protocol/src/domains/emulation/tests.rs
+++ b/moli-protocol/src/domains/emulation/tests.rs
@@ -573,7 +573,9 @@ async fn admitted_idle_override_is_visible_to_concurrent_same_site_navigation()
listener,
Router::new().route(
"/",
- get(|| async { "<!doctype html><title>idle navigation</title>" }),
+ get(|| async {
+ axum::response::Html("<!doctype html><title>idle navigation</title>")
+ }),
),
)
.await
@@ -644,7 +646,9 @@ async fn idle_override_updates_idle_detector_and_clear_restores_actual_state() {
listener,
Router::new().route(
"/",
- get(|| async { "<!doctype html><title>idle detector</title>" }),
+ get(|| async {
+ axum::response::Html("<!doctype html><title>idle detector</title>")
+ }),
),
)
.await
@@ -2219,7 +2223,7 @@ async fn emulation_async_dispatch_updates_live_page_user_agent_and_xhr_header()
#[tokio::test(flavor = "multi_thread")]
async fn emulation_async_dispatch_updates_live_page_surface_without_mutating_accept_language() {
async fn page_handler() -> impl IntoResponse {
- "<!doctype html><html><body>ok</body></html>"
+ axum::response::Html("<!doctype html><html><body>ok</body></html>")
}
async fn xhr_handler(
@@ -2714,9 +2718,9 @@ async fn locale_override_updates_intl_without_mutating_language_surfaces() {
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -2770,9 +2774,9 @@ async fn bidi_user_context_locale_composes_with_user_agent_on_all_identity_surfa
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-user-agent=\"{user_agent}\" data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.userAgent, Intl.DateTimeFormat().resolvedOptions().locale, navigator.language, navigator.languages.join(','), document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-protocol/src/domains/page/tests/lifecycle.rs b/moli-protocol/src/domains/page/tests/lifecycle.rs
index 92e5947f8..afa29aece 100644
--- a/moli-protocol/src/domains/page/tests/lifecycle.rs
+++ b/moli-protocol/src/domains/page/tests/lifecycle.rs
@@ -2384,11 +2384,15 @@ async fn network_navigations_use_unique_document_loader_ids() {
axum::Router::new()
.route(
"/first",
- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>first</main>")
+ }),
)
.route(
"/second",
- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>second</main>")
+ }),
),
)
.await
@@ -2467,11 +2471,15 @@ async fn navigations_without_network_domain_still_use_unique_loader_ids() {
axum::Router::new()
.route(
"/first",
- axum::routing::get(|| async { "<!doctype html><main>first</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>first</main>")
+ }),
)
.route(
"/second",
- axum::routing::get(|| async { "<!doctype html><main>second</main>" }),
+ axum::routing::get(|| async {
+ axum::response::Html("<!doctype html><main>second</main>")
+ }),
),
)
.await
diff --git a/moli-protocol/src/domains/page/tests/runtime.rs b/moli-protocol/src/domains/page/tests/runtime.rs
index ab651a209..1a1e48497 100644
--- a/moli-protocol/src/domains/page/tests/runtime.rs
+++ b/moli-protocol/src/domains/page/tests/runtime.rs
@@ -1269,20 +1269,22 @@ async fn runtime_evaluate_child_history_back_emits_child_frame_navigation_and_li
.route(
"/main",
axum::routing::get(|| async {
- "<iframe id='child' name='child-frame' src='/initial'></iframe>"
+ axum::response::Html(
+ "<iframe id='child' name='child-frame' src='/initial'></iframe>",
+ )
}),
)
.route(
"/initial",
- axum::routing::get(|| async { "<body>initial</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>initial</body>") }),
)
.route(
"/history-a",
- axum::routing::get(|| async { "<body>history-a</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>history-a</body>") }),
)
.route(
"/history-b",
- axum::routing::get(|| async { "<body>history-b</body>" }),
+ axum::routing::get(|| async { axum::response::Html("<body>history-b</body>") }),
),
)
.await
diff --git a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs
index e0670cf8a..d5306df9c 100644
--- a/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs
+++ b/moli-protocol/src/domains/target/tests/tests_background_staging/environment.rs
@@ -284,9 +284,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
.get(axum::http::header::ACCEPT_LANGUAGE)
.and_then(|value| value.to_str().ok())
.unwrap_or("");
- format!(
+ axum::response::Html(format!(
"<!doctype html><html><body data-accept-language=\"{accept_language}\"><script>document.body.textContent = [navigator.language, document.body.dataset.acceptLanguage].join('|');</script></body></html>"
- )
+ ))
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
@@ -451,7 +451,9 @@ async fn same_context_background_session_can_clear_its_own_locale_before_activat
#[tokio::test(flavor = "multi_thread")]
async fn same_context_background_session_can_clear_its_own_timezone_before_activation() {
async fn handler() -> impl IntoResponse {
- "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>"
+ axum::response::Html(
+ "<!doctype html><html><body><script>document.body.textContent = Intl.DateTimeFormat().resolvedOptions().timeZone;</script></body></html>",
+ )
}
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
diff --git a/moli-renderer-v8/src/dom_parser.rs b/moli-renderer-v8/src/dom_parser.rs
index cff5ea1cc..e72deebc2 100644
--- a/moli-renderer-v8/src/dom_parser.rs
+++ b/moli-renderer-v8/src/dom_parser.rs
@@ -378,7 +378,11 @@ pub(crate) fn preserve_decoded_bom_only_browsing_context_body<'a>(
source: &'a str,
content_type: Option<&str>,
) -> std::borrow::Cow<'a, str> {
- if source == "\u{feff}" && !content_type.is_some_and(is_dom_parser_xml_mime) {
+ if source == "\u{feff}"
+ && !content_type.is_some_and(|mime| {
+ is_dom_parser_xml_mime(mime) || moli_web_mime::is_text_document_mime(mime)
+ })
+ {
std::borrow::Cow::Borrowed("<body>\u{feff}</body>")
} else {
std::borrow::Cow::Borrowed(source)
diff --git a/moli-renderer-v8/src/live_document_parser.rs b/moli-renderer-v8/src/live_document_parser.rs
index 1c513162c..e483965c8 100644
--- a/moli-renderer-v8/src/live_document_parser.rs
+++ b/moli-renderer-v8/src/live_document_parser.rs
@@ -757,6 +757,14 @@ impl DocumentParserSession {
)
}
+ pub(crate) fn start_main_text_document(document_url: Url, content_type: &str) -> Self {
+ Self::new_html(
+ HtmlParser::with_scripting_enabled(false)
+ .start_text_document(document_url, content_type),
+ DocumentParserLifetime::Finite,
+ )
+ }
+
pub(crate) fn start_finite_live_document(
document_url: Url,
document_handle: NativeNodeId,
@@ -769,6 +777,21 @@ impl DocumentParserSession {
)
}
+ pub(crate) fn start_finite_live_text_document(
+ document_url: Url,
+ document_handle: NativeNodeId,
+ owner: &mut impl LiveDocumentParserOwner,
+ ) -> Self {
+ Self::new_html(
+ HtmlParser::with_scripting_enabled(false).start_live_text_document_root(
+ document_url,
+ document_handle,
+ owner,
+ ),
+ DocumentParserLifetime::Finite,
+ )
+ }
+
pub(crate) fn start_finite_live_xml_document(
document_url: Url,
document_handle: NativeNodeId,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
index ee356deeb..a1d27afdb 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/commit.rs
@@ -239,6 +239,7 @@ impl JsContextHost {
parser_base_url,
source.as_ref(),
is_xml_document,
+ snapshot.content_type.as_deref(),
);
Some(ChildDocumentInstallResult {
initial_classic_ready_work: parser_start.initial_classic_ready_work,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
index 631bd47aa..fc8f2f3a4 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/live_parser.rs
@@ -1705,6 +1705,7 @@ impl JsContextHost {
document_base_url: Url,
markup: &str,
is_xml_document: bool,
+ content_type: Option<&str>,
) -> ChildLiveDocumentParserStartResult {
let owner = FrameDocumentOwner::new(owner_local_window_id, owner_document_id);
self.child_document_parsers.clear(owner);
@@ -1713,6 +1714,13 @@ impl JsContextHost {
document_base_url,
document_handle,
)
+ } else if content_type.is_some_and(moli_web_mime::is_text_document_mime) {
+ let mut parser_owner = ChildFrameLiveParserOwner::new(self, scope, document_handle);
+ DocumentParserSession::start_finite_live_text_document(
+ document_base_url,
+ document_handle,
+ &mut parser_owner,
+ )
} else {
DocumentParserSession::start_finite_live_document(
document_base_url,
diff --git a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
index ebcbbeb13..d429de742 100644
--- a/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
+++ b/moli-renderer-v8/src/native_bridge/context_host/child_documents/loads.rs
@@ -21,7 +21,8 @@ use crate::{
LoadedChildDocument, SubresourceResponseBody,
},
};
-use moli_encoding::decode_html_document_with_fallback;
+use moli_encoding::{HtmlDocumentStreamingDecoder, decode_html_document_with_fallback};
+use moli_encoding_detector::detect_legacy_html_encoding;
pub(crate) struct AppliedChildDocumentLoadCompletion {
/// Initial parser-classic work produced by the committed child document.
@@ -791,8 +792,24 @@ fn child_document_load_outcome_from_response(
let body_bytes = response_body
.try_bytes()
.map_err(|error| format!("failed to read child document response body: {error}"))?;
- let (markup, character_set) =
- decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback));
+ let (markup, character_set) = if let Some(mime) = content_type
+ .as_deref()
+ .filter(|mime| moli_web_mime::is_text_document_mime(mime))
+ {
+ let mut decoder = HtmlDocumentStreamingDecoder::new_text_document(
+ &head.headers,
+ head.final_url.as_str(),
+ detect_legacy_html_encoding,
+ moli_web_mime::is_json_module_mime(mime) || mime == "text/json",
+ );
+ let mut markup = decoder.push(&body_bytes).concat();
+ if let Some(tail) = decoder.finish() {
+ markup.push_str(&tail);
+ }
+ (markup, decoder.document_encoding_name())
+ } else {
+ decode_html_document_with_fallback(&body_bytes, &head.headers, Some(&fallback))
+ };
(markup, character_set.to_owned())
};
let policy_container =
diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
index 5af62b821..454433e6c 100644
--- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
@@ -25,6 +25,7 @@ impl ConcurrentParseTimeRuntime {
let (page_vm, triggered_navigation) = {
let buffered_document_preloads = &mut state.buffered_document_preloads;
let service_worker_preload_context = state.service_worker_preload_context.as_ref();
+ let document_character_set = &state.document_character_set;
PageVm::new_from_parser_stream_and_run_document_start(
page_id,
local_executor,
@@ -34,6 +35,7 @@ impl ConcurrentParseTimeRuntime {
&mut state.parser_session,
started,
|page_vm| {
+ page_vm.set_document_character_set(document_character_set.clone());
admit_pending_preloads(
page_vm,
buffered_document_preloads,
diff --git a/moli-renderer-v8/src/runtime/phase_one/state.rs b/moli-renderer-v8/src/runtime/phase_one/state.rs
index 3e9302c0d..01ae41548 100644
--- a/moli-renderer-v8/src/runtime/phase_one/state.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/state.rs
@@ -9,6 +9,7 @@ pub(super) struct ParseTimeDriverState {
pub(super) buffered_document_preloads: Box<BufferedDocumentPreloadState>,
pub(super) service_worker_preload_context: Option<ServiceWorkerScriptPreloadContext>,
pub(super) input_closed: bool,
+ pub(super) is_text_document: bool,
}
impl ParseTimeDriverState {
@@ -29,6 +30,23 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
+ is_text_document: false,
+ }
+ }
+
+ pub(super) fn new_text(final_url: Url, mime: &str) -> Self {
+ Self {
+ parser_session: DocumentParserSession::start_main_text_document(
+ final_url.clone(),
+ mime,
+ ),
+ final_url,
+ document_character_set: "UTF-8".to_owned(),
+ scheduler: DocumentScriptScheduler::new(),
+ buffered_document_preloads: Box::default(),
+ service_worker_preload_context: None,
+ input_closed: false,
+ is_text_document: true,
}
}
@@ -41,6 +59,7 @@ impl ParseTimeDriverState {
buffered_document_preloads: Box::default(),
service_worker_preload_context: None,
input_closed: false,
+ is_text_document: false,
}
}
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index f3bfeed07..fa8df43b8 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -94,10 +94,8 @@ impl ConcurrentParseTimeRuntime {
env.apply_navigation_response_headers(&response_final_url, &response_headers);
let mut body_source = RawDocumentBodySource::fetch_response(response);
- let mut state = ParseTimeDriverState::new_with_scripting_enabled(
- response_final_url,
- main_document_parser_scripting_enabled(&env),
- );
+ let (mut state, mut decoder) =
+ response_document_parser(response_final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -124,11 +122,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
- &response_headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- );
// Raw navigation bodies are decoded during prebootstrap scan. The decoder
// is carried forward so split multibyte sequences are not decoded twice or
// lost between the scan and parser handoff.
@@ -332,10 +325,7 @@ impl ConcurrentParseTimeRuntime {
},
)));
}
- let mut state = ParseTimeDriverState::new_with_scripting_enabled(
- final_url,
- main_document_parser_scripting_enabled(&env),
- );
+ let (mut state, mut decoder) = response_document_parser(final_url, &response_headers, &env);
state
.buffered_document_preloads
.set_script_fetch_requires_owner_admission(script_preloads_require_owner_admission(
@@ -362,11 +352,6 @@ impl ConcurrentParseTimeRuntime {
)
});
state.service_worker_preload_context = service_worker_preload_context.clone();
- let mut decoder = HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
- &response_headers,
- state.final_url.as_str(),
- detect_legacy_html_encoding,
- );
// External raw bodies replay captured chunks. Pre-scan only what is already
// buffered before bootstrap so the producer's backpressure boundary still
// controls how far ahead the parser can get.
@@ -451,6 +436,10 @@ impl ConcurrentParseTimeRuntime {
chunk: String,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
+ if self.state.is_text_document {
+ self.state.parser_session.queue_arrived_chunk(chunk);
+ return;
+ }
self.state
.buffered_document_preloads
.append_to_main_document_scan_with_service_worker_context(
@@ -607,6 +596,9 @@ fn scan_prebootstrap_html_chunk_into_state(
chunk: &str,
service_worker_context: Option<&ServiceWorkerScriptPreloadContext>,
) {
+ if state.is_text_document {
+ return;
+ }
state
.buffered_document_preloads
.append_to_main_document_prebootstrap_scan_with_service_worker_context(
@@ -617,6 +609,41 @@ fn scan_prebootstrap_html_chunk_into_state(
);
}
+fn response_document_parser(
+ final_url: Url,
+ headers: &[(String, Vec<u8>)],
+ env: &PageVmEnvConfig,
+) -> (ParseTimeDriverState, HtmlDocumentStreamingDecoder) {
+ let content_type = moli_web_mime::response_document_content_type(headers);
+ let text_type = content_type.filter(|mime| moli_web_mime::is_text_document_mime(mime));
+ let mut state = if let Some(mime) = &text_type {
+ ParseTimeDriverState::new_text(final_url, mime)
+ } else {
+ ParseTimeDriverState::new_with_scripting_enabled(
+ final_url,
+ main_document_parser_scripting_enabled(env),
+ )
+ };
+ let decoder = if let Some(mime) = text_type {
+ HtmlDocumentStreamingDecoder::new_text_document(
+ headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ moli_web_mime::is_json_module_mime(&mime) || mime == "text/json",
+ )
+ } else {
+ HtmlDocumentStreamingDecoder::new_with_legacy_encoding_detector(
+ headers,
+ state.final_url.as_str(),
+ detect_legacy_html_encoding,
+ )
+ };
+ // Headers/defaults are observable even when no body chunk is ready yet.
+ // A later BOM or decoder decision can still replace this tentative value.
+ sync_state_document_character_set_from_decoder(&mut state, &decoder);
+ (state, decoder)
+}
+
pub(super) fn enqueue_streaming_raw_chunk(
runtime: &mut ConcurrentParseTimeRuntime,
decoder: &mut HtmlDocumentStreamingDecoder,
@@ -910,6 +937,105 @@ mod tests {
use crate::parser::ScriptSource;
use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ #[tokio::test]
+ async fn text_document_charset_is_visible_to_document_start_before_body_arrives() {
+ let _js_runtime_owner = crate::JsRuntime::initialize();
+ tokio::task::LocalSet::new().run_until(async {
+ for (mime, chunk, expected) in [
+ ("text/plain; charset=gbk", None, "GBK"),
+ ("application/json", None, "UTF-8"),
+ ("text/plain; charset=gbk", Some(b"\xef\xbb\xbfready".as_slice()), "UTF-8"),
+ ] {
+ let loader_owner = ResourceRequestClient::new(&FetchConfig::default()).unwrap();
+ let loader = loader_owner.handle();
+ let mut env = default_test_page_vm_env_config();
+ env.document_start_scripts.push(crate::DocumentStartScript {
+ registry_key: None,
+ devtools_session: None,
+ source: "document.title = document.characterSet".to_owned(),
+ world_name: None,
+ has_bidi_channel_argument: false,
+ bidi_channel_handoffs: Vec::new(),
+ });
+ let (mut state, mut decoder) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.as_bytes().to_vec())], &env,
+ );
+ if let Some(chunk) = chunk {
+ let chunks = decoder.push(chunk);
+ sync_state_document_character_set_from_decoder(&mut state, &decoder);
+ for chunk in chunks {
+ state.parser_session.queue_arrived_chunk(chunk);
+ }
+ }
+ assert_eq!(state.document_character_set, expected);
+ let (_, page_vm, navigated) = ConcurrentParseTimeRuntime::bootstrap_page_vm_from_state_on_fresh_local_task(
+ PageId::new_for_testing(1), JsLocalExecutor::new(), loader, env,
+ PageVmRuntimeHooks::standalone_without_owner_reservation_for_test(), state,
+ Instant::now(), "charset test bootstrap closed",
+ ).await.unwrap();
+ assert!(!navigated);
+ assert_eq!(page_vm.vm().snapshot_live_document().document_title(), expected, "{mime}");
+ }
+ }).await;
+ }
+
+ #[test]
+ fn response_document_dispatch_distinguishes_text_from_html_and_xml() {
+ for (mime, text) in [
+ ("application/json", true),
+ ("application/problem+json", true),
+ ("text/plain; charset=gbk", true),
+ ("text/javascript", true),
+ ("text/html", false),
+ ("application/xhtml+xml", false),
+ ("application/xml", false),
+ ("text/xml", false),
+ ] {
+ let (state, _) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.as_bytes().to_vec())],
+ &default_test_page_vm_env_config(),
+ );
+ assert_eq!(state.is_text_document, text, "{mime}");
+ }
+ }
+
+ #[tokio::test]
+ async fn text_document_does_not_preload_literal_script_tags() {
+ let loader = ResourceRequestClient::new(&moli_fetch::FetchConfig::default())
+ .expect("default loader");
+ let runtime_hooks = PageVmRuntimeHooks::standalone_without_owner_reservation_for_test();
+ for (mime, expected_preload) in [
+ ("text/plain", false),
+ ("application/json", false),
+ ("text/html", true),
+ ] {
+ let (mut state, _) = response_document_parser(
+ Url::parse("https://example.test/data").unwrap(),
+ &[("Content-Type".to_owned(), mime.as_bytes().to_vec())],
+ &default_test_page_vm_env_config(),
+ );
+ state.buffered_document_preloads.bind_resource_runtime(
+ runtime_hooks.owner_wake(),
+ runtime_hooks.resource_task_runner(),
+ );
+ scan_prebootstrap_html_chunk_into_state(
+ &mut state,
+ &loader,
+ "<script defer src='/literal.js'></script>",
+ None,
+ );
+ assert_eq!(
+ state.buffered_document_preloads.entries.contains_key(
+ &classic_preload_key_for_streaming_test("https://example.test/literal.js")
+ ),
+ expected_preload,
+ "{mime}",
+ );
+ }
+ }
+
#[tokio::test]
async fn external_raw_document_body_source_collects_chunks_and_completion() {
let (completion_tx, completion_rx) = oneshot::channel();
diff --git a/moli-web-mime/src/classification.rs b/moli-web-mime/src/classification.rs
index 3ff3a0dd5..35fca10b5 100644
--- a/moli-web-mime/src/classification.rs
+++ b/moli-web-mime/src/classification.rs
@@ -51,6 +51,14 @@ pub fn is_text_mime_essence(input: &str) -> bool {
input.starts_with("text/")
}
+/// Navigation responses rendered as literal text rather than HTML or XML.
+/// This also includes JSON and JavaScript resources opened as documents.
+pub fn is_text_document_mime(input: &str) -> bool {
+ !is_html_document_mime(input)
+ && !is_dom_parser_xml_mime(input)
+ && (is_text_mime(input) || is_json_module_mime(input) || is_javascript_mime(input))
+}
+
/// Whether a `style` element's raw `type` attribute selects classic CSS.
///
/// HTML's update-a-style-block algorithm compares the untrimmed attribute
diff --git a/moli-web-mime/src/lib.rs b/moli-web-mime/src/lib.rs
index e631473ea..806eb2455 100644
--- a/moli-web-mime/src/lib.rs
+++ b/moli-web-mime/src/lib.rs
@@ -14,9 +14,9 @@ pub use classification::{
is_font_mime, is_font_mime_essence, is_form_urlencoded_mime, is_html_document_mime,
is_image_mime, is_image_mime_essence, is_javascript_mime, is_javascript_mime_essence,
is_json_module_mime, is_multipart_form_data_mime, is_png_image_mime, is_png_image_mime_essence,
- is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence, is_text_mime,
- is_text_mime_essence, is_video_mime, is_video_mime_essence, is_webassembly_mime,
- multipart_form_data_boundary,
+ is_supported_document_mime_type, is_svg_image_mime, is_svg_image_mime_essence,
+ is_text_document_mime, is_text_mime, is_text_mime_essence, is_video_mime,
+ is_video_mime_essence, is_webassembly_mime, multipart_form_data_boundary,
};
pub use data_url::{
data_url_body_and_computed_mime_type, data_url_body_and_mime_type, data_url_mime_type,
diff --git a/moli-web-mime/src/tests.rs b/moli-web-mime/src/tests.rs
index 010209cc7..d29adb69b 100644
--- a/moli-web-mime/src/tests.rs
+++ b/moli-web-mime/src/tests.rs
@@ -870,3 +870,25 @@ fn script_like_mime_type_block_matches_fetch_response_rule() {
));
assert!(!should_script_like_response_be_blocked_due_to_mime_type(&[]));
}
+#[test]
+fn document_text_classification_excludes_html_and_xml() {
+ for mime in [
+ "text/plain",
+ "Text/Css; charset=utf-8",
+ "application/json",
+ "application/ld+json",
+ "application/javascript",
+ ] {
+ assert!(crate::is_text_document_mime(mime), "{mime}");
+ }
+ for mime in [
+ "text/html",
+ "text/xml",
+ "application/xhtml+xml",
+ "image/svg+xml",
+ "image/png",
+ "application/octet-stream",
+ ] {
+ assert!(!crate::is_text_document_mime(mime), "{mime}");
+ }
+}
diff --git a/moli/src/fetch_dump/tests.rs b/moli/src/fetch_dump/tests.rs
index 73240abaf..b2f93f81c 100644
--- a/moli/src/fetch_dump/tests.rs
+++ b/moli/src/fetch_dump/tests.rs
@@ -1,7 +1,7 @@
use std::sync::Arc;
use anyhow::Result;
-use axum::{Router, routing::get};
+use axum::{Router, response::Html, routing::get};
use moli_core::{
LayoutPolicy,
page::Page,
@@ -36,7 +36,7 @@ async fn load_page_with_config(
"/",
get(move || {
let body = Arc::clone(&server_body);
- async move { (*body).clone() }
+ async move { Html((*body).clone()) }
}),
);
axum::serve(listener, app).await.unwrap();