Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(document): decode main XML responses before parsing

ldm083a78cf6e3d7
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
index 9649ec823..ab41e0223 100644
--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs
@@ -158,6 +158,121 @@ async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_pa
classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
}
+#[tokio::test]
+async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() {
+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";
+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";
+ let cases = [
+ (
+ "charset",
+ "; charset=windows-1252",
+ legacy_body.to_vec(),
+ "windows-1252",
+ ),
+ (
+ "declaration",
+ "",
+ [
+ b"<?xml version='1.0' encoding='windows-1252'?>".as_slice(),
+ legacy_body,
+ ]
+ .concat(),
+ "windows-1252",
+ ),
+ (
+ "utf16le",
+ "; charset=windows-1252",
+ [
+ vec![0xff, 0xfe],
+ SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(),
+ ]
+ .concat(),
+ "UTF-16LE",
+ ),
+ (
+ "utf16be",
+ "",
+ [
+ vec![0xfe, 0xff],
+ SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(),
+ ]
+ .concat(),
+ "UTF-16BE",
+ ),
+ ];
+ let mimes = [
+ "application/atom+xml",
+ "application/rss+xml",
+ "text/example+xml",
+ ];
+ let mut fixture = axum::Router::new();
+ for (index, mime) in mimes.iter().enumerate() {
+ for (name, charset, body, _) in &cases {
+ let content_type = format!("{mime}{charset}");
+ let body = body.clone();
+ fixture = fixture.route(
+ &format!("/{index}-{name}"),
+ axum::routing::get(move || {
+ let content_type = content_type.clone();
+ let body = body.clone();
+ async move { ([(header::CONTENT_TYPE, content_type)], body) }
+ }),
+ );
+ }
+ }
+ fixture = fixture.route(
+ "/parent/{kind}",
+ axum::routing::get(
+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {
+ (
+ [(header::CONTENT_TYPE, "text/html; charset=utf-8")],
+ format!("<iframe src='/{kind}'></iframe>"),
+ )
+ },
+ ),
+ );
+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-response-encoding");
+ let app = build_router(test_state());
+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;
+ let session_id = session["value"]["sessionId"].as_str().unwrap();
+ for (index, mime) in mimes.iter().enumerate() {
+ for (name, _, _, encoding) in &cases {
+ for prefix in ["", "parent/"] {
+ let path = format!("{prefix}{index}-{name}");
+ let navigated = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/url"),
+ json!({"url":format!("http://{addr}/{path}")}),
+ )
+ .await;
+ assert_eq!(navigated, json!({"value":null}), "{path}");
+ let observed = classic_request_json_with_body(
+ app.clone(),
+ Method::POST,
+ &format!("/session/{session_id}/execute/sync"),
+ json!({
+ "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet];",
+ "args": [],
+ }),
+ ).await;
+ assert_eq!(
+ observed["value"],
+ json!([
+ "feed",
+ "http://www.w3.org/2005/Atom",
+ "café",
+ mime,
+ encoding
+ ]),
+ "{path}"
+ );
+ }
+ }
+ }
+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;
+}
+
#[tokio::test]
async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {
const PAYLOAD: &str =
diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rs
index d44b8a737..6b5109e78 100644
--- a/moli-protocol/src/conn/runtime_load.rs
+++ b/moli-protocol/src/conn/runtime_load.rs
@@ -156,11 +156,6 @@ async fn prepare_network_error_page_navigation_with_engine_async(
.map(NavigationLoadOutcome::response_commit_ready)
}
-fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {
- moli_web_mime::response_document_content_type(headers)
- .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))
-}
-
#[derive(Clone, Debug)]
pub(crate) struct InitialDocumentPageOwner {
pub(crate) browser_context_id: String,
@@ -306,25 +301,17 @@ pub struct ResponseCommitReady {
network_error_page: Option<NetworkErrorPageNavigation>,
}
-enum ResponseCommitBodyCapture {
- Pending(tokio::task::JoinHandle<anyhow::Result<CapturedBody>>),
- Ready(CapturedBody),
-}
+struct ResponseCommitBodyCapture(tokio::task::JoinHandle<anyhow::Result<CapturedBody>>);
impl ResponseCommitBodyCapture {
fn abort(self) {
- if let Self::Pending(task) = self {
- task.abort();
- }
+ self.0.abort();
}
async fn resolve(self) -> anyhow::Result<CapturedBody> {
- match self {
- Self::Pending(task) => task
- .await
- .context("main document body capture task failed")?,
- Self::Ready(body) => Ok(body),
- }
+ self.0
+ .await
+ .context("main document body capture task failed")?
}
}
@@ -1283,7 +1270,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(
let body_network_progress_state =
body_progress_source.body_network_progress_for_completed_events(network_events);
let body_progress_source_for_body_finish = body_progress_source.clone();
- let mut initial_body_chunk = if response_status_may_use_http_error_page(response_status) {
+ let initial_body_chunk = if response_status_may_use_http_error_page(response_status) {
match first_nonempty_response_body_chunk(&mut response).await? {
Some(chunk) => Some(chunk),
None => {
@@ -1308,97 +1295,8 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(
None
};
- if response_headers_indicate_xml_document(&response_headers) {
- let redirected = response.redirected;
- let mut body_writer = CapturedBodyWriter::default();
- if let Some(chunk) = initial_body_chunk.take() {
- body_writer
- .append(&chunk)
- .context("failed to capture XML page body")?;
- }
- while let Some(chunk) = response.next_chunk().await {
- body_writer
- .append(&chunk)
- .context("failed to capture XML page body")?;
- }
- response
- .finish()
- .await
- .context("failed to read XML page body from stream")?;
- let captured_body = body_writer
- .finish()
- .context("failed to finish captured XML page body")?;
- let response_text = captured_body
- .materialize_bytes()
- .map(|bytes| String::from_utf8_lossy(&bytes).into_owned())
- .context("failed to materialize XML page body")?;
- let page_storage = load_inputs.page_storage_handles();
- let main_document_commit = load_inputs
- .main_document_commit_for_final_url(&final_url, None)
- .map(Arc::new);
- let prepared_page = engine
- .prepare_document_page_from_response_with_storage_and_inspector_session_restores_async(
- page_reservation,
- load_inputs.document_replacement.clone(),
- page_storage.into_navigation_storage(),
- requested_url.clone(),
- final_url.clone(),
- load_inputs.navigation_initiator_url.clone(),
- redirected,
- redirect_chain.len(),
- response_status,
- response_headers.clone(),
- response_text,
- load_inputs.document_start_scripts.clone(),
- load_inputs.runtime_bindings.clone(),
- load_inputs
- .runtime_inspector_session_restore_snapshots
- .clone(),
- load_inputs.extra_http_headers.clone(),
- load_inputs.script_execution_disabled,
- load_inputs.bypass_content_security_policy,
- load_inputs.emulated_media.clone(),
- load_inputs.viewport_surface,
- load_inputs.network_offline,
- load_inputs.blocked_url_patterns.clone(),
- fetch_subresource_interception_enabled,
- fetch_subresource_interception_resource_type,
- load_inputs.root_frame_id.clone(),
- resource_source,
- main_document_commit.as_deref().cloned(),
- )
- .await
- .with_context(|| format!("failed to prepare XML page `{}`", requested_url))?;
- if timing_enabled {
- tracing::info!(
- target: "moli_cdp_nav_timing",
- url = %requested_url,
- stage = "response_commit_ready",
- elapsed_ms = timing_started.elapsed().as_millis(),
- );
- }
- return Ok(NavigationLoadOutcome::response_commit_ready(
- ResponseCommitReady {
- prepared_page: Some(prepared_page),
- body_capture: Some(ResponseCommitBodyCapture::Ready(captured_body)),
- body_completion_sink,
- body_progress_source: body_progress_source_for_body_finish,
- body_network_progress_state: Some(body_network_progress_state),
- synthetic_body: false,
- requested_url,
- final_url,
- request_method,
- request_headers,
- response_status,
- response_headers,
- response_from_cache,
- timing_started: timing_enabled.then_some(timing_started),
- main_document_commit,
- network_error_page: None,
- },
- ));
- }
-
+ // Preserve response bytes until the renderer selects the document encoding,
+ // including XML charsets and BOMs.
let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY);
let (completion_tx, completion_rx) = oneshot::channel();
let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx);
@@ -1467,7 +1365,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(
Ok(NavigationLoadOutcome::response_commit_ready(
ResponseCommitReady {
prepared_page: Some(prepared_page),
- body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)),
+ body_capture: Some(ResponseCommitBodyCapture(body_capture_task)),
body_completion_sink,
body_progress_source: body_progress_source_for_body_finish,
body_network_progress_state: Some(body_network_progress_state),
@@ -3804,7 +3702,7 @@ async fn prepare_captured_document_response_with_engine_async(
Ok(ResponseCommitReady {
prepared_page: Some(prepared_page),
- body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)),
+ body_capture: Some(ResponseCommitBodyCapture(body_capture_task)),
body_completion_sink: None,
body_progress_source,
body_network_progress_state: Some(body_network_progress_state),
@@ -3950,7 +3848,7 @@ mod tests {
let task = super::spawn_streaming_body_capture(response, None, body_tx, completion_tx);
assert_eq!(body_rx.recv().await.unwrap(), b"partial body");
assert!(body_rx.recv().await.is_none());
- let navigation_error = super::ResponseCommitBodyCapture::Pending(task)
+ let navigation_error = super::ResponseCommitBodyCapture(task)
.resolve()
.await
.expect_err("partial response must fail navigation capture");
@@ -3977,7 +3875,7 @@ mod tests {
async fn cancelled_body_capture_retains_join_error() {
let task = tokio::spawn(std::future::pending::<anyhow::Result<super::CapturedBody>>());
task.abort();
- let error = super::ResponseCommitBodyCapture::Pending(task)
+ let error = super::ResponseCommitBodyCapture(task)
.resolve()
.await
.expect_err("cancelled capture must fail");
diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
index 454433e6c..1f5ec22b1 100644
--- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs
@@ -140,11 +140,16 @@ impl ConcurrentParseTimeRuntime {
runtime_hooks: PageVmRuntimeHooks,
final_url: Url,
document_content_type: String,
+ document_character_set: &str,
stage: PageVmInitStage,
source: String,
started: Instant,
) -> Result<ParseTimePageVmCreationOutcome> {
let mut state = ParseTimeDriverState::new_xml(final_url.clone());
+ state.document_character_set = document_character_set.to_owned();
+ state
+ .buffered_document_preloads
+ .set_document_character_set(document_character_set);
state
.parser_session
.set_xml_document_content_type(document_content_type);
diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
index f8c9351b9..395da9c8c 100644
--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs
+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs
@@ -3,7 +3,7 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_pen
use super::parser_blocking_pending::main_parser_blocking_classic_script_item;
use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;
use super::*;
-use crate::document_response_decoder::new_document_response_decoder;
+use crate::document_response_decoder::{decode_document_response, new_document_response_decoder};
use moli_encoding::HtmlDocumentStreamingDecoder;
use moli_web_mime::response_headers_indicate_attachment_download;
use tokio::sync::{mpsc, oneshot};
@@ -301,9 +301,15 @@ impl ConcurrentParseTimeRuntime {
let mut body_source = RawDocumentBodySource::External(raw_body);
if response_headers_indicate_xml_document(&response_headers) {
let body = collect_streaming_raw_body(&mut body_source).await?;
- let source = String::from_utf8_lossy(&body).into_owned();
let content_type = moli_web_mime::response_document_content_type(&response_headers)
.unwrap_or_else(|| "application/xml".to_owned());
+ let (source, character_set) = decode_document_response(
+ &body,
+ &response_headers,
+ Some(&content_type),
+ &final_url,
+ None,
+ );
let outcome = Self::finish_creation_from_xml_bootstrap(
page_id,
local_executor,
@@ -312,6 +318,7 @@ impl ConcurrentParseTimeRuntime {
runtime_hooks,
final_url,
content_type,
+ character_set,
stage,
source,
started,
diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs
index bb3e6a40e..51ad69720 100644
--- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs
+++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs
@@ -1,5 +1,102 @@
use super::*;
+#[tokio::test(flavor = "multi_thread")]
+async fn external_raw_xml_document_decodes_response_bytes() {
+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";
+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";
+ let cases = [
+ (
+ "application/atom+xml; charset=windows-1252",
+ legacy_body.to_vec(),
+ "windows-1252",
+ ),
+ (
+ "application/atom+xml",
+ [
+ b"<?xml version='1.0' encoding='windows-1252'?>".as_slice(),
+ legacy_body,
+ ]
+ .concat(),
+ "windows-1252",
+ ),
+ (
+ "application/atom+xml; charset=windows-1252",
+ [
+ vec![0xff, 0xfe],
+ SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(),
+ ]
+ .concat(),
+ "UTF-16LE",
+ ),
+ (
+ "application/atom+xml",
+ [
+ vec![0xfe, 0xff],
+ SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(),
+ ]
+ .concat(),
+ "UTF-16BE",
+ ),
+ ];
+ let runtime = JsRuntime::initialize();
+ let loader =
+ ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).expect("default loader");
+ for (content_type, body, encoding) in cases {
+ let (completion_tx, completion_rx) = oneshot::channel();
+ let (body_tx, raw_body) = ExternalRawDocumentBodyStream::channel(completion_rx);
+ let producer = tokio::spawn(async move {
+ for byte in body {
+ body_tx
+ .send(vec![byte])
+ .await
+ .expect("body byte should send");
+ }
+ completion_tx.send(Ok(())).expect("body should complete");
+ });
+ let prepared = prepare_test_external_raw_document_with_content_type(
+ &runtime,
+ &loader,
+ url::Url::parse("https://example.test/feed").unwrap(),
+ content_type,
+ raw_body,
+ )
+ .await;
+ let permit = prepared.issue_commit_permit();
+ let (mut page, _, _, _, pending_download) =
+ prepared.commit(permit).await.expect("XML should commit");
+ assert!(pending_download.is_none());
+ producer.await.expect("body producer should finish");
+ let (reply, _) = page
+ .run_async_command(RendererPageCommand::EvaluateExpression {
+ expression: concat!(
+ "JSON.stringify((() => { const root=document.getElementsByTagNameNS(",
+ "'http://www.w3.org/2005/Atom','feed')[0]; ",
+ "return [root.localName,root.namespaceURI,root.textContent,",
+ "document.contentType,document.characterSet]; })())",
+ )
+ .to_owned(),
+ await_promise: false,
+ })
+ .await
+ .expect("XML response metadata and text should evaluate");
+ assert_eq!(
+ renderer_json_value(reply),
+ Some(serde_json::Value::String(
+ serde_json::json!([
+ "feed",
+ "http://www.w3.org/2005/Atom",
+ "café",
+ "application/atom+xml",
+ encoding
+ ])
+ .to_string()
+ )),
+ "{content_type}: {encoding}"
+ );
+ page.close_async().await.expect("XML page should close");
+ }
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn streaming_xml_document_executes_parser_blocking_xhtml_script() {
let runtime = JsRuntime::initialize();