Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(document): decode main XML responses before parsing
ldm083a78cf6e3d7
diff --git a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rsindex 9649ec823..ab41e0223 100644--- a/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs+++ b/moli-protocol-server/src/protocol_server/tests/classic/extracted/navigation.rs@@ -158,6 +158,121 @@ async fn webdriver_classic_xml_suffix_navigation_preserves_namespaces_and_dom_paclassic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;}+#[tokio::test]+async fn webdriver_classic_xml_main_and_child_documents_decode_response_bytes() {+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";+ let cases = [+ (+ "charset",+ "; charset=windows-1252",+ legacy_body.to_vec(),+ "windows-1252",+ ),+ (+ "declaration",+ "",+ [+ b"<?xml version='1.0' encoding='windows-1252'?>".as_slice(),+ legacy_body,+ ]+ .concat(),+ "windows-1252",+ ),+ (+ "utf16le",+ "; charset=windows-1252",+ [+ vec![0xff, 0xfe],+ SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(),+ ]+ .concat(),+ "UTF-16LE",+ ),+ (+ "utf16be",+ "",+ [+ vec![0xfe, 0xff],+ SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(),+ ]+ .concat(),+ "UTF-16BE",+ ),+ ];+ let mimes = [+ "application/atom+xml",+ "application/rss+xml",+ "text/example+xml",+ ];+ let mut fixture = axum::Router::new();+ for (index, mime) in mimes.iter().enumerate() {+ for (name, charset, body, _) in &cases {+ let content_type = format!("{mime}{charset}");+ let body = body.clone();+ fixture = fixture.route(+ &format!("/{index}-{name}"),+ axum::routing::get(move || {+ let content_type = content_type.clone();+ let body = body.clone();+ async move { ([(header::CONTENT_TYPE, content_type)], body) }+ }),+ );+ }+ }+ fixture = fixture.route(+ "/parent/{kind}",+ axum::routing::get(+ |axum::extract::Path(kind): axum::extract::Path<String>| async move {+ (+ [(header::CONTENT_TYPE, "text/html; charset=utf-8")],+ format!("<iframe src='/{kind}'></iframe>"),+ )+ },+ ),+ );+ let (addr, _server) = spawn_dedicated_fixture_server(fixture, "xml-response-encoding");+ let app = build_router(test_state());+ let session = classic_request_json(app.clone(), Method::POST, "/session").await;+ let session_id = session["value"]["sessionId"].as_str().unwrap();+ for (index, mime) in mimes.iter().enumerate() {+ for (name, _, _, encoding) in &cases {+ for prefix in ["", "parent/"] {+ let path = format!("{prefix}{index}-{name}");+ let navigated = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/url"),+ json!({"url":format!("http://{addr}/{path}")}),+ )+ .await;+ assert_eq!(navigated, json!({"value":null}), "{path}");+ let observed = classic_request_json_with_body(+ app.clone(),+ Method::POST,+ &format!("/session/{session_id}/execute/sync"),+ json!({+ "script": "const d=document.querySelector('iframe')?.contentDocument ?? document; const root=d.getElementsByTagNameNS('http://www.w3.org/2005/Atom','feed')[0]; return [root.localName,root.namespaceURI,root.textContent,d.contentType,d.characterSet];",+ "args": [],+ }),+ ).await;+ assert_eq!(+ observed["value"],+ json!([+ "feed",+ "http://www.w3.org/2005/Atom",+ "café",+ mime,+ encoding+ ]),+ "{path}"+ );+ }+ }+ }+ classic_request_json(app, Method::DELETE, &format!("/session/{session_id}")).await;+}+#[tokio::test]async fn webdriver_classic_document_mime_is_shared_by_main_and_child_documents() {const PAYLOAD: &str =diff --git a/moli-protocol/src/conn/runtime_load.rs b/moli-protocol/src/conn/runtime_load.rsindex d44b8a737..6b5109e78 100644--- a/moli-protocol/src/conn/runtime_load.rs+++ b/moli-protocol/src/conn/runtime_load.rs@@ -156,11 +156,6 @@ async fn prepare_network_error_page_navigation_with_engine_async(.map(NavigationLoadOutcome::response_commit_ready)}-fn response_headers_indicate_xml_document(headers: &[(String, Vec<u8>)]) -> bool {- moli_web_mime::response_document_content_type(headers)- .is_some_and(|mime| moli_web_mime::is_xml_document_mime(&mime))-}-#[derive(Clone, Debug)]pub(crate) struct InitialDocumentPageOwner {pub(crate) browser_context_id: String,@@ -306,25 +301,17 @@ pub struct ResponseCommitReady {network_error_page: Option<NetworkErrorPageNavigation>,}-enum ResponseCommitBodyCapture {- Pending(tokio::task::JoinHandle<anyhow::Result<CapturedBody>>),- Ready(CapturedBody),-}+struct ResponseCommitBodyCapture(tokio::task::JoinHandle<anyhow::Result<CapturedBody>>);impl ResponseCommitBodyCapture {fn abort(self) {- if let Self::Pending(task) = self {- task.abort();- }+ self.0.abort();}async fn resolve(self) -> anyhow::Result<CapturedBody> {- match self {- Self::Pending(task) => task- .await- .context("main document body capture task failed")?,- Self::Ready(body) => Ok(body),- }+ self.0+ .await+ .context("main document body capture task failed")?}}@@ -1283,7 +1270,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(let body_network_progress_state =body_progress_source.body_network_progress_for_completed_events(network_events);let body_progress_source_for_body_finish = body_progress_source.clone();- let mut initial_body_chunk = if response_status_may_use_http_error_page(response_status) {+ let initial_body_chunk = if response_status_may_use_http_error_page(response_status) {match first_nonempty_response_body_chunk(&mut response).await? {Some(chunk) => Some(chunk),None => {@@ -1308,97 +1295,8 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(None};- if response_headers_indicate_xml_document(&response_headers) {- let redirected = response.redirected;- let mut body_writer = CapturedBodyWriter::default();- if let Some(chunk) = initial_body_chunk.take() {- body_writer- .append(&chunk)- .context("failed to capture XML page body")?;- }- while let Some(chunk) = response.next_chunk().await {- body_writer- .append(&chunk)- .context("failed to capture XML page body")?;- }- response- .finish()- .await- .context("failed to read XML page body from stream")?;- let captured_body = body_writer- .finish()- .context("failed to finish captured XML page body")?;- let response_text = captured_body- .materialize_bytes()- .map(|bytes| String::from_utf8_lossy(&bytes).into_owned())- .context("failed to materialize XML page body")?;- let page_storage = load_inputs.page_storage_handles();- let main_document_commit = load_inputs- .main_document_commit_for_final_url(&final_url, None)- .map(Arc::new);- let prepared_page = engine- .prepare_document_page_from_response_with_storage_and_inspector_session_restores_async(- page_reservation,- load_inputs.document_replacement.clone(),- page_storage.into_navigation_storage(),- requested_url.clone(),- final_url.clone(),- load_inputs.navigation_initiator_url.clone(),- redirected,- redirect_chain.len(),- response_status,- response_headers.clone(),- response_text,- load_inputs.document_start_scripts.clone(),- load_inputs.runtime_bindings.clone(),- load_inputs- .runtime_inspector_session_restore_snapshots- .clone(),- load_inputs.extra_http_headers.clone(),- load_inputs.script_execution_disabled,- load_inputs.bypass_content_security_policy,- load_inputs.emulated_media.clone(),- load_inputs.viewport_surface,- load_inputs.network_offline,- load_inputs.blocked_url_patterns.clone(),- fetch_subresource_interception_enabled,- fetch_subresource_interception_resource_type,- load_inputs.root_frame_id.clone(),- resource_source,- main_document_commit.as_deref().cloned(),- )- .await- .with_context(|| format!("failed to prepare XML page `{}`", requested_url))?;- if timing_enabled {- tracing::info!(- target: "moli_cdp_nav_timing",- url = %requested_url,- stage = "response_commit_ready",- elapsed_ms = timing_started.elapsed().as_millis(),- );- }- return Ok(NavigationLoadOutcome::response_commit_ready(- ResponseCommitReady {- prepared_page: Some(prepared_page),- body_capture: Some(ResponseCommitBodyCapture::Ready(captured_body)),- body_completion_sink,- body_progress_source: body_progress_source_for_body_finish,- body_network_progress_state: Some(body_network_progress_state),- synthetic_body: false,- requested_url,- final_url,- request_method,- request_headers,- response_status,- response_headers,- response_from_cache,- timing_started: timing_enabled.then_some(timing_started),- main_document_commit,- network_error_page: None,- },- ));- }-+ // Preserve response bytes until the renderer selects the document encoding,+ // including XML charsets and BOMs.let (body_tx, body_rx) = mpsc::channel(EXTERNAL_RAW_BODY_CHANNEL_CAPACITY);let (completion_tx, completion_rx) = oneshot::channel();let raw_body = moli_core::runtime::ExternalRawDocumentBodyStream::new(body_rx, completion_rx);@@ -1467,7 +1365,7 @@ async fn build_navigation_from_streaming_raw_response_with_engine_async(Ok(NavigationLoadOutcome::response_commit_ready(ResponseCommitReady {prepared_page: Some(prepared_page),- body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)),+ body_capture: Some(ResponseCommitBodyCapture(body_capture_task)),body_completion_sink,body_progress_source: body_progress_source_for_body_finish,body_network_progress_state: Some(body_network_progress_state),@@ -3804,7 +3702,7 @@ async fn prepare_captured_document_response_with_engine_async(Ok(ResponseCommitReady {prepared_page: Some(prepared_page),- body_capture: Some(ResponseCommitBodyCapture::Pending(body_capture_task)),+ body_capture: Some(ResponseCommitBodyCapture(body_capture_task)),body_completion_sink: None,body_progress_source,body_network_progress_state: Some(body_network_progress_state),@@ -3950,7 +3848,7 @@ mod tests {let task = super::spawn_streaming_body_capture(response, None, body_tx, completion_tx);assert_eq!(body_rx.recv().await.unwrap(), b"partial body");assert!(body_rx.recv().await.is_none());- let navigation_error = super::ResponseCommitBodyCapture::Pending(task)+ let navigation_error = super::ResponseCommitBodyCapture(task).resolve().await.expect_err("partial response must fail navigation capture");@@ -3977,7 +3875,7 @@ mod tests {async fn cancelled_body_capture_retains_join_error() {let task = tokio::spawn(std::future::pending::<anyhow::Result<super::CapturedBody>>());task.abort();- let error = super::ResponseCommitBodyCapture::Pending(task)+ let error = super::ResponseCommitBodyCapture(task).resolve().await.expect_err("cancelled capture must fail");diff --git a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rsindex 454433e6c..1f5ec22b1 100644--- a/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs+++ b/moli-renderer-v8/src/runtime/phase_one/bootstrap.rs@@ -140,11 +140,16 @@ impl ConcurrentParseTimeRuntime {runtime_hooks: PageVmRuntimeHooks,final_url: Url,document_content_type: String,+ document_character_set: &str,stage: PageVmInitStage,source: String,started: Instant,) -> Result<ParseTimePageVmCreationOutcome> {let mut state = ParseTimeDriverState::new_xml(final_url.clone());+ state.document_character_set = document_character_set.to_owned();+ state+ .buffered_document_preloads+ .set_document_character_set(document_character_set);state.parser_session.set_xml_document_content_type(document_content_type);diff --git a/moli-renderer-v8/src/runtime/phase_one/streaming.rs b/moli-renderer-v8/src/runtime/phase_one/streaming.rsindex f8c9351b9..395da9c8c 100644--- a/moli-renderer-v8/src/runtime/phase_one/streaming.rs+++ b/moli-renderer-v8/src/runtime/phase_one/streaming.rs@@ -3,7 +3,7 @@ use super::super::script_preloads::{ServiceWorkerScriptPreloadContext, admit_penuse super::parser_blocking_pending::main_parser_blocking_classic_script_item;use super::scaffold::continue_phase_one_until_streaming_boundary_on_execution_context;use super::*;-use crate::document_response_decoder::new_document_response_decoder;+use crate::document_response_decoder::{decode_document_response, new_document_response_decoder};use moli_encoding::HtmlDocumentStreamingDecoder;use moli_web_mime::response_headers_indicate_attachment_download;use tokio::sync::{mpsc, oneshot};@@ -301,9 +301,15 @@ impl ConcurrentParseTimeRuntime {let mut body_source = RawDocumentBodySource::External(raw_body);if response_headers_indicate_xml_document(&response_headers) {let body = collect_streaming_raw_body(&mut body_source).await?;- let source = String::from_utf8_lossy(&body).into_owned();let content_type = moli_web_mime::response_document_content_type(&response_headers).unwrap_or_else(|| "application/xml".to_owned());+ let (source, character_set) = decode_document_response(+ &body,+ &response_headers,+ Some(&content_type),+ &final_url,+ None,+ );let outcome = Self::finish_creation_from_xml_bootstrap(page_id,local_executor,@@ -312,6 +318,7 @@ impl ConcurrentParseTimeRuntime {runtime_hooks,final_url,content_type,+ character_set,stage,source,started,diff --git a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rsindex bb3e6a40e..51ad69720 100644--- a/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs+++ b/moli-renderer-v8/src/runtime/tests/extracted/navigation_streaming.rs@@ -1,5 +1,102 @@use super::*;+#[tokio::test(flavor = "multi_thread")]+async fn external_raw_xml_document_decodes_response_bytes() {+ const SOURCE: &str = "<feed xmlns='http://www.w3.org/2005/Atom'><title>café</title></feed>";+ let legacy_body = b"<feed xmlns='http://www.w3.org/2005/Atom'><title>caf\xe9</title></feed>";+ let cases = [+ (+ "application/atom+xml; charset=windows-1252",+ legacy_body.to_vec(),+ "windows-1252",+ ),+ (+ "application/atom+xml",+ [+ b"<?xml version='1.0' encoding='windows-1252'?>".as_slice(),+ legacy_body,+ ]+ .concat(),+ "windows-1252",+ ),+ (+ "application/atom+xml; charset=windows-1252",+ [+ vec![0xff, 0xfe],+ SOURCE.encode_utf16().flat_map(u16::to_le_bytes).collect(),+ ]+ .concat(),+ "UTF-16LE",+ ),+ (+ "application/atom+xml",+ [+ vec![0xfe, 0xff],+ SOURCE.encode_utf16().flat_map(u16::to_be_bytes).collect(),+ ]+ .concat(),+ "UTF-16BE",+ ),+ ];+ let runtime = JsRuntime::initialize();+ let loader =+ ResourceRequestClient::new(&moli_fetch::FetchConfig::default()).expect("default loader");+ for (content_type, body, encoding) in cases {+ let (completion_tx, completion_rx) = oneshot::channel();+ let (body_tx, raw_body) = ExternalRawDocumentBodyStream::channel(completion_rx);+ let producer = tokio::spawn(async move {+ for byte in body {+ body_tx+ .send(vec![byte])+ .await+ .expect("body byte should send");+ }+ completion_tx.send(Ok(())).expect("body should complete");+ });+ let prepared = prepare_test_external_raw_document_with_content_type(+ &runtime,+ &loader,+ url::Url::parse("https://example.test/feed").unwrap(),+ content_type,+ raw_body,+ )+ .await;+ let permit = prepared.issue_commit_permit();+ let (mut page, _, _, _, pending_download) =+ prepared.commit(permit).await.expect("XML should commit");+ assert!(pending_download.is_none());+ producer.await.expect("body producer should finish");+ let (reply, _) = page+ .run_async_command(RendererPageCommand::EvaluateExpression {+ expression: concat!(+ "JSON.stringify((() => { const root=document.getElementsByTagNameNS(",+ "'http://www.w3.org/2005/Atom','feed')[0]; ",+ "return [root.localName,root.namespaceURI,root.textContent,",+ "document.contentType,document.characterSet]; })())",+ )+ .to_owned(),+ await_promise: false,+ })+ .await+ .expect("XML response metadata and text should evaluate");+ assert_eq!(+ renderer_json_value(reply),+ Some(serde_json::Value::String(+ serde_json::json!([+ "feed",+ "http://www.w3.org/2005/Atom",+ "café",+ "application/atom+xml",+ encoding+ ])+ .to_string()+ )),+ "{content_type}: {encoding}"+ );+ page.close_async().await.expect("XML page should close");+ }+}+#[tokio::test(flavor = "multi_thread")]async fn streaming_xml_document_executes_parser_blocking_xhtml_script() {let runtime = JsRuntime::initialize();