Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(cdp): match Chromium intercepted response body encoding
ldm01bfa5ce0cb92
diff --git a/moli-protocol/src/domains/fetch/body_stream.rs b/moli-protocol/src/domains/fetch/body_stream.rsindex 3f19a8927..4f0c97876 100644--- a/moli-protocol/src/domains/fetch/body_stream.rs+++ b/moli-protocol/src/domains/fetch/body_stream.rs@@ -82,10 +82,9 @@ fn get_response_body_without_transfer_command_output_plan(}fn encode_response_body(bytes: Vec<u8>) -> (String, bool) {- match String::from_utf8(bytes) {- Ok(body) => (body, false),- Err(error) => (BASE64_STANDARD.encode(error.into_bytes()), true),- }+ // Chromium returns interception bytes as Base64 even for text responses.+ // Clients can pass the body directly back to Fetch.fulfillRequest.+ (BASE64_STANDARD.encode(bytes), true)}fn response_body_command_output_plan(bytes: Vec<u8>) -> CommandOutputPlan {@@ -193,14 +192,15 @@ mod tests {use super::encode_response_body;#[test]- fn encode_response_body_preserves_utf8_and_base64_encodes_binary() {+ fn encode_response_body_uses_base64_for_text_binary_and_empty_responses() {assert_eq!(encode_response_body(b"hello".to_vec()),- ("hello".to_owned(), false)+ ("aGVsbG8=".to_owned(), true));assert_eq!(encode_response_body(vec![0, 255, b'a']),("AP9h".to_owned(), true));+ assert_eq!(encode_response_body(Vec::new()), (String::new(), true));}}diff --git a/moli-protocol/src/domains/fetch/tests/navigation_auth.rs b/moli-protocol/src/domains/fetch/tests/navigation_auth.rsindex c87a06df2..79352b077 100644--- a/moli-protocol/src/domains/fetch/tests/navigation_auth.rs+++ b/moli-protocol/src/domains/fetch/tests/navigation_auth.rs@@ -6,6 +6,7 @@ use crate::automation::{DevToolsNetworkInterceptId, DevToolsNetworkInterceptPattern, DevToolsNetworkInterceptPhase,DevToolsRequestId, DevToolsSessionId, DevToolsTargetId, FrontendProtocol,};+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};#[tokio::test(flavor = "multi_thread")]async fn continue_with_auth_retries_navigation_with_basic_credentials() {@@ -1139,7 +1140,7 @@ async fn run_navigation_cdp_fetch_then_bidi_network_auth_required_terminal(.await;ctx.expect_result(73_105,- json!({ "body": "auth required", "base64Encoded": false }),+ json!({ "body": BASE64_STANDARD.encode("auth required"), "base64Encoded": true }),Some("SID-fetch"),);diff --git a/moli-protocol/src/domains/fetch/tests/navigation_control.rs b/moli-protocol/src/domains/fetch/tests/navigation_control.rsindex 2b3008f72..f9ef1a3f5 100644--- a/moli-protocol/src/domains/fetch/tests/navigation_control.rs+++ b/moli-protocol/src/domains/fetch/tests/navigation_control.rs@@ -1067,8 +1067,8 @@ async fn continue_request_with_intercept_response_pauses_after_response_until_coctx.expect_result(323,json!({- "body": "<!doctype html><html><body><main>response-stage</main></body></html>",- "base64Encoded": false+ "body": BASE64_STANDARD.encode("<!doctype html><html><body><main>response-stage</main></body></html>"),+ "base64Encoded": true}),Some("SID-1"),);@@ -1104,6 +1104,108 @@ async fn continue_request_with_intercept_response_pauses_after_response_until_coserver.abort();}+#[tokio::test(flavor = "multi_thread")]+async fn preload_navigations_can_round_trip_intercepted_text_through_fulfill_request() {+ const HTML: &str = "<!doctype html><meta charset=utf-8><body><main></main><p>中文 🦀</p><script>document.querySelector('main').textContent = globalThis.preloadMarker || 'missing';</script>";+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ Router::new().route("/page", get(|| async { axum::response::Html(HTML) })),+ )+ .await+ .unwrap();+ });+ let url = format!("http://{addr}/page");+ let mut ctx = TestContext::new();+ with_loaded_http_document(&mut ctx, &url, "SID-1", "TID-1").await;+ ctx.enable_page_events_for_test(Some("SID-1"));+ ctx.sent.clear();+ for (id, method, params) in [+ (1, "Fetch.enable", json!({})),+ (+ 2,+ "Page.addScriptToEvaluateOnNewDocument",+ json!({"source": "globalThis.preloadMarker = 'preloaded';"}),+ ),+ ] {+ ctx.process_async(json!({+ "id": id, "method": method, "sessionId": "SID-1", "params": params+ }))+ .await;+ assert!(take_response_by_id(&mut ctx, id)["error"].is_null());+ }++ for navigation_id in [10, 20] {+ ctx.sent.clear();+ ctx.process_async(json!({+ "id": navigation_id,+ "method": "Page.navigate",+ "sessionId": "SID-1",+ "params": {"url": format!("{url}?navigation={navigation_id}")}+ }))+ .await;+ let paused = take_main_document_request_pause(&mut ctx).await;+ let request_id = paused["params"]["requestId"].clone();+ ctx.process_async(json!({+ "id": navigation_id + 1,+ "method": "Fetch.continueRequest",+ "sessionId": "SID-1",+ "params": {"requestId": request_id, "interceptResponse": true}+ }))+ .await;+ ctx.expect_result(navigation_id + 1, json!({}), Some("SID-1"));+ wait_until_scheduler_message(&mut ctx, "response-stage pause", |message| {+ message["method"] == "Fetch.requestPaused"+ && message["params"]["responseStatusCode"] == 200+ })+ .await;+ let paused = take_main_document_response_pause(&mut ctx);+ ctx.process_async(json!({+ "id": navigation_id + 2,+ "method": "Fetch.getResponseBody",+ "sessionId": "SID-1",+ "params": {"requestId": request_id}+ }))+ .await;+ let body = take_response_by_id(&mut ctx, navigation_id + 2);+ assert_eq!(body["result"]["base64Encoded"], true);+ assert_eq!(+ BASE64_STANDARD+ .decode(body["result"]["body"].as_str().unwrap())+ .unwrap(),+ HTML.as_bytes()+ );++ // Patchright's init-script interception forwards this body as Base64.+ ctx.process_async(json!({+ "id": navigation_id + 3,+ "method": "Fetch.fulfillRequest",+ "sessionId": "SID-1",+ "params": {+ "requestId": request_id,+ "responseCode": 200,+ "responseHeaders": paused["params"]["responseHeaders"],+ "body": body["result"]["body"]+ }+ }))+ .await;+ ctx.expect_result(navigation_id + 3, json!({}), Some("SID-1"));+ assert!(take_response_by_id(&mut ctx, navigation_id)["error"].is_null());+ wait_until_frame_stopped_loading(&mut ctx, "TID-1").await;+ assert!(+ ctx.sent+ .iter()+ .any(|message| { message["method"] == "Page.domContentEventFired" })+ );+ let html = loaded_page_html_for_test(&mut ctx).await;+ assert!(html.contains("<main>preloaded</main>"), "{html}");+ assert!(html.contains("中文 🦀"), "{html}");+ }+ server.abort();+}+#[tokio::test(flavor = "multi_thread")]async fn intercepted_navigation_start_events_stay_before_network_pause_with_background_sender() {let mut ctx = TestContext::new();diff --git a/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs b/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rsindex d8e92119f..d09be8314 100644--- a/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs+++ b/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs@@ -7,6 +7,7 @@ use crate::automation::{DevToolsRequestId, DevToolsSessionId, DevToolsTargetId, FrontendProtocol,};use crate::testing::drain_scheduler_events_like_scheduler;+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};fn is_fetch_event_for_request(message: &serde_json::Value, method: &str, request_id: &str) -> bool {message["method"] == json!(method)@@ -208,8 +209,8 @@ async fn runtime_fetch_subresource_intercept_response_pauses_after_response_untictx.expect_result(386,json!({- "body": "fetch-response-stage",- "base64Encoded": false+ "body": BASE64_STANDARD.encode("fetch-response-stage"),+ "base64Encoded": true}),Some("SID-1"),);@@ -870,8 +871,8 @@ async fn runtime_xhr_subresource_intercept_response_pauses_after_response_until_ctx.expect_result(393,json!({- "body": "xhr-response-stage",- "base64Encoded": false+ "body": BASE64_STANDARD.encode("xhr-response-stage"),+ "base64Encoded": true}),Some("SID-1"),);diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rsindex af605901b..6a18ca8d3 100644--- a/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs+++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs@@ -1,4 +1,5 @@use super::*;+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};#[tokio::test(flavor = "multi_thread")]async fn runtime_local_fetch_and_xhr_bypass_request_and_response_interception() {@@ -584,7 +585,7 @@ async fn attached_session_fetch_response_stage_body_commands_use_session_owner().await;ctx.expect_result(35_923,- json!({ "body": "attached fetch body", "base64Encoded": false }),+ json!({ "body": BASE64_STANDARD.encode("attached fetch body"), "base64Encoded": true }),Some("SID-attached"),);@@ -816,7 +817,7 @@ async fn multiple_fetch_sessions_chain_subresource_response_stage_pauses() {.await;ctx.expect_result(35_945,- json!({ "body": "multi response body", "base64Encoded": false }),+ json!({ "body": BASE64_STANDARD.encode("multi response body"), "base64Encoded": true }),Some("SID-attached"),);diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rsindex b77bf707d..ee673aa8a 100644--- a/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs+++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs@@ -1,4 +1,5 @@use super::*;+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};#[tokio::test(flavor = "multi_thread")]async fn worker_fetch_request_stage_pauses_until_continue_request_then_resolves_promise() {@@ -2608,7 +2609,7 @@ fetch('/worker-auth').await;ctx.expect_result(37_727,- json!({ "body": "auth required", "base64Encoded": false }),+ json!({ "body": BASE64_STANDARD.encode("auth required"), "base64Encoded": true }),Some("SID-1"),);diff --git a/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs b/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rsindex ad6b9c9e9..bfe5da67a 100644--- a/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs+++ b/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs@@ -793,10 +793,10 @@ async fn rust_cdp_p0_document_response_stage_get_body_then_continue() {assert_response_stage_header(&paused, "x-smoke-document-stage", "paused");let body = get_paused_response_body(&mut ctx, &page, 155_007, &paused).await;- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");assert_eq!(body["result"]["body"],- json!("<!doctype html><main>document response-stage body</main>"),+ json!(BASE64_STANDARD.encode("<!doctype html><main>document response-stage body</main>")),"{body}");@@ -953,10 +953,10 @@ async fn rust_cdp_p0_document_redirect_response_stage_get_body_then_continue() {);let body = get_paused_response_body(&mut ctx, &page, 160_007, &paused).await;- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");assert_eq!(body["result"]["body"],- json!("<!doctype html><main>redirect final</main>"),+ json!(BASE64_STANDARD.encode("<!doctype html><main>redirect final</main>")),"{body}");@@ -1072,11 +1072,15 @@ async fn rust_cdp_p0_fetch_redirect_response_stage_get_body_then_continue() {.unwrap_or_else(|| panic!("paused networkId: {paused}"));let body = get_paused_response_body(&mut ctx, &page, 150_008, &paused).await;- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");- let body_value: Value = serde_json::from_str(- body["result"]["body"]- .as_str()- .unwrap_or_else(|| panic!("redirect response body: {body}")),+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");+ let body_value: Value = serde_json::from_slice(+ &BASE64_STANDARD+ .decode(+ body["result"]["body"]+ .as_str()+ .unwrap_or_else(|| panic!("redirect response body: {body}")),+ )+ .expect("Base64 redirect response body"),).unwrap_or_else(|error| panic!("redirect response body json error: {error}: {body}"));assert_eq!(@@ -1159,11 +1163,15 @@ async fn rust_cdp_p0_xhr_redirect_response_stage_get_body_then_continue() {.unwrap_or_else(|| panic!("paused networkId: {paused}"));let body = get_paused_response_body(&mut ctx, &page, 151_008, &paused).await;- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");- let body_value: Value = serde_json::from_str(- body["result"]["body"]- .as_str()- .unwrap_or_else(|| panic!("XHR redirect response body: {body}")),+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");+ let body_value: Value = serde_json::from_slice(+ &BASE64_STANDARD+ .decode(+ body["result"]["body"]+ .as_str()+ .unwrap_or_else(|| panic!("XHR redirect response body: {body}")),+ )+ .expect("Base64 XHR redirect response body"),).unwrap_or_else(|error| panic!("XHR redirect response body json error: {error}: {body}"));assert_eq!(@@ -1253,10 +1261,10 @@ async fn rust_cdp_p0_fetch_response_stage_get_body_then_continue() {let body = get_paused_response_body(&mut ctx, &page, 143_008, &paused).await;assert_eq!(body["result"]["body"],- json!("response-stage body"),+ json!(BASE64_STANDARD.encode("response-stage body")),"{body}");- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");continue_paused_response(&mut ctx, &page, 143_009, &paused).await;let completed = wait_for_global_string(&mut ctx, &page, 143_010, "__p0ResponseStageBody").await;@@ -1872,10 +1880,10 @@ async fn rust_cdp_p0_xhr_response_stage_get_body_then_continue() {let body = get_paused_response_body(&mut ctx, &page, 149_008, &paused).await;assert_eq!(body["result"]["body"],- json!("response-stage body"),+ json!(BASE64_STANDARD.encode("response-stage body")),"{body}");- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");continue_paused_response(&mut ctx, &page, 149_009, &paused).await;let completed = wait_for_global_string(&mut ctx, &page, 149_010, "__p0XhrResponseStage").await;