Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(cdp): match Chromium intercepted response body encoding

ldm01bfa5ce0cb92
diff --git a/moli-protocol/src/domains/fetch/body_stream.rs b/moli-protocol/src/domains/fetch/body_stream.rs
index 3f19a8927..4f0c97876 100644
--- a/moli-protocol/src/domains/fetch/body_stream.rs
+++ b/moli-protocol/src/domains/fetch/body_stream.rs
@@ -82,10 +82,9 @@ fn get_response_body_without_transfer_command_output_plan(
}
fn encode_response_body(bytes: Vec<u8>) -> (String, bool) {
- match String::from_utf8(bytes) {
- Ok(body) => (body, false),
- Err(error) => (BASE64_STANDARD.encode(error.into_bytes()), true),
- }
+ // Chromium returns interception bytes as Base64 even for text responses.
+ // Clients can pass the body directly back to Fetch.fulfillRequest.
+ (BASE64_STANDARD.encode(bytes), true)
}
fn response_body_command_output_plan(bytes: Vec<u8>) -> CommandOutputPlan {
@@ -193,14 +192,15 @@ mod tests {
use super::encode_response_body;
#[test]
- fn encode_response_body_preserves_utf8_and_base64_encodes_binary() {
+ fn encode_response_body_uses_base64_for_text_binary_and_empty_responses() {
assert_eq!(
encode_response_body(b"hello".to_vec()),
- ("hello".to_owned(), false)
+ ("aGVsbG8=".to_owned(), true)
);
assert_eq!(
encode_response_body(vec![0, 255, b'a']),
("AP9h".to_owned(), true)
);
+ assert_eq!(encode_response_body(Vec::new()), (String::new(), true));
}
}
diff --git a/moli-protocol/src/domains/fetch/tests/navigation_auth.rs b/moli-protocol/src/domains/fetch/tests/navigation_auth.rs
index c87a06df2..79352b077 100644
--- a/moli-protocol/src/domains/fetch/tests/navigation_auth.rs
+++ b/moli-protocol/src/domains/fetch/tests/navigation_auth.rs
@@ -6,6 +6,7 @@ use crate::automation::{
DevToolsNetworkInterceptId, DevToolsNetworkInterceptPattern, DevToolsNetworkInterceptPhase,
DevToolsRequestId, DevToolsSessionId, DevToolsTargetId, FrontendProtocol,
};
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[tokio::test(flavor = "multi_thread")]
async fn continue_with_auth_retries_navigation_with_basic_credentials() {
@@ -1139,7 +1140,7 @@ async fn run_navigation_cdp_fetch_then_bidi_network_auth_required_terminal(
.await;
ctx.expect_result(
73_105,
- json!({ "body": "auth required", "base64Encoded": false }),
+ json!({ "body": BASE64_STANDARD.encode("auth required"), "base64Encoded": true }),
Some("SID-fetch"),
);
diff --git a/moli-protocol/src/domains/fetch/tests/navigation_control.rs b/moli-protocol/src/domains/fetch/tests/navigation_control.rs
index 2b3008f72..f9ef1a3f5 100644
--- a/moli-protocol/src/domains/fetch/tests/navigation_control.rs
+++ b/moli-protocol/src/domains/fetch/tests/navigation_control.rs
@@ -1067,8 +1067,8 @@ async fn continue_request_with_intercept_response_pauses_after_response_until_co
ctx.expect_result(
323,
json!({
- "body": "<!doctype html><html><body><main>response-stage</main></body></html>",
- "base64Encoded": false
+ "body": BASE64_STANDARD.encode("<!doctype html><html><body><main>response-stage</main></body></html>"),
+ "base64Encoded": true
}),
Some("SID-1"),
);
@@ -1104,6 +1104,108 @@ async fn continue_request_with_intercept_response_pauses_after_response_until_co
server.abort();
}
+#[tokio::test(flavor = "multi_thread")]
+async fn preload_navigations_can_round_trip_intercepted_text_through_fulfill_request() {
+ const HTML: &str = "<!doctype html><meta charset=utf-8><body><main></main><p>中文 🦀</p><script>document.querySelector('main').textContent = globalThis.preloadMarker || 'missing';</script>";
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new().route("/page", get(|| async { axum::response::Html(HTML) })),
+ )
+ .await
+ .unwrap();
+ });
+ let url = format!("http://{addr}/page");
+ let mut ctx = TestContext::new();
+ with_loaded_http_document(&mut ctx, &url, "SID-1", "TID-1").await;
+ ctx.enable_page_events_for_test(Some("SID-1"));
+ ctx.sent.clear();
+ for (id, method, params) in [
+ (1, "Fetch.enable", json!({})),
+ (
+ 2,
+ "Page.addScriptToEvaluateOnNewDocument",
+ json!({"source": "globalThis.preloadMarker = 'preloaded';"}),
+ ),
+ ] {
+ ctx.process_async(json!({
+ "id": id, "method": method, "sessionId": "SID-1", "params": params
+ }))
+ .await;
+ assert!(take_response_by_id(&mut ctx, id)["error"].is_null());
+ }
+
+ for navigation_id in [10, 20] {
+ ctx.sent.clear();
+ ctx.process_async(json!({
+ "id": navigation_id,
+ "method": "Page.navigate",
+ "sessionId": "SID-1",
+ "params": {"url": format!("{url}?navigation={navigation_id}")}
+ }))
+ .await;
+ let paused = take_main_document_request_pause(&mut ctx).await;
+ let request_id = paused["params"]["requestId"].clone();
+ ctx.process_async(json!({
+ "id": navigation_id + 1,
+ "method": "Fetch.continueRequest",
+ "sessionId": "SID-1",
+ "params": {"requestId": request_id, "interceptResponse": true}
+ }))
+ .await;
+ ctx.expect_result(navigation_id + 1, json!({}), Some("SID-1"));
+ wait_until_scheduler_message(&mut ctx, "response-stage pause", |message| {
+ message["method"] == "Fetch.requestPaused"
+ && message["params"]["responseStatusCode"] == 200
+ })
+ .await;
+ let paused = take_main_document_response_pause(&mut ctx);
+ ctx.process_async(json!({
+ "id": navigation_id + 2,
+ "method": "Fetch.getResponseBody",
+ "sessionId": "SID-1",
+ "params": {"requestId": request_id}
+ }))
+ .await;
+ let body = take_response_by_id(&mut ctx, navigation_id + 2);
+ assert_eq!(body["result"]["base64Encoded"], true);
+ assert_eq!(
+ BASE64_STANDARD
+ .decode(body["result"]["body"].as_str().unwrap())
+ .unwrap(),
+ HTML.as_bytes()
+ );
+
+ // Patchright's init-script interception forwards this body as Base64.
+ ctx.process_async(json!({
+ "id": navigation_id + 3,
+ "method": "Fetch.fulfillRequest",
+ "sessionId": "SID-1",
+ "params": {
+ "requestId": request_id,
+ "responseCode": 200,
+ "responseHeaders": paused["params"]["responseHeaders"],
+ "body": body["result"]["body"]
+ }
+ }))
+ .await;
+ ctx.expect_result(navigation_id + 3, json!({}), Some("SID-1"));
+ assert!(take_response_by_id(&mut ctx, navigation_id)["error"].is_null());
+ wait_until_frame_stopped_loading(&mut ctx, "TID-1").await;
+ assert!(
+ ctx.sent
+ .iter()
+ .any(|message| { message["method"] == "Page.domContentEventFired" })
+ );
+ let html = loaded_page_html_for_test(&mut ctx).await;
+ assert!(html.contains("<main>preloaded</main>"), "{html}");
+ assert!(html.contains("中文 🦀"), "{html}");
+ }
+ server.abort();
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn intercepted_navigation_start_events_stay_before_network_pause_with_background_sender() {
let mut ctx = TestContext::new();
diff --git a/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs b/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs
index d8e92119f..d09be8314 100644
--- a/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs
+++ b/moli-protocol/src/domains/fetch/tests/runtime_auth_response.rs
@@ -7,6 +7,7 @@ use crate::automation::{
DevToolsRequestId, DevToolsSessionId, DevToolsTargetId, FrontendProtocol,
};
use crate::testing::drain_scheduler_events_like_scheduler;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
fn is_fetch_event_for_request(message: &serde_json::Value, method: &str, request_id: &str) -> bool {
message["method"] == json!(method)
@@ -208,8 +209,8 @@ async fn runtime_fetch_subresource_intercept_response_pauses_after_response_unti
ctx.expect_result(
386,
json!({
- "body": "fetch-response-stage",
- "base64Encoded": false
+ "body": BASE64_STANDARD.encode("fetch-response-stage"),
+ "base64Encoded": true
}),
Some("SID-1"),
);
@@ -870,8 +871,8 @@ async fn runtime_xhr_subresource_intercept_response_pauses_after_response_until_
ctx.expect_result(
393,
json!({
- "body": "xhr-response-stage",
- "base64Encoded": false
+ "body": BASE64_STANDARD.encode("xhr-response-stage"),
+ "base64Encoded": true
}),
Some("SID-1"),
);
diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs
index af605901b..6a18ca8d3 100644
--- a/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs
+++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch/session_interception.rs
@@ -1,4 +1,5 @@
use super::*;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[tokio::test(flavor = "multi_thread")]
async fn runtime_local_fetch_and_xhr_bypass_request_and_response_interception() {
@@ -584,7 +585,7 @@ async fn attached_session_fetch_response_stage_body_commands_use_session_owner()
.await;
ctx.expect_result(
35_923,
- json!({ "body": "attached fetch body", "base64Encoded": false }),
+ json!({ "body": BASE64_STANDARD.encode("attached fetch body"), "base64Encoded": true }),
Some("SID-attached"),
);
@@ -816,7 +817,7 @@ async fn multiple_fetch_sessions_chain_subresource_response_stage_pauses() {
.await;
ctx.expect_result(
35_945,
- json!({ "body": "multi response body", "base64Encoded": false }),
+ json!({ "body": BASE64_STANDARD.encode("multi response body"), "base64Encoded": true }),
Some("SID-attached"),
);
diff --git a/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs b/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs
index b77bf707d..ee673aa8a 100644
--- a/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs
+++ b/moli-protocol/src/domains/fetch/tests/runtime_fetch/worker_requests.rs
@@ -1,4 +1,5 @@
use super::*;
+use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64_STANDARD};
#[tokio::test(flavor = "multi_thread")]
async fn worker_fetch_request_stage_pauses_until_continue_request_then_resolves_promise() {
@@ -2608,7 +2609,7 @@ fetch('/worker-auth')
.await;
ctx.expect_result(
37_727,
- json!({ "body": "auth required", "base64Encoded": false }),
+ json!({ "body": BASE64_STANDARD.encode("auth required"), "base64Encoded": true }),
Some("SID-1"),
);
diff --git a/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs b/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs
index ad6b9c9e9..bfe5da67a 100644
--- a/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs
+++ b/moli-protocol/src/domains/target/tests/tests_cdp_chromium_imports/p0_network.rs
@@ -793,10 +793,10 @@ async fn rust_cdp_p0_document_response_stage_get_body_then_continue() {
assert_response_stage_header(&paused, "x-smoke-document-stage", "paused");
let body = get_paused_response_body(&mut ctx, &page, 155_007, &paused).await;
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
assert_eq!(
body["result"]["body"],
- json!("<!doctype html><main>document response-stage body</main>"),
+ json!(BASE64_STANDARD.encode("<!doctype html><main>document response-stage body</main>")),
"{body}"
);
@@ -953,10 +953,10 @@ async fn rust_cdp_p0_document_redirect_response_stage_get_body_then_continue() {
);
let body = get_paused_response_body(&mut ctx, &page, 160_007, &paused).await;
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
assert_eq!(
body["result"]["body"],
- json!("<!doctype html><main>redirect final</main>"),
+ json!(BASE64_STANDARD.encode("<!doctype html><main>redirect final</main>")),
"{body}"
);
@@ -1072,11 +1072,15 @@ async fn rust_cdp_p0_fetch_redirect_response_stage_get_body_then_continue() {
.unwrap_or_else(|| panic!("paused networkId: {paused}"));
let body = get_paused_response_body(&mut ctx, &page, 150_008, &paused).await;
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
- let body_value: Value = serde_json::from_str(
- body["result"]["body"]
- .as_str()
- .unwrap_or_else(|| panic!("redirect response body: {body}")),
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
+ let body_value: Value = serde_json::from_slice(
+ &BASE64_STANDARD
+ .decode(
+ body["result"]["body"]
+ .as_str()
+ .unwrap_or_else(|| panic!("redirect response body: {body}")),
+ )
+ .expect("Base64 redirect response body"),
)
.unwrap_or_else(|error| panic!("redirect response body json error: {error}: {body}"));
assert_eq!(
@@ -1159,11 +1163,15 @@ async fn rust_cdp_p0_xhr_redirect_response_stage_get_body_then_continue() {
.unwrap_or_else(|| panic!("paused networkId: {paused}"));
let body = get_paused_response_body(&mut ctx, &page, 151_008, &paused).await;
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
- let body_value: Value = serde_json::from_str(
- body["result"]["body"]
- .as_str()
- .unwrap_or_else(|| panic!("XHR redirect response body: {body}")),
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
+ let body_value: Value = serde_json::from_slice(
+ &BASE64_STANDARD
+ .decode(
+ body["result"]["body"]
+ .as_str()
+ .unwrap_or_else(|| panic!("XHR redirect response body: {body}")),
+ )
+ .expect("Base64 XHR redirect response body"),
)
.unwrap_or_else(|error| panic!("XHR redirect response body json error: {error}: {body}"));
assert_eq!(
@@ -1253,10 +1261,10 @@ async fn rust_cdp_p0_fetch_response_stage_get_body_then_continue() {
let body = get_paused_response_body(&mut ctx, &page, 143_008, &paused).await;
assert_eq!(
body["result"]["body"],
- json!("response-stage body"),
+ json!(BASE64_STANDARD.encode("response-stage body")),
"{body}"
);
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
continue_paused_response(&mut ctx, &page, 143_009, &paused).await;
let completed = wait_for_global_string(&mut ctx, &page, 143_010, "__p0ResponseStageBody").await;
@@ -1872,10 +1880,10 @@ async fn rust_cdp_p0_xhr_response_stage_get_body_then_continue() {
let body = get_paused_response_body(&mut ctx, &page, 149_008, &paused).await;
assert_eq!(
body["result"]["body"],
- json!("response-stage body"),
+ json!(BASE64_STANDARD.encode("response-stage body")),
"{body}"
);
- assert_eq!(body["result"]["base64Encoded"], json!(false), "{body}");
+ assert_eq!(body["result"]["base64Encoded"], json!(true), "{body}");
continue_paused_response(&mut ctx, &page, 149_009, &paused).await;
let completed = wait_for_global_string(&mut ctx, &page, 149_010, "__p0XhrResponseStage").await;