Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(network): capture main-document POST bodies across Fetch interception

BibekPathak7f6c7634b636
diff --git a/moli-protocol/src/domains/fetch/navigation.rs b/moli-protocol/src/domains/fetch/navigation.rs
index 33d2dcd43..781ebb3e4 100644
--- a/moli-protocol/src/domains/fetch/navigation.rs
+++ b/moli-protocol/src/domains/fetch/navigation.rs
@@ -65,6 +65,7 @@ pub(crate) async fn load_or_pause_navigation_for_auth_into_buffer_async(
prior_network_observation_journal: Option<NetworkObservationJournal>,
) {
let navigate_session_id = pending.navigation.navigate_session_id.clone();
+ network::record_main_document_request_body(conn, &pending.navigation);
let should_handle_auth = conn.target_fetch_matches_auth_required_for_session_owner(
navigate_session_id.as_deref(),
&pending.navigation.requested_url,
diff --git a/moli-protocol/src/domains/fetch/tests/navigation_control.rs b/moli-protocol/src/domains/fetch/tests/navigation_control.rs
index afa0f4020..f79406a43 100644
--- a/moli-protocol/src/domains/fetch/tests/navigation_control.rs
+++ b/moli-protocol/src/domains/fetch/tests/navigation_control.rs
@@ -649,6 +649,155 @@ async fn continue_request_with_post_data_marks_network_request_as_having_post_da
Some("<!doctype html><html><body><main>payload</main></body></html>".to_owned())
);
+ ctx.process_async(json!({
+ "id": 343,
+ "method": "Network.getRequestPostData",
+ "sessionId": "SID-1",
+ "params": { "requestId": LOADER_ID }
+ }))
+ .await;
+ ctx.expect_result(
+ 343,
+ json!({ "postData": "payload", "base64Encoded": false }),
+ Some("SID-1"),
+ );
+
+ server.abort();
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn intercepted_form_post_navigation_body_is_available_by_network_request_id() {
+ async fn page() -> impl IntoResponse {
+ (
+ [(CONTENT_TYPE.as_str(), "text/html")],
+ r#"<!doctype html><form id="f" method="POST" action="/post"><input name="username" value="alice"><input name="pw" value="s3cret"></form>"#,
+ )
+ }
+
+ async fn post_body(body: String) -> impl IntoResponse {
+ (
+ [(CONTENT_TYPE.as_str(), "text/html")],
+ format!("<!doctype html><html><body><main>{body}</main></body></html>"),
+ )
+ }
+
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new()
+ .route("/page", get(page))
+ .route("/post", axum::routing::post(post_body)),
+ )
+ .await
+ .unwrap();
+ });
+
+ let page_url = format!("http://{addr}/page");
+ let post_url = format!("http://{addr}/post");
+ let mut ctx = TestContext::new();
+ ctx.conn.browser_context = Some(attached_browser_context());
+ ctx.install_navigation_fixture_for_session_owner(&page_url, Some("SID-1"))
+ .await;
+ {
+ let bc = ctx.conn.browser_context.as_mut().expect("browser context");
+ bc.active_target
+ .runtime_slot
+ .enable_primary_network_events();
+ }
+ ctx.sent.clear();
+
+ ctx.process_async(json!({
+ "id": 420,
+ "method": "Fetch.enable",
+ "sessionId": "SID-1",
+ "params": {
+ "patterns": [{ "urlPattern": "*/post", "requestStage": "Request", "resourceType": "Document" }]
+ }
+ }))
+ .await;
+ ctx.expect_result(420, json!({}), Some("SID-1"));
+ ctx.sent.clear();
+
+ ctx.process_async(json!({
+ "id": 421,
+ "method": "Runtime.evaluate",
+ "sessionId": "SID-1",
+ "params": {
+ "expression": "document.getElementById('f').submit(); 'scheduled'"
+ }
+ }))
+ .await;
+ let _ = take_response_by_id(&mut ctx, 421);
+
+ wait_until_scheduler_message(
+ &mut ctx,
+ "intercepted form POST Fetch.requestPaused",
+ |message| {
+ message["method"] == json!("Fetch.requestPaused")
+ && message["params"]["resourceType"] == json!("Document")
+ && message["params"]["request"]["url"] == json!(post_url)
+ },
+ )
+ .await;
+ let paused = ctx
+ .sent
+ .iter()
+ .find(|message| {
+ message["method"] == json!("Fetch.requestPaused")
+ && message["params"]["resourceType"] == json!("Document")
+ && message["params"]["request"]["url"] == json!(post_url)
+ })
+ .cloned()
+ .expect("intercepted form POST pause");
+ assert_eq!(paused["params"]["request"]["method"], json!("POST"));
+ assert_eq!(
+ paused["params"]["request"]["postData"],
+ json!("username=alice&pw=s3cret"),
+ "the paused form POST must carry its original body"
+ );
+ let fetch_request_id = paused["params"]["requestId"]
+ .as_str()
+ .expect("fetch request id")
+ .to_owned();
+ let network_request_id = paused["params"]["networkId"]
+ .as_str()
+ .expect("network request id")
+ .to_owned();
+
+ ctx.process_async(json!({
+ "id": 422,
+ "method": "Fetch.continueRequest",
+ "sessionId": "SID-1",
+ "params": { "requestId": fetch_request_id }
+ }))
+ .await;
+ ctx.expect_result(422, json!({}), Some("SID-1"));
+
+ wait_until_scheduler_message(
+ &mut ctx,
+ "intercepted form POST responseReceived",
+ |message| {
+ message["method"] == json!("Network.responseReceived")
+ && message["params"]["requestId"] == json!(network_request_id)
+ },
+ )
+ .await;
+
+ ctx.process_async(json!({
+ "id": 423,
+ "method": "Network.getRequestPostData",
+ "sessionId": "SID-1",
+ "params": { "requestId": network_request_id }
+ }))
+ .await;
+ ctx.expect_result(
+ 423,
+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),
+ Some("SID-1"),
+ );
+
server.abort();
}
diff --git a/moli-protocol/src/domains/network.rs b/moli-protocol/src/domains/network.rs
index a400ace2d..221e7667e 100644
--- a/moli-protocol/src/domains/network.rs
+++ b/moli-protocol/src/domains/network.rs
@@ -102,7 +102,7 @@ pub(crate) use main_document_progress::{
materialize_loaded_navigation_progress,
materialize_navigation_failure_preserving_committed_document,
materialize_navigation_load_result, record_completed_main_document_response_body,
- record_failed_main_document_response_body,
+ record_failed_main_document_response_body, record_main_document_request_body,
response_stage_main_document_navigation_network_progress,
start_observed_main_document_navigation_progress_background_events,
};
diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rs
index e31846286..8f1b15640 100644
--- a/moli-protocol/src/domains/network/main_document_progress/mod.rs
+++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs
@@ -1444,7 +1444,7 @@ pub(crate) fn start_observed_main_document_navigation_progress_background_events
) -> MainDocumentBodyProgressSource {
let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
record_pending_main_document_response_body(conn, state, &session_ids);
- record_main_document_request_body(conn, state, &session_ids);
+ record_main_document_request_body(conn, state);
if let Some(sender) = conn.background_event_sender()
&& let Some(live_source) =
MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(
@@ -2196,10 +2196,9 @@ fn record_pending_main_document_response_body(
}
}
-fn record_main_document_request_body(
+pub(crate) fn record_main_document_request_body(
conn: &mut CdpConnection,
state: &NavigationDispatchState,
- session_ids: &[Option<String>],
) {
if !main_document_network_observed(conn, state.session_id.as_deref()) {
return;
@@ -2207,27 +2206,37 @@ fn record_main_document_request_body(
let Some(request_id) = state.request_id.clone() else {
return;
};
- let Some(request_body) = state.clone_request_body_bytes() else {
+ let Some(transport_bytes) = state.clone_request_body_bytes() else {
return;
};
+ let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
let data_type = crate::devtools_runtime::DevToolsNetworkDataType::Request;
let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
state.session_id.as_deref(),
data_type,
- request_body.len(),
+ transport_bytes.len(),
);
let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);
+ // The BiDi network data collector keeps the raw transport bytes, including
+ // multipart file payloads.
conn.record_collected_network_data_body(
request_id.clone(),
data_type,
- crate::conn::CapturedBody::from_bytes(request_body.clone()),
+ crate::conn::CapturedBody::from_bytes(transport_bytes.clone()),
collector_ids.iter().cloned(),
collection_was_gated,
);
if let Ok(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
+ // CDP getRequestPostData uses the text projection that request events
+ // and Fetch interception expose, which omits multipart file contents.
+ // Never surface raw transport bytes through the CDP captured store.
+ let cdp_request_body = state
+ .request_body
+ .clone()
+ .unwrap_or_else(|| String::from_utf8_lossy(&transport_bytes).into_owned());
runtime_slot.record_captured_request_body_with_collector_scope(
request_id,
- request_body,
+ cdp_request_body.into_bytes(),
session_ids.iter().cloned(),
collector_ids,
collection_was_gated,
diff --git a/moli-protocol/src/domains/network/tests/response_body.rs b/moli-protocol/src/domains/network/tests/response_body.rs
index 931071e0e..98d622a97 100644
--- a/moli-protocol/src/domains/network/tests/response_body.rs
+++ b/moli-protocol/src/domains/network/tests/response_body.rs
@@ -545,6 +545,111 @@ async fn get_request_post_data_returns_main_document_navigation_post_body() {
);
}
+#[tokio::test(flavor = "multi_thread")]
+async fn get_request_post_data_uses_text_projection_while_bidi_collector_keeps_transport_bytes() {
+ let mut ctx = TestContext::new();
+ let mut bc = BrowserContext::new("BID-1".into());
+ bc.set_active_target_id("TID-1".to_owned());
+ bc.attach_active_session("SID-1".to_owned());
+ bc.active_target
+ .runtime_slot
+ .enable_primary_network_events();
+ ctx.conn.browser_context = Some(bc);
+
+ let (result, _) = ctx
+ .conn
+ .execute_devtools_command(
+ crate::devtools_runtime::DevToolsCommand::AddNetworkDataCollector(
+ crate::devtools_runtime::DevToolsAddNetworkDataCollectorCommand {
+ context: crate::devtools_runtime::DevToolsCommandContext {
+ protocol: crate::devtools_runtime::DevToolsProtocol::WebDriverBidi,
+ session_id: Some(crate::devtools_runtime::DevToolsSessionId::from("SID-1")),
+ target_id: None,
+ browser_context_id: None,
+ },
+ collector_id: crate::devtools_runtime::DevToolsNetworkDataCollectorId::from(
+ "collector-multipart-request-bytes",
+ ),
+ data_types: vec![crate::devtools_runtime::DevToolsNetworkDataType::Request],
+ max_encoded_data_size: 1000,
+ target_ids: Vec::new(),
+ browser_context_ids: Vec::new(),
+ },
+ ),
+ )
+ .await
+ .into_parts();
+ assert!(matches!(
+ result,
+ Ok(crate::devtools_runtime::DevToolsCommandResult::AddNetworkDataCollector(_))
+ ));
+
+ let multipart_text = "----MoliFormDataBoundary0000\r\nContent-Disposition: form-data; name=\"field\"\r\n\r\nvalue\r\n----MoliFormDataBoundary0000\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a.bin\"\r\nContent-Type: application/octet-stream\r\n\r\n";
+ let file_bytes: Vec<u8> = vec![0x00, 0xff, 0x01, 0xfe, b'a', b'b'];
+ let mut transport_bytes = multipart_text.as_bytes().to_vec();
+ transport_bytes.extend_from_slice(&file_bytes);
+ transport_bytes.extend_from_slice(b"\r\n----MoliFormDataBoundary0000--\r\n");
+
+ let requested_url = Url::parse("http://127.0.0.1:1/upload").unwrap();
+ let navigation_state = NavigationDispatchState {
+ navigate_id: Some(1),
+ navigate_session_id: Some("SID-1".to_owned()),
+ result_projection: crate::conn::NavigationResultProjection::Cdp(
+ json!({"frameId": "TID-1", "loaderId": LOADER_ID}),
+ ),
+ frame_id: "TID-1".to_owned(),
+ session_id: Some("SID-1".to_owned()),
+ request_id: Some("REQ-multipart".to_owned()),
+ loader_id: LOADER_ID.to_owned(),
+ request_announced: false,
+ requested_url,
+ request_method: "POST".to_owned(),
+ request_body: Some(multipart_text.to_owned()),
+ request_body_bytes: Some(transport_bytes.clone()),
+ request_headers: Vec::new(),
+ request_load_policy: crate::conn::NavigationRequestLoadPolicy::DocumentInitiated,
+ timestamp: 0.0,
+ source_document_security: Default::default(),
+ };
+
+ let mut events = Vec::new();
+ start_observed_main_document_navigation_progress_background_events(
+ &mut ctx.conn,
+ &mut events,
+ &navigation_state,
+ None,
+ );
+
+ ctx.process_async(json!({
+ "id": 7_291,
+ "method": "Network.getRequestPostData",
+ "sessionId": "SID-1",
+ "params": { "requestId": "REQ-multipart" }
+ }))
+ .await;
+ ctx.expect_result(
+ 7_291,
+ json!({ "postData": multipart_text, "base64Encoded": false }),
+ Some("SID-1"),
+ );
+
+ let collected = ctx
+ .conn
+ .network_data_collectors
+ .collected_body(
+ "REQ-multipart",
+ crate::devtools_runtime::DevToolsNetworkDataType::Request,
+ )
+ .expect("BiDi collector should retain the request body");
+ assert_eq!(
+ collected
+ .body_bytes_limited(usize::MAX)
+ .expect("collected body"),
+ transport_bytes,
+ "the BiDi collector must keep the raw transport bytes including file payload"
+ );
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn get_network_data_returns_bidi_response_body_bytes() {
let mut ctx = TestContext::new();