Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(network): capture main-document POST bodies across Fetch interception
BibekPathak7f6c7634b636
diff --git a/moli-protocol/src/domains/fetch/navigation.rs b/moli-protocol/src/domains/fetch/navigation.rsindex 33d2dcd43..781ebb3e4 100644--- a/moli-protocol/src/domains/fetch/navigation.rs+++ b/moli-protocol/src/domains/fetch/navigation.rs@@ -65,6 +65,7 @@ pub(crate) async fn load_or_pause_navigation_for_auth_into_buffer_async(prior_network_observation_journal: Option<NetworkObservationJournal>,) {let navigate_session_id = pending.navigation.navigate_session_id.clone();+ network::record_main_document_request_body(conn, &pending.navigation);let should_handle_auth = conn.target_fetch_matches_auth_required_for_session_owner(navigate_session_id.as_deref(),&pending.navigation.requested_url,diff --git a/moli-protocol/src/domains/fetch/tests/navigation_control.rs b/moli-protocol/src/domains/fetch/tests/navigation_control.rsindex afa0f4020..f79406a43 100644--- a/moli-protocol/src/domains/fetch/tests/navigation_control.rs+++ b/moli-protocol/src/domains/fetch/tests/navigation_control.rs@@ -649,6 +649,155 @@ async fn continue_request_with_post_data_marks_network_request_as_having_post_daSome("<!doctype html><html><body><main>payload</main></body></html>".to_owned()));+ ctx.process_async(json!({+ "id": 343,+ "method": "Network.getRequestPostData",+ "sessionId": "SID-1",+ "params": { "requestId": LOADER_ID }+ }))+ .await;+ ctx.expect_result(+ 343,+ json!({ "postData": "payload", "base64Encoded": false }),+ Some("SID-1"),+ );++ server.abort();+}++#[tokio::test(flavor = "multi_thread")]+async fn intercepted_form_post_navigation_body_is_available_by_network_request_id() {+ async fn page() -> impl IntoResponse {+ (+ [(CONTENT_TYPE.as_str(), "text/html")],+ r#"<!doctype html><form id="f" method="POST" action="/post"><input name="username" value="alice"><input name="pw" value="s3cret"></form>"#,+ )+ }++ async fn post_body(body: String) -> impl IntoResponse {+ (+ [(CONTENT_TYPE.as_str(), "text/html")],+ format!("<!doctype html><html><body><main>{body}</main></body></html>"),+ )+ }++ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ Router::new()+ .route("/page", get(page))+ .route("/post", axum::routing::post(post_body)),+ )+ .await+ .unwrap();+ });++ let page_url = format!("http://{addr}/page");+ let post_url = format!("http://{addr}/post");+ let mut ctx = TestContext::new();+ ctx.conn.browser_context = Some(attached_browser_context());+ ctx.install_navigation_fixture_for_session_owner(&page_url, Some("SID-1"))+ .await;+ {+ let bc = ctx.conn.browser_context.as_mut().expect("browser context");+ bc.active_target+ .runtime_slot+ .enable_primary_network_events();+ }+ ctx.sent.clear();++ ctx.process_async(json!({+ "id": 420,+ "method": "Fetch.enable",+ "sessionId": "SID-1",+ "params": {+ "patterns": [{ "urlPattern": "*/post", "requestStage": "Request", "resourceType": "Document" }]+ }+ }))+ .await;+ ctx.expect_result(420, json!({}), Some("SID-1"));+ ctx.sent.clear();++ ctx.process_async(json!({+ "id": 421,+ "method": "Runtime.evaluate",+ "sessionId": "SID-1",+ "params": {+ "expression": "document.getElementById('f').submit(); 'scheduled'"+ }+ }))+ .await;+ let _ = take_response_by_id(&mut ctx, 421);++ wait_until_scheduler_message(+ &mut ctx,+ "intercepted form POST Fetch.requestPaused",+ |message| {+ message["method"] == json!("Fetch.requestPaused")+ && message["params"]["resourceType"] == json!("Document")+ && message["params"]["request"]["url"] == json!(post_url)+ },+ )+ .await;+ let paused = ctx+ .sent+ .iter()+ .find(|message| {+ message["method"] == json!("Fetch.requestPaused")+ && message["params"]["resourceType"] == json!("Document")+ && message["params"]["request"]["url"] == json!(post_url)+ })+ .cloned()+ .expect("intercepted form POST pause");+ assert_eq!(paused["params"]["request"]["method"], json!("POST"));+ assert_eq!(+ paused["params"]["request"]["postData"],+ json!("username=alice&pw=s3cret"),+ "the paused form POST must carry its original body"+ );+ let fetch_request_id = paused["params"]["requestId"]+ .as_str()+ .expect("fetch request id")+ .to_owned();+ let network_request_id = paused["params"]["networkId"]+ .as_str()+ .expect("network request id")+ .to_owned();++ ctx.process_async(json!({+ "id": 422,+ "method": "Fetch.continueRequest",+ "sessionId": "SID-1",+ "params": { "requestId": fetch_request_id }+ }))+ .await;+ ctx.expect_result(422, json!({}), Some("SID-1"));++ wait_until_scheduler_message(+ &mut ctx,+ "intercepted form POST responseReceived",+ |message| {+ message["method"] == json!("Network.responseReceived")+ && message["params"]["requestId"] == json!(network_request_id)+ },+ )+ .await;++ ctx.process_async(json!({+ "id": 423,+ "method": "Network.getRequestPostData",+ "sessionId": "SID-1",+ "params": { "requestId": network_request_id }+ }))+ .await;+ ctx.expect_result(+ 423,+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),+ Some("SID-1"),+ );+server.abort();}diff --git a/moli-protocol/src/domains/network.rs b/moli-protocol/src/domains/network.rsindex a400ace2d..221e7667e 100644--- a/moli-protocol/src/domains/network.rs+++ b/moli-protocol/src/domains/network.rs@@ -102,7 +102,7 @@ pub(crate) use main_document_progress::{materialize_loaded_navigation_progress,materialize_navigation_failure_preserving_committed_document,materialize_navigation_load_result, record_completed_main_document_response_body,- record_failed_main_document_response_body,+ record_failed_main_document_response_body, record_main_document_request_body,response_stage_main_document_navigation_network_progress,start_observed_main_document_navigation_progress_background_events,};diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rsindex e31846286..8f1b15640 100644--- a/moli-protocol/src/domains/network/main_document_progress/mod.rs+++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs@@ -1444,7 +1444,7 @@ pub(crate) fn start_observed_main_document_navigation_progress_background_events) -> MainDocumentBodyProgressSource {let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());record_pending_main_document_response_body(conn, state, &session_ids);- record_main_document_request_body(conn, state, &session_ids);+ record_main_document_request_body(conn, state);if let Some(sender) = conn.background_event_sender()&& let Some(live_source) =MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(@@ -2196,10 +2196,9 @@ fn record_pending_main_document_response_body(}}-fn record_main_document_request_body(+pub(crate) fn record_main_document_request_body(conn: &mut CdpConnection,state: &NavigationDispatchState,- session_ids: &[Option<String>],) {if !main_document_network_observed(conn, state.session_id.as_deref()) {return;@@ -2207,27 +2206,37 @@ fn record_main_document_request_body(let Some(request_id) = state.request_id.clone() else {return;};- let Some(request_body) = state.clone_request_body_bytes() else {+ let Some(transport_bytes) = state.clone_request_body_bytes() else {return;};+ let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());let data_type = crate::devtools_runtime::DevToolsNetworkDataType::Request;let collector_ids = conn.network_data_collector_ids_for_session_owner_body(state.session_id.as_deref(),data_type,- request_body.len(),+ transport_bytes.len(),);let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);+ // The BiDi network data collector keeps the raw transport bytes, including+ // multipart file payloads.conn.record_collected_network_data_body(request_id.clone(),data_type,- crate::conn::CapturedBody::from_bytes(request_body.clone()),+ crate::conn::CapturedBody::from_bytes(transport_bytes.clone()),collector_ids.iter().cloned(),collection_was_gated,);if let Ok(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {+ // CDP getRequestPostData uses the text projection that request events+ // and Fetch interception expose, which omits multipart file contents.+ // Never surface raw transport bytes through the CDP captured store.+ let cdp_request_body = state+ .request_body+ .clone()+ .unwrap_or_else(|| String::from_utf8_lossy(&transport_bytes).into_owned());runtime_slot.record_captured_request_body_with_collector_scope(request_id,- request_body,+ cdp_request_body.into_bytes(),session_ids.iter().cloned(),collector_ids,collection_was_gated,diff --git a/moli-protocol/src/domains/network/tests/response_body.rs b/moli-protocol/src/domains/network/tests/response_body.rsindex 931071e0e..98d622a97 100644--- a/moli-protocol/src/domains/network/tests/response_body.rs+++ b/moli-protocol/src/domains/network/tests/response_body.rs@@ -545,6 +545,111 @@ async fn get_request_post_data_returns_main_document_navigation_post_body() {);}+#[tokio::test(flavor = "multi_thread")]+async fn get_request_post_data_uses_text_projection_while_bidi_collector_keeps_transport_bytes() {+ let mut ctx = TestContext::new();+ let mut bc = BrowserContext::new("BID-1".into());+ bc.set_active_target_id("TID-1".to_owned());+ bc.attach_active_session("SID-1".to_owned());+ bc.active_target+ .runtime_slot+ .enable_primary_network_events();+ ctx.conn.browser_context = Some(bc);++ let (result, _) = ctx+ .conn+ .execute_devtools_command(+ crate::devtools_runtime::DevToolsCommand::AddNetworkDataCollector(+ crate::devtools_runtime::DevToolsAddNetworkDataCollectorCommand {+ context: crate::devtools_runtime::DevToolsCommandContext {+ protocol: crate::devtools_runtime::DevToolsProtocol::WebDriverBidi,+ session_id: Some(crate::devtools_runtime::DevToolsSessionId::from("SID-1")),+ target_id: None,+ browser_context_id: None,+ },+ collector_id: crate::devtools_runtime::DevToolsNetworkDataCollectorId::from(+ "collector-multipart-request-bytes",+ ),+ data_types: vec![crate::devtools_runtime::DevToolsNetworkDataType::Request],+ max_encoded_data_size: 1000,+ target_ids: Vec::new(),+ browser_context_ids: Vec::new(),+ },+ ),+ )+ .await+ .into_parts();+ assert!(matches!(+ result,+ Ok(crate::devtools_runtime::DevToolsCommandResult::AddNetworkDataCollector(_))+ ));++ let multipart_text = "----MoliFormDataBoundary0000\r\nContent-Disposition: form-data; name=\"field\"\r\n\r\nvalue\r\n----MoliFormDataBoundary0000\r\nContent-Disposition: form-data; name=\"file\"; filename=\"a.bin\"\r\nContent-Type: application/octet-stream\r\n\r\n";+ let file_bytes: Vec<u8> = vec![0x00, 0xff, 0x01, 0xfe, b'a', b'b'];+ let mut transport_bytes = multipart_text.as_bytes().to_vec();+ transport_bytes.extend_from_slice(&file_bytes);+ transport_bytes.extend_from_slice(b"\r\n----MoliFormDataBoundary0000--\r\n");++ let requested_url = Url::parse("http://127.0.0.1:1/upload").unwrap();+ let navigation_state = NavigationDispatchState {+ navigate_id: Some(1),+ navigate_session_id: Some("SID-1".to_owned()),+ result_projection: crate::conn::NavigationResultProjection::Cdp(+ json!({"frameId": "TID-1", "loaderId": LOADER_ID}),+ ),+ frame_id: "TID-1".to_owned(),+ session_id: Some("SID-1".to_owned()),+ request_id: Some("REQ-multipart".to_owned()),+ loader_id: LOADER_ID.to_owned(),+ request_announced: false,+ requested_url,+ request_method: "POST".to_owned(),+ request_body: Some(multipart_text.to_owned()),+ request_body_bytes: Some(transport_bytes.clone()),+ request_headers: Vec::new(),+ request_load_policy: crate::conn::NavigationRequestLoadPolicy::DocumentInitiated,+ timestamp: 0.0,+ source_document_security: Default::default(),+ };++ let mut events = Vec::new();+ start_observed_main_document_navigation_progress_background_events(+ &mut ctx.conn,+ &mut events,+ &navigation_state,+ None,+ );++ ctx.process_async(json!({+ "id": 7_291,+ "method": "Network.getRequestPostData",+ "sessionId": "SID-1",+ "params": { "requestId": "REQ-multipart" }+ }))+ .await;+ ctx.expect_result(+ 7_291,+ json!({ "postData": multipart_text, "base64Encoded": false }),+ Some("SID-1"),+ );++ let collected = ctx+ .conn+ .network_data_collectors+ .collected_body(+ "REQ-multipart",+ crate::devtools_runtime::DevToolsNetworkDataType::Request,+ )+ .expect("BiDi collector should retain the request body");+ assert_eq!(+ collected+ .body_bytes_limited(usize::MAX)+ .expect("collected body"),+ transport_bytes,+ "the BiDi collector must keep the raw transport bytes including file payload"+ );+}+#[tokio::test(flavor = "multi_thread")]async fn get_network_data_returns_bidi_response_body_bytes() {let mut ctx = TestContext::new();