Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(network): make main-document POST bodies available to getRequestPostData
BibekPathak9eb226adfc48
diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rsindex 796866d0f..e31846286 100644--- a/moli-protocol/src/domains/network/main_document_progress/mod.rs+++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs@@ -1444,6 +1444,7 @@ pub(crate) fn start_observed_main_document_navigation_progress_background_events) -> MainDocumentBodyProgressSource {let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());record_pending_main_document_response_body(conn, state, &session_ids);+ record_main_document_request_body(conn, state, &session_ids);if let Some(sender) = conn.background_event_sender()&& let Some(live_source) =MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(@@ -2195,6 +2196,45 @@ fn record_pending_main_document_response_body(}}+fn record_main_document_request_body(+ conn: &mut CdpConnection,+ state: &NavigationDispatchState,+ session_ids: &[Option<String>],+) {+ if !main_document_network_observed(conn, state.session_id.as_deref()) {+ return;+ }+ let Some(request_id) = state.request_id.clone() else {+ return;+ };+ let Some(request_body) = state.clone_request_body_bytes() else {+ return;+ };+ let data_type = crate::devtools_runtime::DevToolsNetworkDataType::Request;+ let collector_ids = conn.network_data_collector_ids_for_session_owner_body(+ state.session_id.as_deref(),+ data_type,+ request_body.len(),+ );+ let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);+ conn.record_collected_network_data_body(+ request_id.clone(),+ data_type,+ crate::conn::CapturedBody::from_bytes(request_body.clone()),+ collector_ids.iter().cloned(),+ collection_was_gated,+ );+ if let Ok(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {+ runtime_slot.record_captured_request_body_with_collector_scope(+ request_id,+ request_body,+ session_ids.iter().cloned(),+ collector_ids,+ collection_was_gated,+ );+ }+}+pub(crate) fn record_failed_main_document_response_body(conn: &mut CdpConnection,state: &NavigationDispatchState,diff --git a/moli-protocol/src/domains/network/tests/response_body.rs b/moli-protocol/src/domains/network/tests/response_body.rsindex 01ce12fc5..931071e0e 100644--- a/moli-protocol/src/domains/network/tests/response_body.rs+++ b/moli-protocol/src/domains/network/tests/response_body.rs@@ -10,6 +10,7 @@ use crate::domains::network::{PendingSubresourceNetworkActivitySession, TargetNetworkBacklogRequestIdResolver,TargetNetworkOutputQueue, TargetSubresourcePlanOutput,emit_pending_network_backlog_activity_background_events,+ start_observed_main_document_navigation_progress_background_events,};use super::*;@@ -489,6 +490,61 @@ async fn get_request_post_data_respects_recorded_session_visibility() {);}+#[tokio::test(flavor = "multi_thread")]+async fn get_request_post_data_returns_main_document_navigation_post_body() {+ let mut ctx = TestContext::new();+ let mut bc = BrowserContext::new("BID-1".into());+ bc.set_active_target_id("TID-1".to_owned());+ bc.attach_active_session("SID-1".to_owned());+ bc.active_target+ .runtime_slot+ .enable_primary_network_events();+ ctx.conn.browser_context = Some(bc);++ let requested_url = Url::parse("http://127.0.0.1:1/post").unwrap();+ let navigation_state = NavigationDispatchState {+ navigate_id: Some(1),+ navigate_session_id: Some("SID-1".to_owned()),+ result_projection: crate::conn::NavigationResultProjection::Cdp(+ json!({"frameId": "TID-1", "loaderId": LOADER_ID}),+ ),+ frame_id: "TID-1".to_owned(),+ session_id: Some("SID-1".to_owned()),+ request_id: Some("REQ-main-post".to_owned()),+ loader_id: LOADER_ID.to_owned(),+ request_announced: false,+ requested_url,+ request_method: "POST".to_owned(),+ request_body: Some("username=alice&pw=s3cret".to_owned()),+ request_body_bytes: Some(b"username=alice&pw=s3cret".to_vec()),+ request_headers: Vec::new(),+ request_load_policy: crate::conn::NavigationRequestLoadPolicy::DocumentInitiated,+ timestamp: 0.0,+ source_document_security: Default::default(),+ };++ let mut events = Vec::new();+ start_observed_main_document_navigation_progress_background_events(+ &mut ctx.conn,+ &mut events,+ &navigation_state,+ None,+ );++ ctx.process_async(json!({+ "id": 7_290,+ "method": "Network.getRequestPostData",+ "sessionId": "SID-1",+ "params": { "requestId": "REQ-main-post" }+ }))+ .await;+ ctx.expect_result(+ 7_290,+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),+ Some("SID-1"),+ );+}+#[tokio::test(flavor = "multi_thread")]async fn get_network_data_returns_bidi_response_body_bytes() {let mut ctx = TestContext::new();diff --git a/moli-protocol/src/domains/network/tests/runtime.rs b/moli-protocol/src/domains/network/tests/runtime.rsindex 3c5dc31af..7606b98b0 100644--- a/moli-protocol/src/domains/network/tests/runtime.rs+++ b/moli-protocol/src/domains/network/tests/runtime.rs@@ -308,6 +308,100 @@ async fn runtime_fetch_post_body_is_available_by_network_request_id() {server.abort();}+#[tokio::test(flavor = "multi_thread")]+async fn runtime_form_post_navigation_body_is_available_by_network_request_id() {+ async fn page() -> impl IntoResponse {+ (+ [(CONTENT_TYPE.as_str(), "text/html")],+ r#"<!doctype html><form id="f" method="POST" action="/post"><input name="username" value="alice"><input name="pw" value="s3cret"></form>"#,+ )+ }++ async fn post_body(body: String) -> impl IntoResponse {+ body+ }++ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();+ let addr = listener.local_addr().unwrap();+ let server = tokio::spawn(async move {+ axum::serve(+ listener,+ Router::new()+ .route("/page", get(page))+ .route("/post", post(post_body)),+ )+ .await+ .unwrap();+ });++ let page_url = format!("http://{addr}/page");+ let mut ctx = TestContext::new();+ let mut bc = BrowserContext::new("BID-1".into());+ bc.attach_active_session("SID-1".to_owned());+ bc.set_active_target_id("TID-1".to_owned());+ ctx.conn.browser_context = Some(bc);+ ctx.install_navigation_fixture_for_session_owner(&page_url, Some("SID-1"))+ .await;+ ctx.sent.clear();++ ctx.process_async(json!({+ "id": 10_210,+ "method": "Network.enable",+ "sessionId": "SID-1"+ }))+ .await;+ ctx.expect_result(10_210, json!({}), Some("SID-1"));++ ctx.process_async(json!({+ "id": 10_211,+ "method": "Runtime.evaluate",+ "sessionId": "SID-1",+ "params": {+ "expression": "document.getElementById('f').submit(); 'scheduled'"+ }+ }))+ .await;+ let _ = ctx.take_response_by_id(10_211);++ wait_until_messages(+ &mut ctx,+ "SID-1",+ "POST navigation requestWillBeSent",+ |messages| {+ messages.iter().any(|message| {+ message["method"] == json!("Network.requestWillBeSent")+ && message["params"]["request"]["method"] == json!("POST")+ })+ },+ )+ .await;+ let request_id = ctx+ .sent+ .iter()+ .find(|message| {+ message["method"] == json!("Network.requestWillBeSent")+ && message["params"]["request"]["method"] == json!("POST")+ })+ .and_then(|message| message["params"]["requestId"].as_str())+ .expect("POST navigation request id")+ .to_owned();++ ctx.process_async(json!({+ "id": 10_212,+ "method": "Network.getRequestPostData",+ "sessionId": "SID-1",+ "params": { "requestId": request_id }+ }))+ .await;+ ctx.expect_result(+ 10_212,+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),+ Some("SID-1"),+ );++ server.abort();+}+#[tokio::test(flavor = "multi_thread")]async fn websocket_runtime_activity_emits_cdp_websocket_events_without_payload() {let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();