Skip to content

armgit / moli

Public Imported from GitHub
0

Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust

fix(network): make main-document POST bodies available to getRequestPostData

BibekPathak9eb226adfc48
diff --git a/moli-protocol/src/domains/network/main_document_progress/mod.rs b/moli-protocol/src/domains/network/main_document_progress/mod.rs
index 796866d0f..e31846286 100644
--- a/moli-protocol/src/domains/network/main_document_progress/mod.rs
+++ b/moli-protocol/src/domains/network/main_document_progress/mod.rs
@@ -1444,6 +1444,7 @@ pub(crate) fn start_observed_main_document_navigation_progress_background_events
) -> MainDocumentBodyProgressSource {
let session_ids = main_document_network_event_session_ids(conn, state.session_id.as_deref());
record_pending_main_document_response_body(conn, state, &session_ids);
+ record_main_document_request_body(conn, state, &session_ids);
if let Some(sender) = conn.background_event_sender()
&& let Some(live_source) =
MainDocumentLiveNetworkProgressSource::from_navigation_dispatch_state(
@@ -2195,6 +2196,45 @@ fn record_pending_main_document_response_body(
}
}
+fn record_main_document_request_body(
+ conn: &mut CdpConnection,
+ state: &NavigationDispatchState,
+ session_ids: &[Option<String>],
+) {
+ if !main_document_network_observed(conn, state.session_id.as_deref()) {
+ return;
+ }
+ let Some(request_id) = state.request_id.clone() else {
+ return;
+ };
+ let Some(request_body) = state.clone_request_body_bytes() else {
+ return;
+ };
+ let data_type = crate::devtools_runtime::DevToolsNetworkDataType::Request;
+ let collector_ids = conn.network_data_collector_ids_for_session_owner_body(
+ state.session_id.as_deref(),
+ data_type,
+ request_body.len(),
+ );
+ let collection_was_gated = conn.network_data_collection_is_gated_for_body(data_type);
+ conn.record_collected_network_data_body(
+ request_id.clone(),
+ data_type,
+ crate::conn::CapturedBody::from_bytes(request_body.clone()),
+ collector_ids.iter().cloned(),
+ collection_was_gated,
+ );
+ if let Ok(runtime_slot) = conn.runtime_session_owner_slot_mut(state.session_id.as_deref()) {
+ runtime_slot.record_captured_request_body_with_collector_scope(
+ request_id,
+ request_body,
+ session_ids.iter().cloned(),
+ collector_ids,
+ collection_was_gated,
+ );
+ }
+}
+
pub(crate) fn record_failed_main_document_response_body(
conn: &mut CdpConnection,
state: &NavigationDispatchState,
diff --git a/moli-protocol/src/domains/network/tests/response_body.rs b/moli-protocol/src/domains/network/tests/response_body.rs
index 01ce12fc5..931071e0e 100644
--- a/moli-protocol/src/domains/network/tests/response_body.rs
+++ b/moli-protocol/src/domains/network/tests/response_body.rs
@@ -10,6 +10,7 @@ use crate::domains::network::{
PendingSubresourceNetworkActivitySession, TargetNetworkBacklogRequestIdResolver,
TargetNetworkOutputQueue, TargetSubresourcePlanOutput,
emit_pending_network_backlog_activity_background_events,
+ start_observed_main_document_navigation_progress_background_events,
};
use super::*;
@@ -489,6 +490,61 @@ async fn get_request_post_data_respects_recorded_session_visibility() {
);
}
+#[tokio::test(flavor = "multi_thread")]
+async fn get_request_post_data_returns_main_document_navigation_post_body() {
+ let mut ctx = TestContext::new();
+ let mut bc = BrowserContext::new("BID-1".into());
+ bc.set_active_target_id("TID-1".to_owned());
+ bc.attach_active_session("SID-1".to_owned());
+ bc.active_target
+ .runtime_slot
+ .enable_primary_network_events();
+ ctx.conn.browser_context = Some(bc);
+
+ let requested_url = Url::parse("http://127.0.0.1:1/post").unwrap();
+ let navigation_state = NavigationDispatchState {
+ navigate_id: Some(1),
+ navigate_session_id: Some("SID-1".to_owned()),
+ result_projection: crate::conn::NavigationResultProjection::Cdp(
+ json!({"frameId": "TID-1", "loaderId": LOADER_ID}),
+ ),
+ frame_id: "TID-1".to_owned(),
+ session_id: Some("SID-1".to_owned()),
+ request_id: Some("REQ-main-post".to_owned()),
+ loader_id: LOADER_ID.to_owned(),
+ request_announced: false,
+ requested_url,
+ request_method: "POST".to_owned(),
+ request_body: Some("username=alice&pw=s3cret".to_owned()),
+ request_body_bytes: Some(b"username=alice&pw=s3cret".to_vec()),
+ request_headers: Vec::new(),
+ request_load_policy: crate::conn::NavigationRequestLoadPolicy::DocumentInitiated,
+ timestamp: 0.0,
+ source_document_security: Default::default(),
+ };
+
+ let mut events = Vec::new();
+ start_observed_main_document_navigation_progress_background_events(
+ &mut ctx.conn,
+ &mut events,
+ &navigation_state,
+ None,
+ );
+
+ ctx.process_async(json!({
+ "id": 7_290,
+ "method": "Network.getRequestPostData",
+ "sessionId": "SID-1",
+ "params": { "requestId": "REQ-main-post" }
+ }))
+ .await;
+ ctx.expect_result(
+ 7_290,
+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),
+ Some("SID-1"),
+ );
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn get_network_data_returns_bidi_response_body_bytes() {
let mut ctx = TestContext::new();
diff --git a/moli-protocol/src/domains/network/tests/runtime.rs b/moli-protocol/src/domains/network/tests/runtime.rs
index 3c5dc31af..7606b98b0 100644
--- a/moli-protocol/src/domains/network/tests/runtime.rs
+++ b/moli-protocol/src/domains/network/tests/runtime.rs
@@ -308,6 +308,100 @@ async fn runtime_fetch_post_body_is_available_by_network_request_id() {
server.abort();
}
+#[tokio::test(flavor = "multi_thread")]
+async fn runtime_form_post_navigation_body_is_available_by_network_request_id() {
+ async fn page() -> impl IntoResponse {
+ (
+ [(CONTENT_TYPE.as_str(), "text/html")],
+ r#"<!doctype html><form id="f" method="POST" action="/post"><input name="username" value="alice"><input name="pw" value="s3cret"></form>"#,
+ )
+ }
+
+ async fn post_body(body: String) -> impl IntoResponse {
+ body
+ }
+
+ let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let addr = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move {
+ axum::serve(
+ listener,
+ Router::new()
+ .route("/page", get(page))
+ .route("/post", post(post_body)),
+ )
+ .await
+ .unwrap();
+ });
+
+ let page_url = format!("http://{addr}/page");
+ let mut ctx = TestContext::new();
+ let mut bc = BrowserContext::new("BID-1".into());
+ bc.attach_active_session("SID-1".to_owned());
+ bc.set_active_target_id("TID-1".to_owned());
+ ctx.conn.browser_context = Some(bc);
+ ctx.install_navigation_fixture_for_session_owner(&page_url, Some("SID-1"))
+ .await;
+ ctx.sent.clear();
+
+ ctx.process_async(json!({
+ "id": 10_210,
+ "method": "Network.enable",
+ "sessionId": "SID-1"
+ }))
+ .await;
+ ctx.expect_result(10_210, json!({}), Some("SID-1"));
+
+ ctx.process_async(json!({
+ "id": 10_211,
+ "method": "Runtime.evaluate",
+ "sessionId": "SID-1",
+ "params": {
+ "expression": "document.getElementById('f').submit(); 'scheduled'"
+ }
+ }))
+ .await;
+ let _ = ctx.take_response_by_id(10_211);
+
+ wait_until_messages(
+ &mut ctx,
+ "SID-1",
+ "POST navigation requestWillBeSent",
+ |messages| {
+ messages.iter().any(|message| {
+ message["method"] == json!("Network.requestWillBeSent")
+ && message["params"]["request"]["method"] == json!("POST")
+ })
+ },
+ )
+ .await;
+ let request_id = ctx
+ .sent
+ .iter()
+ .find(|message| {
+ message["method"] == json!("Network.requestWillBeSent")
+ && message["params"]["request"]["method"] == json!("POST")
+ })
+ .and_then(|message| message["params"]["requestId"].as_str())
+ .expect("POST navigation request id")
+ .to_owned();
+
+ ctx.process_async(json!({
+ "id": 10_212,
+ "method": "Network.getRequestPostData",
+ "sessionId": "SID-1",
+ "params": { "requestId": request_id }
+ }))
+ .await;
+ ctx.expect_result(
+ 10_212,
+ json!({ "postData": "username=alice&pw=s3cret", "base64Encoded": false }),
+ Some("SID-1"),
+ );
+
+ server.abort();
+}
+
#[tokio::test(flavor = "multi_thread")]
async fn websocket_runtime_activity_emits_cdp_websocket_events_without_payload() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();