The headless browser for AI agents and web scraping
fix(cdp): service routed requests during awaited evaluations
SGavrl5e58c8673729
diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rsindex 8603184d..b263f729 100644--- a/crates/obscura-cdp/src/server.rs+++ b/crates/obscura-cdp/src/server.rs@@ -1344,11 +1344,20 @@ async fn cdp_processor(&mut intercept_rx, &mut intercepted_paused,&mut deferred, true,).await;+ } else if ctx.pages.iter().any(|page| page.intercept_enabled)+ && serde_json::from_str::<CdpRequest>(&cdp_msg.text).is_ok_and(|req| {+ matches!(req.method.as_str(), "Runtime.evaluate" | "Runtime.callFunctionOn")+ && req.params.get("awaitPromise").and_then(|v| v.as_bool()) == Some(true)+ })+ {+ process_awaited_with_interception(+ &cdp_msg.text, &mut ctx, &cdp_msg.reply_tx, &mut rx,+ &mut intercept_rx, &mut intercepted_paused, &mut deferred,+ ).await;} else {let fetch_was_resolved = cdp_msg.text.contains("Fetch.")&& handle_fetch_resolution(&cdp_msg.text,- &mut ctx,&cdp_msg.reply_tx,&mut intercepted_paused,);@@ -1408,6 +1417,15 @@ fn emit_intercepted_request(let (session_id, _) = ctx.sessions.iter().find(|(_, page_id)| **page_id == page.id)?;Some((page.frame_id.as_str(), Some(session_id.clone())))});+ emit_routed_intercepted_request(intercepted, route, reply_tx, intercepted_paused);+}++fn emit_routed_intercepted_request(+ intercepted: obscura_js::ops::InterceptedRequest,+ route: Option<(&str, Option<String>)>,+ reply_tx: &mpsc::UnboundedSender<String>,+ intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,+) {let Some((frame_id, session_id)) = route else {let _ = intercepted.resolver.send(obscura_js::ops::InterceptResolution::Fail {reason: "Aborted".into(),@@ -1658,12 +1676,14 @@ pub(crate) fn parse_cdp_headers(params: &serde_json::Value) -> Option<HashMap<Stfn handle_fetch_resolution(text: &str,- _ctx: &mut CdpContext,reply_tx: &mpsc::UnboundedSender<String>,intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,) -> bool {if let Ok(req) = serde_json::from_str::<CdpRequest>(text) {let method = req.method.as_str();+ if !matches!(method, "Fetch.continueRequest" | "Fetch.fulfillRequest" | "Fetch.failRequest") {+ return false;+ }let request_id = req.params.get("requestId").and_then(|v| v.as_str()).unwrap_or("");tracing::info!("INTERCEPTION resolution: {} for {}, paused_count={}", method, request_id, intercepted_paused.len());@@ -1702,7 +1722,21 @@ fn handle_fetch_resolution(}_ => return false,};- let _ = resolver.send(resolution);+ if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {+ let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");+ let mut error_text = String::from("net::ERR_");+ for (index, ch) in reason.chars().enumerate() {+ if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }+ error_text.push(ch.to_ascii_uppercase());+ }+ let _ = reply_tx.send(json!({+ "method": "Network.loadingFailed", "sessionId": req.session_id,+ "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,+ "canceled": reason == "Aborted",+ "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default().as_secs_f64()},+ }).to_string());+ }let resp = crate::types::CdpResponse::success(req.id, json!({}), req.session_id);if let Ok(json) = serde_json::to_string(&resp) {let _ = reply_tx.send(json);@@ -1888,7 +1922,7 @@ async fn process_with_interception(// op inside the spawned nav task. No V8 enter on// this side; the actual V8 work happens back on// the nav task's thread.- handle_fetch_resolution(&msg.text, ctx, &msg.reply_tx, intercepted_paused);+ handle_fetch_resolution(&msg.text, &msg.reply_tx, intercepted_paused);} else {// UNSAFE during nav: would route through dispatch,// which can `suspend_js` other pages and trip the@@ -1980,6 +2014,59 @@ async fn process_with_interception(}}+async fn process_awaited_with_interception(+ text: &str,+ ctx: &mut CdpContext,+ reply_tx: &mpsc::UnboundedSender<String>,+ rx: &mut mpsc::UnboundedReceiver<ServerMessage>,+ intercept_rx: &mut Option<mpsc::UnboundedReceiver<obscura_js::ops::InterceptedRequest>>,+ intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,+ deferred: &mut std::collections::VecDeque<ServerMessage>,+) {+ // The evaluation owns ctx (and V8) until it completes. Snapshot only routing+ // metadata so request replies can unblock it without entering another isolate.+ let routes: HashMap<_, _> = ctx.pages.iter().filter_map(|page| {+ let (session_id, _) = ctx.sessions.iter().find(|(_, page_id)| **page_id == page.id)?;+ Some((page.id.clone(), (page.frame_id.clone(), session_id.clone())))+ }).collect();+ let command = process_cdp_message(text, ctx, reply_tx);+ tokio::pin!(command);+ loop {+ tokio::select! {+ () = &mut command => break,+ Some(intercepted) = async {+ match intercept_rx.as_mut() {+ Some(receiver) => receiver.recv().await,+ None => std::future::pending().await,+ }+ } => {+ let route = routes.get(&intercepted.page_id)+ .map(|(frame_id, session_id)| (frame_id.as_str(), Some(session_id.clone())));+ emit_routed_intercepted_request(intercepted, route, reply_tx, intercepted_paused);+ }+ Some(message) = rx.recv() => {+ if let ServerMessage::Cdp(msg) = &message {+ if handle_fetch_resolution(&msg.text, &msg.reply_tx, intercepted_paused) {+ continue;+ }+ if deferred.len() >= MAX_DEFERRED_MESSAGES {+ if let Ok(req) = serde_json::from_str::<CdpRequest>(&msg.text) {+ let response = crate::types::CdpResponse::error(+ req.id, -32000, "Server busy: evaluation in progress, try again later".into(), req.session_id,+ );+ if let Ok(json) = serde_json::to_string(&response) {+ let _ = msg.reply_tx.send(json);+ }+ }+ continue;+ }+ }+ deferred.push_back(message);+ }+ }+ }+}+async fn process_cdp_message(text: &str,ctx: &mut CdpContext,@@ -2741,6 +2828,83 @@ mod tests {}).await;}+ #[tokio::test(flavor = "current_thread")]+ async fn awaited_runtime_requests_keep_servicing_route_replies() {+ use super::{cdp_processor, CdpContext, CdpMessage, ServerMessage};+ use serde_json::Value;+ use std::time::Duration;+ use tokio::sync::mpsc;+ std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");+ tokio::task::LocalSet::new().run_until(async {+ let (server_tx, server_rx) = mpsc::unbounded_channel();+ let (reply_tx, mut reply_rx) = mpsc::unbounded_channel();+ let context = CdpContext::new().default_context;+ let processor = tokio::task::spawn_local(cdp_processor(+ server_rx, context, std::sync::Arc::new(tokio::sync::Notify::new()),+ ));+ server_tx.send(ServerMessage::NewConnection { reply_tx: reply_tx.clone() }).unwrap();+ reply_rx.recv().await.unwrap();+ let send = |value: Value| {+ server_tx.send(ServerMessage::Cdp(CdpMessage {+ text: value.to_string(), reply_tx: reply_tx.clone(),+ })).unwrap();+ };+ send(json!({"id": 1, "method": "Target.createTarget", "params": {"url": "about:blank"}}));+ let mut session = None;+ loop {+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();+ if let Some(id) = value["params"]["sessionId"].as_str() { session = Some(id.to_string()); }+ if value["id"] == 1 { break; }+ }+ let session = session.expect("attached session");+ send(json!({"id": 2, "method": "Fetch.enable", "sessionId": session,+ "params": {"patterns": [{"urlPattern": "http://127.0.0.1:9/*"}]}}));+ loop {+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();+ if value["id"] == 2 { break; }+ }+ let body = "let result=''; for(let i=0;i<2;i++) result+=await (await fetch('http://127.0.0.1:9/'+i)).text(); return result";+ for (id, method, params) in [+ (3, "Runtime.evaluate", json!({"expression": format!("(async()=>{{{body}}})()"), "awaitPromise": true, "returnByValue": true})),+ (4, "Runtime.callFunctionOn", json!({"functionDeclaration": format!("async function(){{{body}}}"), "awaitPromise": true, "returnByValue": true})),+ ] {+ send(json!({"id": id, "method": method, "sessionId": session, "params": params}));+ send(json!({"id": 10 + id, "method": "Runtime.evaluate", "sessionId": session,+ "params": {"expression": "21*2", "returnByValue": true}}));+ let mut pauses = 0;+ let mut completed = false;+ let mut finished = 0;+ let mut deferred_completed = false;+ tokio::time::timeout(Duration::from_secs(3), async {+ loop {+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();+ if value["method"] == "Fetch.requestPaused" {+ assert_eq!(value["sessionId"], session);+ pauses += 1;+ send(json!({"id": 100 + pauses, "method": "Fetch.fulfillRequest", "sessionId": session,+ "params": {"requestId": value["params"]["requestId"], "responseCode": 200,+ "body": "b2s=", "responseHeaders": [{"name": "access-control-allow-origin", "value": "*"}]}}));+ }+ if value["id"] == id {+ assert_eq!(value["result"]["result"]["value"], "okok", "{value}");+ assert_eq!(pauses, 2);+ completed = true;+ }+ if value["method"] == "Network.loadingFinished" { finished += 1; }+ if value["id"] == 10 + id {+ assert!(completed, "unrelated V8 work must stay deferred");+ assert_eq!(value["result"]["result"]["value"].as_f64(), Some(42.0));+ deferred_completed = true;+ }+ if completed && deferred_completed && finished == 2 { break; }+ }+ }).await.expect("awaited evaluation must service route replies");+ }+ drop(server_tx);+ tokio::time::timeout(Duration::from_secs(2), processor).await.unwrap().unwrap();+ }).await;+ }+#[tokio::test(flavor = "current_thread")]async fn intercepted_requests_keep_their_page_and_unique_identity() {use serde_json::Value;@@ -2880,11 +3044,14 @@ mod tests {let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();-+ assert!(!handle_fetch_resolution(+ r#"{"id":16,"method":"Network.getResponseBody","params":{"requestId":"request-1"}}"#,+ &reply_tx,+ &mut paused,+ ));+ assert!(paused.contains_key("request-1"), "unrelated commands must not consume the resolver");assert!(handle_fetch_resolution(r#"{"id":17,"method":"Fetch.continueRequest","params":{"requestId":"request-1"}}"#,- &mut ctx,&reply_tx,&mut paused,));@@ -2896,6 +3063,23 @@ mod tests {serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();assert_eq!(response["id"], 17);assert!(reply_rx.try_recv().is_err(), "must not emit a duplicate response");++ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ paused.insert("request-2".into(), resolution_tx);+ assert!(handle_fetch_resolution(+ r#"{"id":18,"method":"Fetch.failRequest","sessionId":"page-session","params":{"requestId":"request-2","errorReason":"Aborted"}}"#,+ &reply_tx, &mut paused,+ ));+ assert!(matches!(resolution_rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Fail { reason }) if reason == "Aborted"));+ let event: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();+ assert_eq!(event["method"], "Network.loadingFailed");+ assert_eq!(event["sessionId"], "page-session");+ assert_eq!(event["params"]["requestId"], "request-2");+ assert_eq!(event["params"]["errorText"], "net::ERR_ABORTED");+ assert_eq!(event["params"]["canceled"], true);+ let response: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();+ assert_eq!(response["id"], 18);+ assert!(reply_rx.try_recv().is_err(), "abort emits one failure and one response");}#[tokio::test(flavor = "current_thread")]diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex 767ce274..fa51378c 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -583,6 +583,36 @@ fn response_body_byte_limit() -> usize {.unwrap_or(2 * 1024 * 1024)}+fn record_js_network_completion(+ state: &RefCell<OpState>,+ event: JsNetworkEvent,+ body: &str,+ base64_encoded: bool,+) {+ let state_borrow = state.borrow();+ let gs = state_borrow.borrow::<SharedState>().clone();+ let mut gs = gs.borrow_mut();+ let max_entries = response_body_entry_limit();+ let max_bytes = response_body_byte_limit();+ if max_entries > 0 && max_bytes > 0 && event.body_size <= max_bytes {+ gs.network_response_bodies.insert(event.request_id.clone(), StoredNetworkResponseBody {+ body: body.to_string(), base64_encoded,+ });+ gs.network_response_body_order.push_back(event.request_id.clone());+ while gs.network_response_body_order.len() > max_entries {+ if let Some(oldest) = gs.network_response_body_order.pop_front() {+ gs.network_response_bodies.remove(&oldest);+ }+ }+ }+ gs.js_network_events.push(event);+ const MAX_JS_NETWORK_EVENTS: usize = 4096;+ if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {+ let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;+ gs.js_network_events.drain(0..overflow);+ }+}+/// Hard cap on a single JS fetch/XHR response body buffered fully in memory./// `op_fetch_url` reads the whole body, then makes a UTF-8 copy and a base64/// copy of it, so an unbounded body OOMs the process. This bounds the initial@@ -3178,7 +3208,18 @@ async fn op_fetch_url(body: b,body_base64: bb,}) => {- return Ok(intercept_fulfill_response(+ // Keep fulfilled responses visible to CDP just like transport+ // responses, including exact binary bodies and bounded retention.+ let encoded = !bb.is_empty();+ let body_size = if encoded { bb.trim_end_matches('=').len() * 3 / 4 } else { b.len() };+ record_js_network_completion(&state, JsNetworkEvent {+ request_id: request_id.clone(), intercepted: true,+ url: url.clone(), method: method.clone(), status,+ response_headers: h.clone(), body_size,+ timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default().as_secs_f64(),+ }, if encoded { &bb } else { &b }, encoded);+ let mut response = intercept_fulfill_response(status,h,&b,@@ -3188,8 +3229,9 @@ async fn op_fetch_url(&mode,credentials,internal_load,- )- .to_string());+ );+ response["requestId"] = serde_json::json!(request_id);+ return Ok(response.to_string());}Ok(InterceptResolution::Fail { reason }) => {return Ok(serde_json::json!({@@ -3666,52 +3708,13 @@ async fn op_fetch_url(cbs.fire_response(&info, &resp).await;}}- let response_request_id = {- let state_borrow = state.borrow();- let gs = state_borrow.borrow::<SharedState>().clone();- let mut gs = gs.borrow_mut();- let max_entries = response_body_entry_limit();- let max_bytes = response_body_byte_limit();- if max_entries > 0 && max_bytes > 0 && resp_bytes.len() <= max_bytes {- gs.network_response_bodies.insert(- request_id.clone(),- StoredNetworkResponseBody {- body: resp_body.clone(),- base64_encoded: false,- },- );- gs.network_response_body_order.push_back(request_id.clone());- while gs.network_response_body_order.len() > max_entries {- if let Some(oldest) = gs.network_response_body_order.pop_front() {- gs.network_response_bodies.remove(&oldest);- }- }- }- // Record a network event so the CDP layer emits requestWillBeSent /- // responseReceived for this script-initiated request (#406). Keyed by- // the same fetch-{N} id as the stored body so Network.getResponseBody- // resolves. Capped to keep a long-lived page from growing unbounded.- let timestamp = std::time::SystemTime::now()- .duration_since(std::time::UNIX_EPOCH)- .unwrap_or_default()- .as_secs_f64();- gs.js_network_events.push(JsNetworkEvent {- request_id: request_id.clone(),- intercepted: was_intercepted,- url: current_url.clone(),- method: current_method.as_str().to_string(),- status,- response_headers: resp_headers.clone(),- body_size: resp_bytes.len(),- timestamp,- });- const MAX_JS_NETWORK_EVENTS: usize = 4096;- if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {- let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;- gs.js_network_events.drain(0..overflow);- }- request_id- };+ record_js_network_completion(&state, JsNetworkEvent {+ request_id: request_id.clone(), intercepted: was_intercepted,+ url: current_url.clone(), method: current_method.as_str().to_string(), status,+ response_headers: resp_headers.clone(), body_size: resp_bytes.len(),+ timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)+ .unwrap_or_default().as_secs_f64(),+ }, &resp_body, false);tracing::debug!("op_fetch_url completed: {} {} ({} bytes)",@@ -3731,7 +3734,7 @@ async fn op_fetch_url("status": if opaque { 0 } else { status },"body": if opaque { String::new() } else { resp_body },"bodyBase64": if opaque { String::new() } else { resp_body_base64 },- "requestId": response_request_id,+ "requestId": request_id,"url": current_url,"redirected": redirected,"opaque": opaque,