Skip to content

The headless browser for AI agents and web scraping

fix(cdp): service routed requests during awaited evaluations

SGavrl5e58c8673729
diff --git a/crates/obscura-cdp/src/server.rs b/crates/obscura-cdp/src/server.rs
index 8603184d..b263f729 100644
--- a/crates/obscura-cdp/src/server.rs
+++ b/crates/obscura-cdp/src/server.rs
@@ -1344,11 +1344,20 @@ async fn cdp_processor(
&mut intercept_rx, &mut intercepted_paused,
&mut deferred, true,
).await;
+ } else if ctx.pages.iter().any(|page| page.intercept_enabled)
+ && serde_json::from_str::<CdpRequest>(&cdp_msg.text).is_ok_and(|req| {
+ matches!(req.method.as_str(), "Runtime.evaluate" | "Runtime.callFunctionOn")
+ && req.params.get("awaitPromise").and_then(|v| v.as_bool()) == Some(true)
+ })
+ {
+ process_awaited_with_interception(
+ &cdp_msg.text, &mut ctx, &cdp_msg.reply_tx, &mut rx,
+ &mut intercept_rx, &mut intercepted_paused, &mut deferred,
+ ).await;
} else {
let fetch_was_resolved = cdp_msg.text.contains("Fetch.")
&& handle_fetch_resolution(
&cdp_msg.text,
- &mut ctx,
&cdp_msg.reply_tx,
&mut intercepted_paused,
);
@@ -1408,6 +1417,15 @@ fn emit_intercepted_request(
let (session_id, _) = ctx.sessions.iter().find(|(_, page_id)| **page_id == page.id)?;
Some((page.frame_id.as_str(), Some(session_id.clone())))
});
+ emit_routed_intercepted_request(intercepted, route, reply_tx, intercepted_paused);
+}
+
+fn emit_routed_intercepted_request(
+ intercepted: obscura_js::ops::InterceptedRequest,
+ route: Option<(&str, Option<String>)>,
+ reply_tx: &mpsc::UnboundedSender<String>,
+ intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,
+) {
let Some((frame_id, session_id)) = route else {
let _ = intercepted.resolver.send(obscura_js::ops::InterceptResolution::Fail {
reason: "Aborted".into(),
@@ -1658,12 +1676,14 @@ pub(crate) fn parse_cdp_headers(params: &serde_json::Value) -> Option<HashMap<St
fn handle_fetch_resolution(
text: &str,
- _ctx: &mut CdpContext,
reply_tx: &mpsc::UnboundedSender<String>,
intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,
) -> bool {
if let Ok(req) = serde_json::from_str::<CdpRequest>(text) {
let method = req.method.as_str();
+ if !matches!(method, "Fetch.continueRequest" | "Fetch.fulfillRequest" | "Fetch.failRequest") {
+ return false;
+ }
let request_id = req.params.get("requestId").and_then(|v| v.as_str()).unwrap_or("");
tracing::info!("INTERCEPTION resolution: {} for {}, paused_count={}", method, request_id, intercepted_paused.len());
@@ -1702,7 +1722,21 @@ fn handle_fetch_resolution(
}
_ => return false,
};
- let _ = resolver.send(resolution);
+ if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {
+ let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");
+ let mut error_text = String::from("net::ERR_");
+ for (index, ch) in reason.chars().enumerate() {
+ if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }
+ error_text.push(ch.to_ascii_uppercase());
+ }
+ let _ = reply_tx.send(json!({
+ "method": "Network.loadingFailed", "sessionId": req.session_id,
+ "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,
+ "canceled": reason == "Aborted",
+ "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default().as_secs_f64()},
+ }).to_string());
+ }
let resp = crate::types::CdpResponse::success(req.id, json!({}), req.session_id);
if let Ok(json) = serde_json::to_string(&resp) {
let _ = reply_tx.send(json);
@@ -1888,7 +1922,7 @@ async fn process_with_interception(
// op inside the spawned nav task. No V8 enter on
// this side; the actual V8 work happens back on
// the nav task's thread.
- handle_fetch_resolution(&msg.text, ctx, &msg.reply_tx, intercepted_paused);
+ handle_fetch_resolution(&msg.text, &msg.reply_tx, intercepted_paused);
} else {
// UNSAFE during nav: would route through dispatch,
// which can `suspend_js` other pages and trip the
@@ -1980,6 +2014,59 @@ async fn process_with_interception(
}
}
+async fn process_awaited_with_interception(
+ text: &str,
+ ctx: &mut CdpContext,
+ reply_tx: &mpsc::UnboundedSender<String>,
+ rx: &mut mpsc::UnboundedReceiver<ServerMessage>,
+ intercept_rx: &mut Option<mpsc::UnboundedReceiver<obscura_js::ops::InterceptedRequest>>,
+ intercepted_paused: &mut HashMap<String, tokio::sync::oneshot::Sender<obscura_js::ops::InterceptResolution>>,
+ deferred: &mut std::collections::VecDeque<ServerMessage>,
+) {
+ // The evaluation owns ctx (and V8) until it completes. Snapshot only routing
+ // metadata so request replies can unblock it without entering another isolate.
+ let routes: HashMap<_, _> = ctx.pages.iter().filter_map(|page| {
+ let (session_id, _) = ctx.sessions.iter().find(|(_, page_id)| **page_id == page.id)?;
+ Some((page.id.clone(), (page.frame_id.clone(), session_id.clone())))
+ }).collect();
+ let command = process_cdp_message(text, ctx, reply_tx);
+ tokio::pin!(command);
+ loop {
+ tokio::select! {
+ () = &mut command => break,
+ Some(intercepted) = async {
+ match intercept_rx.as_mut() {
+ Some(receiver) => receiver.recv().await,
+ None => std::future::pending().await,
+ }
+ } => {
+ let route = routes.get(&intercepted.page_id)
+ .map(|(frame_id, session_id)| (frame_id.as_str(), Some(session_id.clone())));
+ emit_routed_intercepted_request(intercepted, route, reply_tx, intercepted_paused);
+ }
+ Some(message) = rx.recv() => {
+ if let ServerMessage::Cdp(msg) = &message {
+ if handle_fetch_resolution(&msg.text, &msg.reply_tx, intercepted_paused) {
+ continue;
+ }
+ if deferred.len() >= MAX_DEFERRED_MESSAGES {
+ if let Ok(req) = serde_json::from_str::<CdpRequest>(&msg.text) {
+ let response = crate::types::CdpResponse::error(
+ req.id, -32000, "Server busy: evaluation in progress, try again later".into(), req.session_id,
+ );
+ if let Ok(json) = serde_json::to_string(&response) {
+ let _ = msg.reply_tx.send(json);
+ }
+ }
+ continue;
+ }
+ }
+ deferred.push_back(message);
+ }
+ }
+ }
+}
+
async fn process_cdp_message(
text: &str,
ctx: &mut CdpContext,
@@ -2741,6 +2828,83 @@ mod tests {
}).await;
}
+ #[tokio::test(flavor = "current_thread")]
+ async fn awaited_runtime_requests_keep_servicing_route_replies() {
+ use super::{cdp_processor, CdpContext, CdpMessage, ServerMessage};
+ use serde_json::Value;
+ use std::time::Duration;
+ use tokio::sync::mpsc;
+ std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
+ tokio::task::LocalSet::new().run_until(async {
+ let (server_tx, server_rx) = mpsc::unbounded_channel();
+ let (reply_tx, mut reply_rx) = mpsc::unbounded_channel();
+ let context = CdpContext::new().default_context;
+ let processor = tokio::task::spawn_local(cdp_processor(
+ server_rx, context, std::sync::Arc::new(tokio::sync::Notify::new()),
+ ));
+ server_tx.send(ServerMessage::NewConnection { reply_tx: reply_tx.clone() }).unwrap();
+ reply_rx.recv().await.unwrap();
+ let send = |value: Value| {
+ server_tx.send(ServerMessage::Cdp(CdpMessage {
+ text: value.to_string(), reply_tx: reply_tx.clone(),
+ })).unwrap();
+ };
+ send(json!({"id": 1, "method": "Target.createTarget", "params": {"url": "about:blank"}}));
+ let mut session = None;
+ loop {
+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();
+ if let Some(id) = value["params"]["sessionId"].as_str() { session = Some(id.to_string()); }
+ if value["id"] == 1 { break; }
+ }
+ let session = session.expect("attached session");
+ send(json!({"id": 2, "method": "Fetch.enable", "sessionId": session,
+ "params": {"patterns": [{"urlPattern": "http://127.0.0.1:9/*"}]}}));
+ loop {
+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();
+ if value["id"] == 2 { break; }
+ }
+ let body = "let result=''; for(let i=0;i<2;i++) result+=await (await fetch('http://127.0.0.1:9/'+i)).text(); return result";
+ for (id, method, params) in [
+ (3, "Runtime.evaluate", json!({"expression": format!("(async()=>{{{body}}})()"), "awaitPromise": true, "returnByValue": true})),
+ (4, "Runtime.callFunctionOn", json!({"functionDeclaration": format!("async function(){{{body}}}"), "awaitPromise": true, "returnByValue": true})),
+ ] {
+ send(json!({"id": id, "method": method, "sessionId": session, "params": params}));
+ send(json!({"id": 10 + id, "method": "Runtime.evaluate", "sessionId": session,
+ "params": {"expression": "21*2", "returnByValue": true}}));
+ let mut pauses = 0;
+ let mut completed = false;
+ let mut finished = 0;
+ let mut deferred_completed = false;
+ tokio::time::timeout(Duration::from_secs(3), async {
+ loop {
+ let value: Value = serde_json::from_str(&reply_rx.recv().await.unwrap()).unwrap();
+ if value["method"] == "Fetch.requestPaused" {
+ assert_eq!(value["sessionId"], session);
+ pauses += 1;
+ send(json!({"id": 100 + pauses, "method": "Fetch.fulfillRequest", "sessionId": session,
+ "params": {"requestId": value["params"]["requestId"], "responseCode": 200,
+ "body": "b2s=", "responseHeaders": [{"name": "access-control-allow-origin", "value": "*"}]}}));
+ }
+ if value["id"] == id {
+ assert_eq!(value["result"]["result"]["value"], "okok", "{value}");
+ assert_eq!(pauses, 2);
+ completed = true;
+ }
+ if value["method"] == "Network.loadingFinished" { finished += 1; }
+ if value["id"] == 10 + id {
+ assert!(completed, "unrelated V8 work must stay deferred");
+ assert_eq!(value["result"]["result"]["value"].as_f64(), Some(42.0));
+ deferred_completed = true;
+ }
+ if completed && deferred_completed && finished == 2 { break; }
+ }
+ }).await.expect("awaited evaluation must service route replies");
+ }
+ drop(server_tx);
+ tokio::time::timeout(Duration::from_secs(2), processor).await.unwrap().unwrap();
+ }).await;
+ }
+
#[tokio::test(flavor = "current_thread")]
async fn intercepted_requests_keep_their_page_and_unique_identity() {
use serde_json::Value;
@@ -2880,11 +3044,14 @@ mod tests {
let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
-
+ assert!(!handle_fetch_resolution(
+ r#"{"id":16,"method":"Network.getResponseBody","params":{"requestId":"request-1"}}"#,
+ &reply_tx,
+ &mut paused,
+ ));
+ assert!(paused.contains_key("request-1"), "unrelated commands must not consume the resolver");
assert!(handle_fetch_resolution(
r#"{"id":17,"method":"Fetch.continueRequest","params":{"requestId":"request-1"}}"#,
- &mut ctx,
&reply_tx,
&mut paused,
));
@@ -2896,6 +3063,23 @@ mod tests {
serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();
assert_eq!(response["id"], 17);
assert!(reply_rx.try_recv().is_err(), "must not emit a duplicate response");
+
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ paused.insert("request-2".into(), resolution_tx);
+ assert!(handle_fetch_resolution(
+ r#"{"id":18,"method":"Fetch.failRequest","sessionId":"page-session","params":{"requestId":"request-2","errorReason":"Aborted"}}"#,
+ &reply_tx, &mut paused,
+ ));
+ assert!(matches!(resolution_rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Fail { reason }) if reason == "Aborted"));
+ let event: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
+ assert_eq!(event["method"], "Network.loadingFailed");
+ assert_eq!(event["sessionId"], "page-session");
+ assert_eq!(event["params"]["requestId"], "request-2");
+ assert_eq!(event["params"]["errorText"], "net::ERR_ABORTED");
+ assert_eq!(event["params"]["canceled"], true);
+ let response: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
+ assert_eq!(response["id"], 18);
+ assert!(reply_rx.try_recv().is_err(), "abort emits one failure and one response");
}
#[tokio::test(flavor = "current_thread")]
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index 767ce274..fa51378c 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -583,6 +583,36 @@ fn response_body_byte_limit() -> usize {
.unwrap_or(2 * 1024 * 1024)
}
+fn record_js_network_completion(
+ state: &RefCell<OpState>,
+ event: JsNetworkEvent,
+ body: &str,
+ base64_encoded: bool,
+) {
+ let state_borrow = state.borrow();
+ let gs = state_borrow.borrow::<SharedState>().clone();
+ let mut gs = gs.borrow_mut();
+ let max_entries = response_body_entry_limit();
+ let max_bytes = response_body_byte_limit();
+ if max_entries > 0 && max_bytes > 0 && event.body_size <= max_bytes {
+ gs.network_response_bodies.insert(event.request_id.clone(), StoredNetworkResponseBody {
+ body: body.to_string(), base64_encoded,
+ });
+ gs.network_response_body_order.push_back(event.request_id.clone());
+ while gs.network_response_body_order.len() > max_entries {
+ if let Some(oldest) = gs.network_response_body_order.pop_front() {
+ gs.network_response_bodies.remove(&oldest);
+ }
+ }
+ }
+ gs.js_network_events.push(event);
+ const MAX_JS_NETWORK_EVENTS: usize = 4096;
+ if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {
+ let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;
+ gs.js_network_events.drain(0..overflow);
+ }
+}
+
/// Hard cap on a single JS fetch/XHR response body buffered fully in memory.
/// `op_fetch_url` reads the whole body, then makes a UTF-8 copy and a base64
/// copy of it, so an unbounded body OOMs the process. This bounds the initial
@@ -3178,7 +3208,18 @@ async fn op_fetch_url(
body: b,
body_base64: bb,
}) => {
- return Ok(intercept_fulfill_response(
+ // Keep fulfilled responses visible to CDP just like transport
+ // responses, including exact binary bodies and bounded retention.
+ let encoded = !bb.is_empty();
+ let body_size = if encoded { bb.trim_end_matches('=').len() * 3 / 4 } else { b.len() };
+ record_js_network_completion(&state, JsNetworkEvent {
+ request_id: request_id.clone(), intercepted: true,
+ url: url.clone(), method: method.clone(), status,
+ response_headers: h.clone(), body_size,
+ timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default().as_secs_f64(),
+ }, if encoded { &bb } else { &b }, encoded);
+ let mut response = intercept_fulfill_response(
status,
h,
&b,
@@ -3188,8 +3229,9 @@ async fn op_fetch_url(
&mode,
credentials,
internal_load,
- )
- .to_string());
+ );
+ response["requestId"] = serde_json::json!(request_id);
+ return Ok(response.to_string());
}
Ok(InterceptResolution::Fail { reason }) => {
return Ok(serde_json::json!({
@@ -3666,52 +3708,13 @@ async fn op_fetch_url(
cbs.fire_response(&info, &resp).await;
}
}
- let response_request_id = {
- let state_borrow = state.borrow();
- let gs = state_borrow.borrow::<SharedState>().clone();
- let mut gs = gs.borrow_mut();
- let max_entries = response_body_entry_limit();
- let max_bytes = response_body_byte_limit();
- if max_entries > 0 && max_bytes > 0 && resp_bytes.len() <= max_bytes {
- gs.network_response_bodies.insert(
- request_id.clone(),
- StoredNetworkResponseBody {
- body: resp_body.clone(),
- base64_encoded: false,
- },
- );
- gs.network_response_body_order.push_back(request_id.clone());
- while gs.network_response_body_order.len() > max_entries {
- if let Some(oldest) = gs.network_response_body_order.pop_front() {
- gs.network_response_bodies.remove(&oldest);
- }
- }
- }
- // Record a network event so the CDP layer emits requestWillBeSent /
- // responseReceived for this script-initiated request (#406). Keyed by
- // the same fetch-{N} id as the stored body so Network.getResponseBody
- // resolves. Capped to keep a long-lived page from growing unbounded.
- let timestamp = std::time::SystemTime::now()
- .duration_since(std::time::UNIX_EPOCH)
- .unwrap_or_default()
- .as_secs_f64();
- gs.js_network_events.push(JsNetworkEvent {
- request_id: request_id.clone(),
- intercepted: was_intercepted,
- url: current_url.clone(),
- method: current_method.as_str().to_string(),
- status,
- response_headers: resp_headers.clone(),
- body_size: resp_bytes.len(),
- timestamp,
- });
- const MAX_JS_NETWORK_EVENTS: usize = 4096;
- if gs.js_network_events.len() > MAX_JS_NETWORK_EVENTS {
- let overflow = gs.js_network_events.len() - MAX_JS_NETWORK_EVENTS;
- gs.js_network_events.drain(0..overflow);
- }
- request_id
- };
+ record_js_network_completion(&state, JsNetworkEvent {
+ request_id: request_id.clone(), intercepted: was_intercepted,
+ url: current_url.clone(), method: current_method.as_str().to_string(), status,
+ response_headers: resp_headers.clone(), body_size: resp_bytes.len(),
+ timestamp: std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
+ .unwrap_or_default().as_secs_f64(),
+ }, &resp_body, false);
tracing::debug!(
"op_fetch_url completed: {} {} ({} bytes)",
@@ -3731,7 +3734,7 @@ async fn op_fetch_url(
"status": if opaque { 0 } else { status },
"body": if opaque { String::new() } else { resp_body },
"bodyBase64": if opaque { String::new() } else { resp_body_base64 },
- "requestId": response_request_id,
+ "requestId": request_id,
"url": current_url,
"redirected": redirected,
"opaque": opaque,