Skip to content

The headless browser for AI agents and web scraping

Merge current engine and repair binary interception

SGavrl0e68c8a2c221
diff --cc crates/obscura-cdp/src/domains/fetch.rs
index 7b09890c,7b09890c..e5512a57
--- a/crates/obscura-cdp/src/domains/fetch.rs
+++ b/crates/obscura-cdp/src/domains/fetch.rs
@@@ -18,7 -18,7 +18,7 @@@ pub enum FetchResolution
url: Option<String>,
method: Option<String>,
headers: Option<HashMap<String, String>>,
-- post_data: Option<String>,
++ post_data: Option<Vec<u8>>,
},
Fulfill {
status: u16,
@@@ -113,6 -113,6 +113,10 @@@ pub async fn handle
.and_then(|v| v.as_str())
.ok_or("requestId required")?;
++ let post_data = crate::server::decode_optional_binary(
++ params, "postData", "continueRequest postData",
++ )?;
++
if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {
let _ = paused.resolver.send(FetchResolution::Continue {
url: params
@@@ -126,10 -126,10 +130,7 @@@
// Honor client header overrides (route.continue({ headers }))
// — parity with server.rs handle_fetch_resolution (#919).
headers: crate::server::parse_cdp_headers(params),
-- post_data: params
-- .get("postData")
-- .and_then(|v| v.as_str())
-- .map(|s| s.to_string()),
++ post_data,
});
}
Ok(json!({}))
@@@ -274,6 -274,6 +275,23 @@@ mod tests
}
}
++ #[tokio::test]
++ async fn continue_request_decodes_binary_post_data_and_preserves_invalid_pause() {
++ let mut ctx = CdpContext::new();
++ let mut rx = pause(&mut ctx, "binary");
++ assert!(handle("continueRequest", &json!({"requestId": "binary", "postData": 42}),
++ &mut ctx, &None).await.is_err());
++ assert!(matches!(rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)));
++ handle("continueRequest", &json!({"requestId": "binary", "postData": "AID/EMMo"}),
++ &mut ctx, &None).await.unwrap();
++ match rx.await.unwrap() {
++ FetchResolution::Continue { post_data: Some(body), .. } => {
++ assert_eq!(serde_json::to_value(body).unwrap(), json!([0, 128, 255, 16, 195, 40]));
++ }
++ _ => panic!("expected binary Continue"),
++ }
++ }
++
// Parity with server.rs: the fulfillRequest body is base64-encoded per CDP
// and must be decoded, not passed through as raw base64 text. See #919/#912.
#[tokio::test]
diff --cc crates/obscura-cdp/src/server.rs
index 6ecd2f9c,1241ec2b..93fd8f84
--- a/crates/obscura-cdp/src/server.rs
+++ b/crates/obscura-cdp/src/server.rs
@@@ -1,8 -1,12 +1,13 @@@
- use std::collections::HashMap;
+ use std::collections::{HashMap, HashSet};
+ use std::io::Write;
use std::net::SocketAddr;
- use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
+ #[cfg(unix)]
+ use std::os::unix::fs::OpenOptionsExt;
+ use std::path::Path;
+ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};
use std::sync::Arc;
++use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::net::TcpStream;
@@@ -1538,20 -1770,7 +1775,16 @@@ fn handle_fetch_resolution
Some((h.get("name")?.as_str()?.to_string(), h.get("value")?.as_str()?.to_string()))
}).collect())
.unwrap_or_default();
- // `body` is a lossy text view; `body_base64` carries the CDP body
- // (already base64) through unchanged so op_fetch_url can hand JS
- // the exact bytes for a binary fulfill (#912). Decoding it here is
- // also what checks it: a body that is not base64 reaches JS as the
- // raw string either way, so it is refused at the boundary instead.
- match decode_base64_bytes(raw_body) {
- obscura_js::ops::InterceptResolution::Fulfill { status, headers, body, body_base64 }
++ // Validate before consuming the pause; keep exact bytes for JS.
++ match decode_optional_binary(&req.params, "body", "fulfillRequest body") {
+ Ok(bytes) => Ok(obscura_js::ops::InterceptResolution::Fulfill {
+ status,
+ headers,
- body: String::from_utf8_lossy(&bytes).to_string(),
++ body: String::from_utf8_lossy(&bytes.unwrap_or_default()).to_string(),
+ body_base64: raw_body.to_string(),
+ }),
- Err(why) => Err(format!("fulfillRequest body is not valid base64: {why}")),
++ Err(message) => Err(message),
+ }
}
"Fetch.failRequest" => {
let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed").to_string();
@@@ -1559,18 -1778,22 +1792,32 @@@
}
_ => return false,
};
- if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {
- let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");
- let mut error_text = String::from("net::ERR_");
- for (index, ch) in reason.chars().enumerate() {
- if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }
- error_text.push(ch.to_ascii_uppercase());
+ let resp = match resolution {
+ Ok(resolution) => {
+ let resolver = intercepted_paused
+ .remove(request_id)
+ .expect("the pause was present a few lines above and nothing else can take it");
- let _ = resolver.send(resolution);
++ if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {
++ let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");
++ let mut error_text = String::from("net::ERR_");
++ for (index, ch) in reason.chars().enumerate() {
++ if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }
++ error_text.push(ch.to_ascii_uppercase());
++ }
++ let _ = reply_tx.send(json!({
++ "method": "Network.loadingFailed", "sessionId": req.session_id,
++ "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,
++ "canceled": reason == "Aborted",
++ "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
++ .unwrap_or_default().as_secs_f64()},
++ }).to_string());
++ }
+ crate::types::CdpResponse::success(req.id, json!({}), req.session_id)
}
- let _ = reply_tx.send(json!({
- "method": "Network.loadingFailed", "sessionId": req.session_id,
- "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,
- "canceled": reason == "Aborted",
- "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)
- .unwrap_or_default().as_secs_f64()},
- }).to_string());
- }
- let resp = crate::types::CdpResponse::success(req.id, json!({}), req.session_id);
+ // -32602 is the invalid-params code; the pause is deliberately left
+ // in place so the client can answer this requestId again.
+ Err(message) => crate::types::CdpResponse::error(req.id, -32602, message, req.session_id),
+ };
if let Ok(json) = serde_json::to_string(&resp) {
let _ = reply_tx.send(json);
}
@@@ -1871,101 -2213,33 +2237,39 @@@ async fn process_cdp_message
let _ = reply_tx.send(json);
}
- if let Some((nav_url, nav_method, nav_body)) = check_pending_navigation(ctx, &req.session_id) {
- tracing::info!("JS-triggered nav: {} {} (body: {} bytes)", nav_method, nav_url, nav_body.len());
- let nav_req = CdpRequest {
- id: 0,
- method: "Page.navigate".to_string(),
- params: json!({"url": nav_url, "__method": nav_method, "__body": nav_body}),
- session_id: req.session_id.clone(),
- };
- let _ = dispatch::dispatch(&nav_req, ctx).await;
- for event in ctx.pending_events.drain(..) {
- if let Ok(json) = serde_json::to_string(&event) {
- let _ = reply_tx.send(json);
- }
- }
- }
}
+/// Lossy text view of a base64 CDP field, for the two callers that serve it
+/// as a string. `decode_base64_bytes` is the decoder; this only flattens its
+/// result the way those call sites already expect.
pub(crate) fn decode_base64(input: &str) -> String {
- fn val(c: u8) -> Option<u8> {
- match c {
- b'A'..=b'Z' => Some(c - b'A'),
- b'a'..=b'z' => Some(c - b'a' + 26),
- b'0'..=b'9' => Some(c - b'0' + 52),
- b'+' => Some(62),
- b'/' => Some(63),
- _ => None,
- }
+ String::from_utf8_lossy(&decode_base64_bytes(input).unwrap_or_default()).to_string()
+}
+
+/// Read an optional CDP `binary` parameter as the bytes it encodes.
+///
+/// `Ok(None)` means the field was absent, which is "leave this alone" rather
+/// than "make it empty". An unparseable value is an error rather than a
+/// best-effort decode: base64 that is not base64 is a client mistake, and
+/// answering it silently is what let a raw-text `postData` reach the wire as
+/// mangled bytes with nothing anywhere saying so.
- fn decode_optional_binary(
++pub(crate) fn decode_optional_binary(
+ params: &serde_json::Value,
+ field: &str,
+ what: &str,
+) -> Result<Option<Vec<u8>>, String> {
- let Some(raw) = params.get(field).and_then(|v| v.as_str()) else {
++ let Some(value) = params.get(field) else {
+ return Ok(None);
+ };
++ let raw = value.as_str().ok_or_else(|| format!("{what} must be a base64 string"))?;
+ match decode_base64_bytes(raw) {
+ Ok(bytes) => Ok(Some(bytes)),
+ Err(why) => Err(format!("{what} is not valid base64: {why}")),
}
- let bytes: Vec<u8> = input.bytes().filter_map(val).collect();
- let mut out = Vec::with_capacity(bytes.len() * 3 / 4);
- for chunk in bytes.chunks(4) {
- let b = [
- chunk.first().copied().unwrap_or(0),
- chunk.get(1).copied().unwrap_or(0),
- chunk.get(2).copied().unwrap_or(0),
- chunk.get(3).copied().unwrap_or(0),
- ];
- out.push((b[0] << 2) | (b[1] >> 4));
- if chunk.len() > 2 { out.push((b[1] << 4) | (b[2] >> 2)); }
- if chunk.len() > 3 { out.push((b[2] << 6) | b[3]); }
- }
- String::from_utf8_lossy(&out).to_string()
+}
+
- /// Decode standard base64 to bytes, rejecting what is not base64.
- ///
- /// Padding may be omitted, since encoders differ on it and the length still
- /// determines the group, but a character outside the alphabet and a group left
- /// one symbol short are both refused.
+fn decode_base64_bytes(input: &str) -> Result<Vec<u8>, String> {
- fn val(c: u8) -> Option<u8> {
- match c {
- b'A'..=b'Z' => Some(c - b'A'),
- b'a'..=b'z' => Some(c - b'a' + 26),
- b'0'..=b'9' => Some(c - b'0' + 52),
- b'+' => Some(62),
- b'/' => Some(63),
- _ => None,
- }
- }
- let mut symbols: Vec<u8> = Vec::with_capacity(input.len());
- let mut padding = 0usize;
- for (index, c) in input.bytes().enumerate() {
- if c == b'=' {
- padding += 1;
- continue;
- }
- if padding > 0 {
- return Err(format!("character at byte {index} follows the padding"));
- }
- match val(c) {
- Some(v) => symbols.push(v),
- None => return Err(format!("byte {index} is not a base64 character")),
- }
- }
- if padding > 2 {
- return Err(format!("{padding} padding characters, at most 2 are valid"));
- }
- if symbols.len() % 4 == 1 {
- return Err("the last group has one symbol, which encodes no byte".to_string());
- }
- let mut out = Vec::with_capacity(symbols.len() * 3 / 4);
- for chunk in symbols.chunks(4) {
- let b = [
- chunk[0],
- chunk.get(1).copied().unwrap_or(0),
- chunk.get(2).copied().unwrap_or(0),
- chunk.get(3).copied().unwrap_or(0),
- ];
- out.push((b[0] << 2) | (b[1] >> 4));
- if chunk.len() > 2 { out.push((b[1] << 4) | (b[2] >> 2)); }
- if chunk.len() > 3 { out.push((b[2] << 6) | b[3]); }
- }
- Ok(out)
++ BASE64.decode(input).map_err(|error| error.to_string())
}
fn fast_path_response(text: &str) -> Option<String> {
@@@ -2540,6 -3397,292 +3427,292 @@@ mod tests
assert!(parse_cdp_headers(&json!({"url": "https://example.com"})).is_none());
}
+ async fn assert_navigation_intercept_reply_without_timer(method: &str) {
+ use super::{process_with_interception, CdpMessage, ServerMessage, MAX_DEFERRED_MESSAGES};
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+ use obscura_js::ops::{InterceptedRequest, InterceptResolution};
+
+ tokio::task::LocalSet::new().run_until(async {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let (requested_tx, requested_rx) = tokio::sync::oneshot::channel();
+ let (release_tx, release_rx) = tokio::sync::oneshot::channel();
+ let fixture = tokio::task::spawn_local(async move {
+ let (mut stream, _) = listener.accept().await.unwrap();
+ let mut request = Vec::new();
+ let mut buffer = [0u8; 1024];
+ while !request.windows(4).any(|part| part == b"\r\n\r\n") {
+ let count = stream.read(&mut buffer).await.unwrap();
+ assert!(count > 0);
+ request.extend_from_slice(&buffer[..count]);
+ }
+ requested_tx.send(()).unwrap();
+ release_rx.await.unwrap();
+ let body = b"<!doctype html><title>Finished</title><h1>Finished</h1>";
+ let head = format!("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());
+ stream.write_all(head.as_bytes()).await.unwrap();
+ stream.write_all(body).await.unwrap();
+ });
+
+ let context = obscura_browser::context::BrowserContext::with_storage_and_network(
+ "interception-test".into(), None, false, None, None, true,
+ );
+ let mut ctx = crate::dispatch::CdpContext::new_with_shared_context(std::sync::Arc::new(context));
+ let page_id = ctx.create_page();
+ let session_id = format!("{page_id}-session");
+ ctx.sessions.insert(session_id.clone(), page_id.clone());
+ let (server_tx, mut server_rx) = tokio::sync::mpsc::unbounded_channel();
+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();
+ let (intercept_tx, intercept_rx) = tokio::sync::mpsc::unbounded_channel();
+ ctx.intercept_tx = Some(intercept_tx.clone());
+ let command = json!({"id": 1, "method": "Page.navigate", "sessionId": session_id,
+ "params": {"url": format!("http://{address}/"), "waitUntil": "load"}}).to_string();
+ let navigation_reply = reply_tx.clone();
+ let navigation = tokio::task::spawn_local(async move {
+ let mut intercept_rx = Some(intercept_rx);
+ let mut paused = HashMap::new();
+ let mut deferred = std::collections::VecDeque::new();
+ process_with_interception(&command, &mut ctx, &navigation_reply,
+ &mut server_rx, &mut intercept_rx, &mut paused, &mut deferred, true).await;
+ (ctx, paused, deferred)
+ });
+ tokio::time::timeout(std::time::Duration::from_secs(5), requested_rx)
+ .await.expect("navigation must reach the held response").unwrap();
+
+ // The real socket is already connected and its response is gated.
+ // Only the channel handshake below runs with a paused clock. An
+ // unconditional timer auto-advances it, independent of host load.
+ tokio::time::pause();
+ let started = tokio::time::Instant::now();
+ let (resolution_tx, resolution_rx) = tokio::sync::oneshot::channel();
+ intercept_tx.send(InterceptedRequest {
+ page_id: page_id.clone(),
+ request_id: "pending-test".into(), url: format!("http://{address}/pending"),
+ method: "GET".into(), headers: HashMap::new(), resource_type: "Fetch".into(),
+ resolver: resolution_tx,
+ }).unwrap();
+ loop {
+ let value: serde_json::Value = serde_json::from_str(
+ &tokio::time::timeout(std::time::Duration::from_secs(1), reply_rx.recv())
+ .await.expect("paused notification").unwrap()).unwrap();
+ assert_ne!(value["id"], 1, "navigation must wait for its response");
+ if value["method"] == "Fetch.requestPaused" {
+ assert_eq!(value["params"]["requestId"], "pending-test");
+ break;
+ }
+ }
+ let send = |value: serde_json::Value| {
+ server_tx.send(ServerMessage::Cdp(CdpMessage {
+ text: value.to_string(), reply_tx: reply_tx.clone(),
+ })).unwrap();
+ };
+ let foreign_count = if method == "failRequest" { MAX_DEFERRED_MESSAGES + 1 } else { 1 };
+ for index in 0..foreign_count {
+ send(json!({"id": 100 + index, "method": "Runtime.evaluate",
+ "sessionId": session_id, "params": {"expression": "globalThis.foreignRan=true"}}));
+ }
+ let mut params = json!({"requestId": "pending-test"});
+ match method {
+ "continueRequest" => {
+ params["url"] = json!("https://example.com/rewritten");
+ params["method"] = json!("POST");
+ params["headers"] = json!([{"name": "X-Check", "value": "kept"}]);
- params["postData"] = json!("payload");
++ params["postData"] = json!("cGF5bG9hZA==");
+ }
+ "fulfillRequest" => {
+ params["responseCode"] = json!(201);
+ params["responseHeaders"] = json!([{"name": "X-Check", "value": "kept"}]);
+ params["body"] = json!("aGVsbG8=");
+ }
+ "failRequest" => params["errorReason"] = json!("Aborted"),
+ _ => unreachable!(),
+ }
+ send(json!({"id": 10, "method": format!("Fetch.{method}"),
+ "sessionId": session_id, "params": params}));
+ let mut overflow_responses = 0;
+ loop {
+ let value: serde_json::Value = serde_json::from_str(
+ &tokio::time::timeout(std::time::Duration::from_secs(1), reply_rx.recv())
+ .await.expect("Fetch resolution reply").unwrap()).unwrap();
+ assert_ne!(value["id"], 1, "must not acknowledge unfinished navigation");
+ if value["id"] == 10 {
+ assert!(value.get("error").is_none(), "{value}");
+ break;
+ }
+ if value.get("id").is_some() {
+ assert_eq!(value["id"], 100 + MAX_DEFERRED_MESSAGES);
+ assert_eq!(value["error"]["code"], -32000);
+ overflow_responses += 1;
+ }
+ }
+ let elapsed = started.elapsed();
+ match (method, resolution_rx.await.unwrap()) {
+ ("continueRequest", InterceptResolution::Continue { url, method, headers, body }) => {
+ assert_eq!(url.as_deref(), Some("https://example.com/rewritten"));
+ assert_eq!(method.as_deref(), Some("POST"));
+ assert_eq!(headers.unwrap().get("X-Check").map(String::as_str), Some("kept"));
- assert_eq!(body.as_deref(), Some("payload"));
++ assert_eq!(body.as_deref(), Some(b"payload".as_slice()));
+ }
+ ("fulfillRequest", InterceptResolution::Fulfill { status, headers, body, body_base64 }) => {
+ assert_eq!(status, 201);
+ assert_eq!(headers.get("X-Check").map(String::as_str), Some("kept"));
+ assert_eq!(body, "hello");
+ assert_eq!(body_base64, "aGVsbG8=");
+ }
+ ("failRequest", InterceptResolution::Fail { reason }) => assert_eq!(reason, "Aborted"),
+ _ => panic!("resolution kind changed"),
+ }
+ tokio::time::resume();
+ release_tx.send(()).unwrap();
+ let (ctx, paused, deferred) = tokio::time::timeout(std::time::Duration::from_secs(5), navigation)
+ .await.expect("navigation completion").unwrap();
+ fixture.await.unwrap();
+ assert_eq!(ctx.pages.len(), 1);
+ assert_eq!(ctx.pages[0].url_string(), format!("http://{address}/"));
+ assert!(paused.is_empty());
+ assert_eq!(deferred.len(), foreign_count.min(MAX_DEFERRED_MESSAGES));
+ assert_eq!(overflow_responses, usize::from(foreign_count > MAX_DEFERRED_MESSAGES));
+ let replies: Vec<serde_json::Value> = std::iter::from_fn(|| reply_rx.try_recv().ok())
+ .map(|text| serde_json::from_str(&text).unwrap()).collect();
+ assert_eq!(replies.iter().filter(|reply| reply["id"] == 1).count(), 1);
+ let navigation_reply = replies.iter().find(|reply| reply["id"] == 1).unwrap();
+ assert!(navigation_reply.get("error").is_none(), "{navigation_reply}");
+ assert!(!replies.iter().any(|reply| reply["id"] == 10), "no duplicate resolution reply");
+ assert_eq!(elapsed, std::time::Duration::ZERO, "ready replies must not wait for a timer");
+ }).await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn navigation_intercept_continue_has_no_timer_delay() {
+ assert_navigation_intercept_reply_without_timer("continueRequest").await;
+ }
+
+ async fn assert_stop_loading_cancels_navigation(script: bool, method: &str) {
+ use super::{process_cdp_message, process_with_interception, CdpMessage, ServerMessage};
+ use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+ tokio::task::LocalSet::new().run_until(async {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let (requested_tx, requested_rx) = tokio::sync::oneshot::channel();
+ let fixture = tokio::task::spawn_local(async move {
+ if script {
+ let (mut stream, _) = listener.accept().await.unwrap();
+ let mut buffer = [0; 2048];
+ stream.read(&mut buffer).await.unwrap();
+ let body = "<!doctype html><title>Partial</title><script src='/held.js'></script>";
+ let response = format!("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());
+ stream.write_all(response.as_bytes()).await.unwrap();
+ }
+ let (mut stream, _) = listener.accept().await.unwrap();
+ let mut buffer = [0; 2048];
+ assert!(stream.read(&mut buffer).await.unwrap() > 0);
+ requested_tx.send(()).unwrap();
+ // No response: cancellation must drop the pending HTTP request.
+ assert_eq!(stream.read(&mut buffer).await.unwrap(), 0);
+ });
+ let context = obscura_browser::context::BrowserContext::with_storage_and_network(
+ "stop-loading-test".into(), None, false, None, None, true,
+ );
+ let mut ctx = crate::dispatch::CdpContext::new_with_shared_context(std::sync::Arc::new(context));
+ let page_id = ctx.create_page();
+ let other_page = ctx.create_page();
+ let session = format!("{page_id}-session");
+ ctx.sessions.insert(session.clone(), page_id.clone());
+ ctx.sessions.insert("raw-session".into(), page_id.clone());
+ ctx.sessions.insert("other-session".into(), other_page.clone());
+ ctx.get_page_mut(&page_id).unwrap().resume_js();
+ let original_url = ctx.pages[0].url_string();
+ let (server_tx, mut server_rx) = tokio::sync::mpsc::unbounded_channel();
+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();
+ let navigation_reply = reply_tx.clone();
+ let url = format!("http://{address}/");
+ let command = json!({"id": 1, "method": "Page.navigate", "sessionId": session,
+ "params": {"url": url, "waitUntil": "load"}}).to_string();
+ let navigation = tokio::task::spawn_local(async move {
+ let mut paused = HashMap::new();
+ let mut deferred = std::collections::VecDeque::new();
+ process_with_interception(&command, &mut ctx, &navigation_reply,
+ &mut server_rx, &mut None, &mut paused, &mut deferred, true).await;
+ (ctx, paused, deferred)
+ });
+ tokio::time::timeout(std::time::Duration::from_secs(5), requested_rx)
+ .await.expect("fixture request").unwrap();
+ for (id, session) in [(2, "other-session"), (3, "raw-session")] {
+ server_tx.send(ServerMessage::Cdp(CdpMessage {
+ text: json!({"id": id, "method": if id == 2 { "Page.stopLoading" } else { method },
+ "sessionId": session, "params": {"targetId": page_id}}).to_string(),
+ reply_tx: reply_tx.clone(),
+ })).unwrap();
+ }
+ let (mut ctx, paused, deferred) = tokio::time::timeout(std::time::Duration::from_secs(2), navigation)
+ .await.expect("Page.stopLoading must cancel without waiting for the HTTP timeout").unwrap();
+ tokio::time::timeout(std::time::Duration::from_secs(2), fixture)
+ .await.expect("pending request socket must close").unwrap();
+ assert!(paused.is_empty());
+ assert_eq!(deferred.len(), if method == "Page.stopLoading" { 1 } else { 2 },
+ "another target's stop must not cancel this navigation");
+ for message in deferred {
+ if let ServerMessage::Cdp(message) = message {
+ process_cdp_message(&message.text, &mut ctx, &message.reply_tx).await;
+ }
+ }
+ let replies: Vec<serde_json::Value> = std::iter::from_fn(|| reply_rx.try_recv().ok())
+ .map(|text| serde_json::from_str(&text).unwrap()).collect();
+ assert_eq!(replies.iter().filter(|reply| reply["id"] == 3).count(), 1);
+ let stopped = replies.iter().find(|reply| reply["id"] == 3).unwrap();
+ assert!(stopped.get("error").is_none(), "{stopped}");
+ let aborted = replies.iter().find(|reply| reply["id"] == 1).unwrap();
+ assert!(aborted.get("error").is_none(), "{aborted}");
+ if !script {
+ assert_eq!(aborted["result"]["errorText"], "net::ERR_ABORTED", "{aborted}");
+ }
+ assert!(!replies.iter().any(|reply| reply["method"] == "Page.loadEventFired"));
+ assert!(replies.iter().any(|reply| reply["method"] == "Page.frameStoppedLoading"));
+ assert!(ctx.get_page(&other_page).is_some());
+ if method != "Page.stopLoading" {
+ assert!(ctx.get_page(&page_id).is_none());
+ assert_eq!(replies.iter().filter(|reply| reply["method"] == "Target.targetDestroyed").count(), 1);
+ return;
+ }
+ assert_eq!(ctx.get_page(&page_id).unwrap().url_string(), if script { url } else { original_url });
+ assert_eq!(ctx.get_page_mut(&page_id).unwrap().evaluate("1 + 2").as_f64(), Some(3.0));
+ process_cdp_message(&json!({"id": 4, "method": "Page.navigate", "sessionId": session,
+ "params": {"url": "about:blank"}}).to_string(), &mut ctx, &reply_tx).await;
+ assert_eq!(ctx.get_page(&page_id).unwrap().url_string(), "about:blank");
+ }).await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn stop_loading_cancels_a_held_document_request() {
+ assert_stop_loading_cancels_navigation(false, "Page.stopLoading").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn stop_loading_cancels_a_held_parser_script_request() {
+ assert_stop_loading_cancels_navigation(true, "Page.stopLoading").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn stop_loading_cancels_for_page_close() {
+ assert_stop_loading_cancels_navigation(false, "Page.close").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn stop_loading_cancels_for_target_close() {
+ assert_stop_loading_cancels_navigation(false, "Target.closeTarget").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn navigation_intercept_fulfill_has_no_timer_delay() {
+ assert_navigation_intercept_reply_without_timer("fulfillRequest").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn navigation_intercept_fail_keeps_deferral_bounded_without_delay() {
+ assert_navigation_intercept_reply_without_timer("failRequest").await;
+ }
+
#[test]
fn fetch_resolution_is_handled_once_by_the_outer_processor() {
let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
@@@ -2561,164 -3707,131 +3737,299 @@@
serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();
assert_eq!(response["id"], 17);
assert!(reply_rx.try_recv().is_err(), "must not emit a duplicate response");
+
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ paused.insert("request-2".into(), resolution_tx);
+ assert!(handle_fetch_resolution(
+ r#"{"id":18,"method":"Fetch.failRequest","sessionId":"page-session","params":{"requestId":"request-2","errorReason":"Aborted"}}"#,
+ &reply_tx, &mut paused,
+ ));
+ assert!(matches!(resolution_rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Fail { reason }) if reason == "Aborted"));
+ let event: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
+ assert_eq!(event["method"], "Network.loadingFailed");
+ assert_eq!(event["sessionId"], "page-session");
+ assert_eq!(event["params"]["requestId"], "request-2");
+ assert_eq!(event["params"]["errorText"], "net::ERR_ABORTED");
+ assert_eq!(event["params"]["canceled"], true);
+ let response: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
+ assert_eq!(response["id"], 18);
+ assert!(reply_rx.try_recv().is_err(), "abort emits one failure and one response");
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn indexeddb_versionchange_rearms_an_idle_page_until_its_delayed_close() {
+ let mut ctx = crate::dispatch::CdpContext::new();
+ let mut ids = Vec::new();
+ for _ in 0..2 {
+ let id = ctx.create_page();
+ let session = Some(format!("{id}-session"));
+ ctx.sessions.insert(session.clone().unwrap(),id.clone());
+ crate::domains::page::handle("navigate", &json!({"url":"data:text/html,<body>idb</body>"}), &mut ctx, &session).await.unwrap();
+ ctx.get_page_mut(&id).unwrap().js.as_mut().unwrap().set_url("https://idb.test/");
+ ids.push(id);
+ }
+ let mut idle = Default::default();
+ ctx.get_page_mut(&ids[0]).unwrap().evaluate(r#"(() => {
+ const r = indexedDB.open('idle-upgrade',1);
+ globalThis.opened = false;
+ r.onsuccess = () => {globalThis.opened = true;
+ r.result.onversionchange = () => setTimeout(() => r.result.close(),10);};
+ })()"#);
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while !pump_live_page_event_loop(&mut ctx,&mut idle).await.unwrap() {}
+ }).await.unwrap();
+ assert!(idle.contains(&ids[0]));
+ assert_eq!(ctx.get_page_mut(&ids[0]).unwrap().evaluate("opened"),json!(true));
+ ctx.get_page_mut(&ids[1]).unwrap().evaluate(r#"(() => {
+ globalThis.upgraded = false;
+ setTimeout(() => {const r = indexedDB.open('idle-upgrade',2);
+ r.onsuccess = () => globalThis.upgraded = r.result.version === 2;},10);
+ })()"#);
+ idle.remove(&ids[1]);
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ loop {
+ pump_live_page_event_loop(&mut ctx,&mut idle).await.unwrap();
+ if ctx.get_page_mut(&ids[1]).unwrap().evaluate("upgraded") == json!(true) {break;}
+ }
+ }).await.expect("versionchange must wake the idle page and retain its delayed-close work");
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn autonomous_pages_do_not_wait_for_another_pages_distant_timer() {
+ let mut ctx = crate::dispatch::CdpContext::new();
+ let mut idle_pages = Default::default();
+ let mut pages = Vec::new();
+ for _ in 0..3 {
+ let id = ctx.create_page();
+ let session = Some(format!("{id}-session"));
+ ctx.sessions.insert(session.clone().unwrap(), id.clone());
+ crate::domains::page::handle(
+ "navigate", &json!({"url": "data:text/html,<body>timer</body>"}),
+ &mut ctx, &session,
+ ).await.unwrap();
+ pages.push(id);
+ }
+ ctx.get_page_mut(&pages[0]).unwrap().evaluate(
+ "globalThis.slow = setTimeout(() => {}, 60000)",
+ );
+ for id in &pages[1..] {
+ assert_eq!(ctx.get_page_mut(id).unwrap().evaluate(
+ "(() => { globalThis.done = false; setTimeout(() => { globalThis.done = true; }, 20); return globalThis.done; })()",
+ ), json!(false));
+ }
+ let started = std::time::Instant::now();
+ let completed = tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ loop {
+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();
+ if pages[1..].iter().all(|id| {
+ ctx.get_page_mut(id).unwrap().evaluate("globalThis.done") == json!(true)
+ }) {
+ break;
+ }
+ }
+ }).await;
+ assert!(completed.is_ok(), "a parked page must not prevent other pages' timers from running");
+ assert!(started.elapsed() < std::time::Duration::from_millis(500),
+ "ready timers must not need the outer timeout's wake: {:?}", started.elapsed());
+ ctx.get_page_mut(&pages[2]).unwrap().evaluate(
+ "setTimeout(() => { globalThis.finalTurn = true; }, 0)",
+ );
+ idle_pages.clear(); // A new command rearms previously idle pages.
+ // This turn can do work and become idle in the same poll. Its result
+ // must still be published while the first page remains parked.
+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;
+ let started = std::time::Instant::now();
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ loop {
+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();
+ if ctx.get_page_mut(&pages[2]).unwrap().evaluate("globalThis.finalTurn") == json!(true) {
+ break;
+ }
+ }
+ }).await.expect("the last completed task must not remain buffered");
+ assert!(started.elapsed() < std::time::Duration::from_millis(500),
+ "a task becoming idle must still publish progress: {:?}", started.elapsed());
+ let parked = tokio::time::timeout(std::time::Duration::from_millis(50), async {
+ for _ in 0..8 {
+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();
+ }
+ }).await;
+ assert!(parked.is_err(), "idle peers must not repeatedly wake a parked connection");
+ ctx.get_page_mut(&pages[0]).unwrap().evaluate("clearTimeout(globalThis.slow)");
+ idle_pages.clear();
+ tokio::time::timeout(std::time::Duration::from_secs(1), async {
+ while !pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap() {}
+ }).await.expect("all idle pages must disarm the pump instead of spinning");
}
+ // #569 / #365 — postData is declared `binary` in the CDP domain, so a
+ // client sends it base64-encoded. Passing the encoded text straight through
+ // made the request body the base64 itself, which a server reads as neither
+ // the JSON nor the form the page meant to send.
+ #[test]
+ fn continue_request_decodes_a_base64_post_data() {
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
+
+ // base64 of {"a":1}
+ assert!(handle_fetch_resolution(
+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"eyJhIjoxfQ=="}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+
+ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {
+ panic!("continueRequest must resolve as Continue");
+ };
+ assert_eq!(
+ body.as_deref(),
+ Some(br#"{"a":1}"#.as_slice()),
+ "the request body must be the bytes the client encoded, not the base64 text"
+ );
+ }
+
+ // The body a client rewrites is not always text -- a protobuf, a gzip
+ // stream, a multipart file part. Carrying it through `String` replaced
+ // every non-UTF-8 sequence with U+FFFD before it reached the wire.
+ #[test]
+ fn continue_request_keeps_a_non_utf8_post_data_intact() {
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
+
+ // base64 of the three bytes 00 FF FE, which are not valid UTF-8
+ assert!(handle_fetch_resolution(
+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"AP/+"}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+
+ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {
+ panic!("continueRequest must resolve as Continue");
+ };
+ assert_eq!(
+ body.as_deref(),
+ Some([0x00u8, 0xFF, 0xFE].as_slice()),
+ "a binary body must reach the wire byte for byte"
+ );
+ }
+
+ // Base64 that is not base64 is a client mistake. Answering it silently is
+ // how raw text reached the wire as mangled bytes; answering it with an
+ // error while keeping the pause lets the client correct itself.
+ #[test]
+ fn continue_request_rejects_post_data_that_is_not_base64_without_wedging_it() {
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
+
+ assert!(handle_fetch_resolution(
+ r#"{"id":9,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"!!!not-base64"}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+
+ let rejected: serde_json::Value =
+ serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();
+ assert_eq!(rejected["id"], 9);
+ assert_eq!(rejected["error"]["code"], -32602);
+ assert!(
+ rejected["error"]["message"]
+ .as_str()
+ .unwrap_or_default()
+ .starts_with("continueRequest postData is not valid base64"),
+ "the error must name the field, got {:?}",
+ rejected["error"]["message"]
+ );
+ assert!(
+ resolution_rx.try_recv().is_err(),
+ "a rejected command must not resolve the pause"
+ );
+ assert!(
+ paused.contains_key("request-1"),
+ "a rejected command must leave the request paused so it can be answered again"
+ );
+
+ // the same requestId, answered correctly, still drives the fetch
+ assert!(handle_fetch_resolution(
+ r#"{"id":10,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"eyJhIjoxfQ=="}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {
+ panic!("the corrected continueRequest must resolve as Continue");
+ };
+ assert_eq!(body.as_deref(), Some(br#"{"a":1}"#.as_slice()));
+ }
+
+ // The same strictness on the other arm carrying a CDP `binary` field.
+ #[test]
+ fn fulfill_request_rejects_a_body_that_is_not_base64() {
+ let (resolution_tx, _resolution_rx) = tokio::sync::oneshot::channel();
+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
+
+ assert!(handle_fetch_resolution(
+ r#"{"id":11,"method":"Fetch.fulfillRequest","params":{"requestId":"request-1","responseCode":200,"body":"<html>"}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+
+ let rejected: serde_json::Value =
+ serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();
+ assert_eq!(rejected["error"]["code"], -32602);
+ assert!(
+ rejected["error"]["message"]
+ .as_str()
+ .unwrap_or_default()
+ .starts_with("fulfillRequest body is not valid base64")
+ );
+ assert!(paused.contains_key("request-1"), "the request must stay paused");
+ }
+
++ #[test]
++ fn continue_request_rejects_invalid_binary_values_and_allows_retry() {
++ for value in [json!(null), json!(42), json!("AA="), json!("AB=="), json!("====")] {
++ let (tx, mut rx) = tokio::sync::oneshot::channel();
++ let mut paused = HashMap::from([("request-1".to_string(), tx)]);
++ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();
++ let command = json!({"id": 1, "method": "Fetch.continueRequest",
++ "params": {"requestId": "request-1", "postData": value}}).to_string();
++ assert!(handle_fetch_resolution(&command, &reply_tx, &mut paused));
++ let reply: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
++ assert_eq!(reply["error"]["code"], -32602, "accepted invalid postData: {value}");
++ assert!(matches!(rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)));
++ assert!(paused.contains_key("request-1"));
++ assert!(handle_fetch_resolution(
++ r#"{"id":2,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":""}}"#,
++ &reply_tx, &mut paused,
++ ));
++ assert!(matches!(rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Continue { body: Some(body), .. }) if body.is_empty()));
++ let reply: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();
++ assert!(reply.get("error").is_none());
++ }
++ }
++
+ // An absent postData still means "leave the body alone", not "send empty".
+ #[test]
+ fn continue_request_without_post_data_leaves_the_body_alone() {
+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();
+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);
+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();
- let mut ctx = crate::dispatch::CdpContext::new();
+
+ assert!(handle_fetch_resolution(
+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1"}}"#,
- &mut ctx,
+ &reply_tx,
+ &mut paused,
+ ));
+
+ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {
+ panic!("continueRequest must resolve as Continue");
+ };
+ assert!(body.is_none(), "an absent postData must not become an empty body");
+ }
+
#[cfg(feature = "render")]
#[tokio::test(flavor = "current_thread")]
async fn autonomous_screencast_pumps_timers_and_retains_backpressured_damage() {
diff --cc docs/Use-as-a-Rust-library.md
index 97df3f33,97df3f33..acb49400
--- a/docs/Use-as-a-Rust-library.md
+++ b/docs/Use-as-a-Rust-library.md
@@@ -122,6 -122,6 +122,8 @@@ page.settle(2000).await
A `Continue` with `url: Some(...)` rewrites the target. The new URL is re-checked against the SSRF / private-network gate, so a rewrite cannot reach an internal address that would otherwise need `--allow-private-network`.
++`Continue.body` takes raw bytes as `Option<Vec<u8>>`, not a base64 string. Migrate text overrides from `Some(text)` to `Some(text.into_bytes())`. `None` preserves the original request body; `Some(Vec::new())` replaces it with an empty body. CDP clients still send base64 `postData` as specified by the protocol; Obscura decodes it before sending the request.
++
### Preload scripts
`add_preload_script` runs a script before any of the page's own `<script>` tags (the CDP `Page.addScriptToEvaluateOnNewDocument` contract), so it can install hooks before the page bootstraps. Call it before `goto`.