The headless browser for AI agents and web scraping
Merge current engine and repair binary interception
SGavrl0e68c8a2c221
diff --cc crates/obscura-cdp/src/domains/fetch.rsindex 7b09890c,7b09890c..e5512a57--- a/crates/obscura-cdp/src/domains/fetch.rs+++ b/crates/obscura-cdp/src/domains/fetch.rs@@@ -18,7 -18,7 +18,7 @@@ pub enum FetchResolutionurl: Option<String>,method: Option<String>,headers: Option<HashMap<String, String>>,-- post_data: Option<String>,++ post_data: Option<Vec<u8>>,},Fulfill {status: u16,@@@ -113,6 -113,6 +113,10 @@@ pub async fn handle.and_then(|v| v.as_str()).ok_or("requestId required")?;++ let post_data = crate::server::decode_optional_binary(++ params, "postData", "continueRequest postData",++ )?;++if let Some(paused) = ctx.fetch_intercept.paused.remove(request_id) {let _ = paused.resolver.send(FetchResolution::Continue {url: params@@@ -126,10 -126,10 +130,7 @@@// Honor client header overrides (route.continue({ headers }))// — parity with server.rs handle_fetch_resolution (#919).headers: crate::server::parse_cdp_headers(params),-- post_data: params-- .get("postData")-- .and_then(|v| v.as_str())-- .map(|s| s.to_string()),++ post_data,});}Ok(json!({}))@@@ -274,6 -274,6 +275,23 @@@ mod tests}}++ #[tokio::test]++ async fn continue_request_decodes_binary_post_data_and_preserves_invalid_pause() {++ let mut ctx = CdpContext::new();++ let mut rx = pause(&mut ctx, "binary");++ assert!(handle("continueRequest", &json!({"requestId": "binary", "postData": 42}),++ &mut ctx, &None).await.is_err());++ assert!(matches!(rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)));++ handle("continueRequest", &json!({"requestId": "binary", "postData": "AID/EMMo"}),++ &mut ctx, &None).await.unwrap();++ match rx.await.unwrap() {++ FetchResolution::Continue { post_data: Some(body), .. } => {++ assert_eq!(serde_json::to_value(body).unwrap(), json!([0, 128, 255, 16, 195, 40]));++ }++ _ => panic!("expected binary Continue"),++ }++ }++// Parity with server.rs: the fulfillRequest body is base64-encoded per CDP// and must be decoded, not passed through as raw base64 text. See #919/#912.#[tokio::test]diff --cc crates/obscura-cdp/src/server.rsindex 6ecd2f9c,1241ec2b..93fd8f84--- a/crates/obscura-cdp/src/server.rs+++ b/crates/obscura-cdp/src/server.rs@@@ -1,8 -1,12 +1,13 @@@- use std::collections::HashMap;+ use std::collections::{HashMap, HashSet};+ use std::io::Write;use std::net::SocketAddr;- use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};+ #[cfg(unix)]+ use std::os::unix::fs::OpenOptionsExt;+ use std::path::Path;+ use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering};use std::sync::Arc;++use base64::{Engine as _, engine::general_purpose::STANDARD as BASE64};use futures_util::{SinkExt, StreamExt};use serde_json::json;use tokio::net::TcpStream;@@@ -1538,20 -1770,7 +1775,16 @@@ fn handle_fetch_resolutionSome((h.get("name")?.as_str()?.to_string(), h.get("value")?.as_str()?.to_string()))}).collect()).unwrap_or_default();- // `body` is a lossy text view; `body_base64` carries the CDP body- // (already base64) through unchanged so op_fetch_url can hand JS- // the exact bytes for a binary fulfill (#912). Decoding it here is- // also what checks it: a body that is not base64 reaches JS as the- // raw string either way, so it is refused at the boundary instead.- match decode_base64_bytes(raw_body) {- obscura_js::ops::InterceptResolution::Fulfill { status, headers, body, body_base64 }++ // Validate before consuming the pause; keep exact bytes for JS.++ match decode_optional_binary(&req.params, "body", "fulfillRequest body") {+ Ok(bytes) => Ok(obscura_js::ops::InterceptResolution::Fulfill {+ status,+ headers,- body: String::from_utf8_lossy(&bytes).to_string(),++ body: String::from_utf8_lossy(&bytes.unwrap_or_default()).to_string(),+ body_base64: raw_body.to_string(),+ }),- Err(why) => Err(format!("fulfillRequest body is not valid base64: {why}")),++ Err(message) => Err(message),+ }}"Fetch.failRequest" => {let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed").to_string();@@@ -1559,18 -1778,22 +1792,32 @@@}_ => return false,};- if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {- let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");- let mut error_text = String::from("net::ERR_");- for (index, ch) in reason.chars().enumerate() {- if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }- error_text.push(ch.to_ascii_uppercase());+ let resp = match resolution {+ Ok(resolution) => {+ let resolver = intercepted_paused+ .remove(request_id)+ .expect("the pause was present a few lines above and nothing else can take it");- let _ = resolver.send(resolution);++ if resolver.send(resolution).is_ok() && method == "Fetch.failRequest" {++ let reason = req.params.get("errorReason").and_then(|v| v.as_str()).unwrap_or("Failed");++ let mut error_text = String::from("net::ERR_");++ for (index, ch) in reason.chars().enumerate() {++ if index > 0 && ch.is_ascii_uppercase() { error_text.push('_'); }++ error_text.push(ch.to_ascii_uppercase());++ }++ let _ = reply_tx.send(json!({++ "method": "Network.loadingFailed", "sessionId": req.session_id,++ "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,++ "canceled": reason == "Aborted",++ "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)++ .unwrap_or_default().as_secs_f64()},++ }).to_string());++ }+ crate::types::CdpResponse::success(req.id, json!({}), req.session_id)}- let _ = reply_tx.send(json!({- "method": "Network.loadingFailed", "sessionId": req.session_id,- "params": {"requestId": request_id, "type": "Fetch", "errorText": error_text,- "canceled": reason == "Aborted",- "timestamp": std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH)- .unwrap_or_default().as_secs_f64()},- }).to_string());- }- let resp = crate::types::CdpResponse::success(req.id, json!({}), req.session_id);+ // -32602 is the invalid-params code; the pause is deliberately left+ // in place so the client can answer this requestId again.+ Err(message) => crate::types::CdpResponse::error(req.id, -32602, message, req.session_id),+ };if let Ok(json) = serde_json::to_string(&resp) {let _ = reply_tx.send(json);}@@@ -1871,101 -2213,33 +2237,39 @@@ async fn process_cdp_messagelet _ = reply_tx.send(json);}- if let Some((nav_url, nav_method, nav_body)) = check_pending_navigation(ctx, &req.session_id) {- tracing::info!("JS-triggered nav: {} {} (body: {} bytes)", nav_method, nav_url, nav_body.len());- let nav_req = CdpRequest {- id: 0,- method: "Page.navigate".to_string(),- params: json!({"url": nav_url, "__method": nav_method, "__body": nav_body}),- session_id: req.session_id.clone(),- };- let _ = dispatch::dispatch(&nav_req, ctx).await;- for event in ctx.pending_events.drain(..) {- if let Ok(json) = serde_json::to_string(&event) {- let _ = reply_tx.send(json);- }- }- }}+/// Lossy text view of a base64 CDP field, for the two callers that serve it+/// as a string. `decode_base64_bytes` is the decoder; this only flattens its+/// result the way those call sites already expect.pub(crate) fn decode_base64(input: &str) -> String {- fn val(c: u8) -> Option<u8> {- match c {- b'A'..=b'Z' => Some(c - b'A'),- b'a'..=b'z' => Some(c - b'a' + 26),- b'0'..=b'9' => Some(c - b'0' + 52),- b'+' => Some(62),- b'/' => Some(63),- _ => None,- }+ String::from_utf8_lossy(&decode_base64_bytes(input).unwrap_or_default()).to_string()+}++/// Read an optional CDP `binary` parameter as the bytes it encodes.+///+/// `Ok(None)` means the field was absent, which is "leave this alone" rather+/// than "make it empty". An unparseable value is an error rather than a+/// best-effort decode: base64 that is not base64 is a client mistake, and+/// answering it silently is what let a raw-text `postData` reach the wire as+/// mangled bytes with nothing anywhere saying so.- fn decode_optional_binary(++pub(crate) fn decode_optional_binary(+ params: &serde_json::Value,+ field: &str,+ what: &str,+) -> Result<Option<Vec<u8>>, String> {- let Some(raw) = params.get(field).and_then(|v| v.as_str()) else {++ let Some(value) = params.get(field) else {+ return Ok(None);+ };++ let raw = value.as_str().ok_or_else(|| format!("{what} must be a base64 string"))?;+ match decode_base64_bytes(raw) {+ Ok(bytes) => Ok(Some(bytes)),+ Err(why) => Err(format!("{what} is not valid base64: {why}")),}- let bytes: Vec<u8> = input.bytes().filter_map(val).collect();- let mut out = Vec::with_capacity(bytes.len() * 3 / 4);- for chunk in bytes.chunks(4) {- let b = [- chunk.first().copied().unwrap_or(0),- chunk.get(1).copied().unwrap_or(0),- chunk.get(2).copied().unwrap_or(0),- chunk.get(3).copied().unwrap_or(0),- ];- out.push((b[0] << 2) | (b[1] >> 4));- if chunk.len() > 2 { out.push((b[1] << 4) | (b[2] >> 2)); }- if chunk.len() > 3 { out.push((b[2] << 6) | b[3]); }- }- String::from_utf8_lossy(&out).to_string()+}+- /// Decode standard base64 to bytes, rejecting what is not base64.- ///- /// Padding may be omitted, since encoders differ on it and the length still- /// determines the group, but a character outside the alphabet and a group left- /// one symbol short are both refused.+fn decode_base64_bytes(input: &str) -> Result<Vec<u8>, String> {- fn val(c: u8) -> Option<u8> {- match c {- b'A'..=b'Z' => Some(c - b'A'),- b'a'..=b'z' => Some(c - b'a' + 26),- b'0'..=b'9' => Some(c - b'0' + 52),- b'+' => Some(62),- b'/' => Some(63),- _ => None,- }- }- let mut symbols: Vec<u8> = Vec::with_capacity(input.len());- let mut padding = 0usize;- for (index, c) in input.bytes().enumerate() {- if c == b'=' {- padding += 1;- continue;- }- if padding > 0 {- return Err(format!("character at byte {index} follows the padding"));- }- match val(c) {- Some(v) => symbols.push(v),- None => return Err(format!("byte {index} is not a base64 character")),- }- }- if padding > 2 {- return Err(format!("{padding} padding characters, at most 2 are valid"));- }- if symbols.len() % 4 == 1 {- return Err("the last group has one symbol, which encodes no byte".to_string());- }- let mut out = Vec::with_capacity(symbols.len() * 3 / 4);- for chunk in symbols.chunks(4) {- let b = [- chunk[0],- chunk.get(1).copied().unwrap_or(0),- chunk.get(2).copied().unwrap_or(0),- chunk.get(3).copied().unwrap_or(0),- ];- out.push((b[0] << 2) | (b[1] >> 4));- if chunk.len() > 2 { out.push((b[1] << 4) | (b[2] >> 2)); }- if chunk.len() > 3 { out.push((b[2] << 6) | b[3]); }- }- Ok(out)++ BASE64.decode(input).map_err(|error| error.to_string())}fn fast_path_response(text: &str) -> Option<String> {@@@ -2540,6 -3397,292 +3427,292 @@@ mod testsassert!(parse_cdp_headers(&json!({"url": "https://example.com"})).is_none());}+ async fn assert_navigation_intercept_reply_without_timer(method: &str) {+ use super::{process_with_interception, CdpMessage, ServerMessage, MAX_DEFERRED_MESSAGES};+ use tokio::io::{AsyncReadExt, AsyncWriteExt};+ use obscura_js::ops::{InterceptedRequest, InterceptResolution};++ tokio::task::LocalSet::new().run_until(async {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let (requested_tx, requested_rx) = tokio::sync::oneshot::channel();+ let (release_tx, release_rx) = tokio::sync::oneshot::channel();+ let fixture = tokio::task::spawn_local(async move {+ let (mut stream, _) = listener.accept().await.unwrap();+ let mut request = Vec::new();+ let mut buffer = [0u8; 1024];+ while !request.windows(4).any(|part| part == b"\r\n\r\n") {+ let count = stream.read(&mut buffer).await.unwrap();+ assert!(count > 0);+ request.extend_from_slice(&buffer[..count]);+ }+ requested_tx.send(()).unwrap();+ release_rx.await.unwrap();+ let body = b"<!doctype html><title>Finished</title><h1>Finished</h1>";+ let head = format!("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len());+ stream.write_all(head.as_bytes()).await.unwrap();+ stream.write_all(body).await.unwrap();+ });++ let context = obscura_browser::context::BrowserContext::with_storage_and_network(+ "interception-test".into(), None, false, None, None, true,+ );+ let mut ctx = crate::dispatch::CdpContext::new_with_shared_context(std::sync::Arc::new(context));+ let page_id = ctx.create_page();+ let session_id = format!("{page_id}-session");+ ctx.sessions.insert(session_id.clone(), page_id.clone());+ let (server_tx, mut server_rx) = tokio::sync::mpsc::unbounded_channel();+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();+ let (intercept_tx, intercept_rx) = tokio::sync::mpsc::unbounded_channel();+ ctx.intercept_tx = Some(intercept_tx.clone());+ let command = json!({"id": 1, "method": "Page.navigate", "sessionId": session_id,+ "params": {"url": format!("http://{address}/"), "waitUntil": "load"}}).to_string();+ let navigation_reply = reply_tx.clone();+ let navigation = tokio::task::spawn_local(async move {+ let mut intercept_rx = Some(intercept_rx);+ let mut paused = HashMap::new();+ let mut deferred = std::collections::VecDeque::new();+ process_with_interception(&command, &mut ctx, &navigation_reply,+ &mut server_rx, &mut intercept_rx, &mut paused, &mut deferred, true).await;+ (ctx, paused, deferred)+ });+ tokio::time::timeout(std::time::Duration::from_secs(5), requested_rx)+ .await.expect("navigation must reach the held response").unwrap();++ // The real socket is already connected and its response is gated.+ // Only the channel handshake below runs with a paused clock. An+ // unconditional timer auto-advances it, independent of host load.+ tokio::time::pause();+ let started = tokio::time::Instant::now();+ let (resolution_tx, resolution_rx) = tokio::sync::oneshot::channel();+ intercept_tx.send(InterceptedRequest {+ page_id: page_id.clone(),+ request_id: "pending-test".into(), url: format!("http://{address}/pending"),+ method: "GET".into(), headers: HashMap::new(), resource_type: "Fetch".into(),+ resolver: resolution_tx,+ }).unwrap();+ loop {+ let value: serde_json::Value = serde_json::from_str(+ &tokio::time::timeout(std::time::Duration::from_secs(1), reply_rx.recv())+ .await.expect("paused notification").unwrap()).unwrap();+ assert_ne!(value["id"], 1, "navigation must wait for its response");+ if value["method"] == "Fetch.requestPaused" {+ assert_eq!(value["params"]["requestId"], "pending-test");+ break;+ }+ }+ let send = |value: serde_json::Value| {+ server_tx.send(ServerMessage::Cdp(CdpMessage {+ text: value.to_string(), reply_tx: reply_tx.clone(),+ })).unwrap();+ };+ let foreign_count = if method == "failRequest" { MAX_DEFERRED_MESSAGES + 1 } else { 1 };+ for index in 0..foreign_count {+ send(json!({"id": 100 + index, "method": "Runtime.evaluate",+ "sessionId": session_id, "params": {"expression": "globalThis.foreignRan=true"}}));+ }+ let mut params = json!({"requestId": "pending-test"});+ match method {+ "continueRequest" => {+ params["url"] = json!("https://example.com/rewritten");+ params["method"] = json!("POST");+ params["headers"] = json!([{"name": "X-Check", "value": "kept"}]);- params["postData"] = json!("payload");++ params["postData"] = json!("cGF5bG9hZA==");+ }+ "fulfillRequest" => {+ params["responseCode"] = json!(201);+ params["responseHeaders"] = json!([{"name": "X-Check", "value": "kept"}]);+ params["body"] = json!("aGVsbG8=");+ }+ "failRequest" => params["errorReason"] = json!("Aborted"),+ _ => unreachable!(),+ }+ send(json!({"id": 10, "method": format!("Fetch.{method}"),+ "sessionId": session_id, "params": params}));+ let mut overflow_responses = 0;+ loop {+ let value: serde_json::Value = serde_json::from_str(+ &tokio::time::timeout(std::time::Duration::from_secs(1), reply_rx.recv())+ .await.expect("Fetch resolution reply").unwrap()).unwrap();+ assert_ne!(value["id"], 1, "must not acknowledge unfinished navigation");+ if value["id"] == 10 {+ assert!(value.get("error").is_none(), "{value}");+ break;+ }+ if value.get("id").is_some() {+ assert_eq!(value["id"], 100 + MAX_DEFERRED_MESSAGES);+ assert_eq!(value["error"]["code"], -32000);+ overflow_responses += 1;+ }+ }+ let elapsed = started.elapsed();+ match (method, resolution_rx.await.unwrap()) {+ ("continueRequest", InterceptResolution::Continue { url, method, headers, body }) => {+ assert_eq!(url.as_deref(), Some("https://example.com/rewritten"));+ assert_eq!(method.as_deref(), Some("POST"));+ assert_eq!(headers.unwrap().get("X-Check").map(String::as_str), Some("kept"));- assert_eq!(body.as_deref(), Some("payload"));++ assert_eq!(body.as_deref(), Some(b"payload".as_slice()));+ }+ ("fulfillRequest", InterceptResolution::Fulfill { status, headers, body, body_base64 }) => {+ assert_eq!(status, 201);+ assert_eq!(headers.get("X-Check").map(String::as_str), Some("kept"));+ assert_eq!(body, "hello");+ assert_eq!(body_base64, "aGVsbG8=");+ }+ ("failRequest", InterceptResolution::Fail { reason }) => assert_eq!(reason, "Aborted"),+ _ => panic!("resolution kind changed"),+ }+ tokio::time::resume();+ release_tx.send(()).unwrap();+ let (ctx, paused, deferred) = tokio::time::timeout(std::time::Duration::from_secs(5), navigation)+ .await.expect("navigation completion").unwrap();+ fixture.await.unwrap();+ assert_eq!(ctx.pages.len(), 1);+ assert_eq!(ctx.pages[0].url_string(), format!("http://{address}/"));+ assert!(paused.is_empty());+ assert_eq!(deferred.len(), foreign_count.min(MAX_DEFERRED_MESSAGES));+ assert_eq!(overflow_responses, usize::from(foreign_count > MAX_DEFERRED_MESSAGES));+ let replies: Vec<serde_json::Value> = std::iter::from_fn(|| reply_rx.try_recv().ok())+ .map(|text| serde_json::from_str(&text).unwrap()).collect();+ assert_eq!(replies.iter().filter(|reply| reply["id"] == 1).count(), 1);+ let navigation_reply = replies.iter().find(|reply| reply["id"] == 1).unwrap();+ assert!(navigation_reply.get("error").is_none(), "{navigation_reply}");+ assert!(!replies.iter().any(|reply| reply["id"] == 10), "no duplicate resolution reply");+ assert_eq!(elapsed, std::time::Duration::ZERO, "ready replies must not wait for a timer");+ }).await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn navigation_intercept_continue_has_no_timer_delay() {+ assert_navigation_intercept_reply_without_timer("continueRequest").await;+ }++ async fn assert_stop_loading_cancels_navigation(script: bool, method: &str) {+ use super::{process_cdp_message, process_with_interception, CdpMessage, ServerMessage};+ use tokio::io::{AsyncReadExt, AsyncWriteExt};++ tokio::task::LocalSet::new().run_until(async {+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let (requested_tx, requested_rx) = tokio::sync::oneshot::channel();+ let fixture = tokio::task::spawn_local(async move {+ if script {+ let (mut stream, _) = listener.accept().await.unwrap();+ let mut buffer = [0; 2048];+ stream.read(&mut buffer).await.unwrap();+ let body = "<!doctype html><title>Partial</title><script src='/held.js'></script>";+ let response = format!("HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", body.len());+ stream.write_all(response.as_bytes()).await.unwrap();+ }+ let (mut stream, _) = listener.accept().await.unwrap();+ let mut buffer = [0; 2048];+ assert!(stream.read(&mut buffer).await.unwrap() > 0);+ requested_tx.send(()).unwrap();+ // No response: cancellation must drop the pending HTTP request.+ assert_eq!(stream.read(&mut buffer).await.unwrap(), 0);+ });+ let context = obscura_browser::context::BrowserContext::with_storage_and_network(+ "stop-loading-test".into(), None, false, None, None, true,+ );+ let mut ctx = crate::dispatch::CdpContext::new_with_shared_context(std::sync::Arc::new(context));+ let page_id = ctx.create_page();+ let other_page = ctx.create_page();+ let session = format!("{page_id}-session");+ ctx.sessions.insert(session.clone(), page_id.clone());+ ctx.sessions.insert("raw-session".into(), page_id.clone());+ ctx.sessions.insert("other-session".into(), other_page.clone());+ ctx.get_page_mut(&page_id).unwrap().resume_js();+ let original_url = ctx.pages[0].url_string();+ let (server_tx, mut server_rx) = tokio::sync::mpsc::unbounded_channel();+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();+ let navigation_reply = reply_tx.clone();+ let url = format!("http://{address}/");+ let command = json!({"id": 1, "method": "Page.navigate", "sessionId": session,+ "params": {"url": url, "waitUntil": "load"}}).to_string();+ let navigation = tokio::task::spawn_local(async move {+ let mut paused = HashMap::new();+ let mut deferred = std::collections::VecDeque::new();+ process_with_interception(&command, &mut ctx, &navigation_reply,+ &mut server_rx, &mut None, &mut paused, &mut deferred, true).await;+ (ctx, paused, deferred)+ });+ tokio::time::timeout(std::time::Duration::from_secs(5), requested_rx)+ .await.expect("fixture request").unwrap();+ for (id, session) in [(2, "other-session"), (3, "raw-session")] {+ server_tx.send(ServerMessage::Cdp(CdpMessage {+ text: json!({"id": id, "method": if id == 2 { "Page.stopLoading" } else { method },+ "sessionId": session, "params": {"targetId": page_id}}).to_string(),+ reply_tx: reply_tx.clone(),+ })).unwrap();+ }+ let (mut ctx, paused, deferred) = tokio::time::timeout(std::time::Duration::from_secs(2), navigation)+ .await.expect("Page.stopLoading must cancel without waiting for the HTTP timeout").unwrap();+ tokio::time::timeout(std::time::Duration::from_secs(2), fixture)+ .await.expect("pending request socket must close").unwrap();+ assert!(paused.is_empty());+ assert_eq!(deferred.len(), if method == "Page.stopLoading" { 1 } else { 2 },+ "another target's stop must not cancel this navigation");+ for message in deferred {+ if let ServerMessage::Cdp(message) = message {+ process_cdp_message(&message.text, &mut ctx, &message.reply_tx).await;+ }+ }+ let replies: Vec<serde_json::Value> = std::iter::from_fn(|| reply_rx.try_recv().ok())+ .map(|text| serde_json::from_str(&text).unwrap()).collect();+ assert_eq!(replies.iter().filter(|reply| reply["id"] == 3).count(), 1);+ let stopped = replies.iter().find(|reply| reply["id"] == 3).unwrap();+ assert!(stopped.get("error").is_none(), "{stopped}");+ let aborted = replies.iter().find(|reply| reply["id"] == 1).unwrap();+ assert!(aborted.get("error").is_none(), "{aborted}");+ if !script {+ assert_eq!(aborted["result"]["errorText"], "net::ERR_ABORTED", "{aborted}");+ }+ assert!(!replies.iter().any(|reply| reply["method"] == "Page.loadEventFired"));+ assert!(replies.iter().any(|reply| reply["method"] == "Page.frameStoppedLoading"));+ assert!(ctx.get_page(&other_page).is_some());+ if method != "Page.stopLoading" {+ assert!(ctx.get_page(&page_id).is_none());+ assert_eq!(replies.iter().filter(|reply| reply["method"] == "Target.targetDestroyed").count(), 1);+ return;+ }+ assert_eq!(ctx.get_page(&page_id).unwrap().url_string(), if script { url } else { original_url });+ assert_eq!(ctx.get_page_mut(&page_id).unwrap().evaluate("1 + 2").as_f64(), Some(3.0));+ process_cdp_message(&json!({"id": 4, "method": "Page.navigate", "sessionId": session,+ "params": {"url": "about:blank"}}).to_string(), &mut ctx, &reply_tx).await;+ assert_eq!(ctx.get_page(&page_id).unwrap().url_string(), "about:blank");+ }).await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn stop_loading_cancels_a_held_document_request() {+ assert_stop_loading_cancels_navigation(false, "Page.stopLoading").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn stop_loading_cancels_a_held_parser_script_request() {+ assert_stop_loading_cancels_navigation(true, "Page.stopLoading").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn stop_loading_cancels_for_page_close() {+ assert_stop_loading_cancels_navigation(false, "Page.close").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn stop_loading_cancels_for_target_close() {+ assert_stop_loading_cancels_navigation(false, "Target.closeTarget").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn navigation_intercept_fulfill_has_no_timer_delay() {+ assert_navigation_intercept_reply_without_timer("fulfillRequest").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn navigation_intercept_fail_keeps_deferral_bounded_without_delay() {+ assert_navigation_intercept_reply_without_timer("failRequest").await;+ }+#[test]fn fetch_resolution_is_handled_once_by_the_outer_processor() {let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();@@@ -2561,164 -3707,131 +3737,299 @@@serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();assert_eq!(response["id"], 17);assert!(reply_rx.try_recv().is_err(), "must not emit a duplicate response");++ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ paused.insert("request-2".into(), resolution_tx);+ assert!(handle_fetch_resolution(+ r#"{"id":18,"method":"Fetch.failRequest","sessionId":"page-session","params":{"requestId":"request-2","errorReason":"Aborted"}}"#,+ &reply_tx, &mut paused,+ ));+ assert!(matches!(resolution_rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Fail { reason }) if reason == "Aborted"));+ let event: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();+ assert_eq!(event["method"], "Network.loadingFailed");+ assert_eq!(event["sessionId"], "page-session");+ assert_eq!(event["params"]["requestId"], "request-2");+ assert_eq!(event["params"]["errorText"], "net::ERR_ABORTED");+ assert_eq!(event["params"]["canceled"], true);+ let response: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();+ assert_eq!(response["id"], 18);+ assert!(reply_rx.try_recv().is_err(), "abort emits one failure and one response");+ }++ #[tokio::test(flavor = "current_thread")]+ async fn indexeddb_versionchange_rearms_an_idle_page_until_its_delayed_close() {+ let mut ctx = crate::dispatch::CdpContext::new();+ let mut ids = Vec::new();+ for _ in 0..2 {+ let id = ctx.create_page();+ let session = Some(format!("{id}-session"));+ ctx.sessions.insert(session.clone().unwrap(),id.clone());+ crate::domains::page::handle("navigate", &json!({"url":"data:text/html,<body>idb</body>"}), &mut ctx, &session).await.unwrap();+ ctx.get_page_mut(&id).unwrap().js.as_mut().unwrap().set_url("https://idb.test/");+ ids.push(id);+ }+ let mut idle = Default::default();+ ctx.get_page_mut(&ids[0]).unwrap().evaluate(r#"(() => {+ const r = indexedDB.open('idle-upgrade',1);+ globalThis.opened = false;+ r.onsuccess = () => {globalThis.opened = true;+ r.result.onversionchange = () => setTimeout(() => r.result.close(),10);};+ })()"#);+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while !pump_live_page_event_loop(&mut ctx,&mut idle).await.unwrap() {}+ }).await.unwrap();+ assert!(idle.contains(&ids[0]));+ assert_eq!(ctx.get_page_mut(&ids[0]).unwrap().evaluate("opened"),json!(true));+ ctx.get_page_mut(&ids[1]).unwrap().evaluate(r#"(() => {+ globalThis.upgraded = false;+ setTimeout(() => {const r = indexedDB.open('idle-upgrade',2);+ r.onsuccess = () => globalThis.upgraded = r.result.version === 2;},10);+ })()"#);+ idle.remove(&ids[1]);+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ loop {+ pump_live_page_event_loop(&mut ctx,&mut idle).await.unwrap();+ if ctx.get_page_mut(&ids[1]).unwrap().evaluate("upgraded") == json!(true) {break;}+ }+ }).await.expect("versionchange must wake the idle page and retain its delayed-close work");+ }++ #[tokio::test(flavor = "current_thread")]+ async fn autonomous_pages_do_not_wait_for_another_pages_distant_timer() {+ let mut ctx = crate::dispatch::CdpContext::new();+ let mut idle_pages = Default::default();+ let mut pages = Vec::new();+ for _ in 0..3 {+ let id = ctx.create_page();+ let session = Some(format!("{id}-session"));+ ctx.sessions.insert(session.clone().unwrap(), id.clone());+ crate::domains::page::handle(+ "navigate", &json!({"url": "data:text/html,<body>timer</body>"}),+ &mut ctx, &session,+ ).await.unwrap();+ pages.push(id);+ }+ ctx.get_page_mut(&pages[0]).unwrap().evaluate(+ "globalThis.slow = setTimeout(() => {}, 60000)",+ );+ for id in &pages[1..] {+ assert_eq!(ctx.get_page_mut(id).unwrap().evaluate(+ "(() => { globalThis.done = false; setTimeout(() => { globalThis.done = true; }, 20); return globalThis.done; })()",+ ), json!(false));+ }+ let started = std::time::Instant::now();+ let completed = tokio::time::timeout(std::time::Duration::from_secs(1), async {+ loop {+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();+ if pages[1..].iter().all(|id| {+ ctx.get_page_mut(id).unwrap().evaluate("globalThis.done") == json!(true)+ }) {+ break;+ }+ }+ }).await;+ assert!(completed.is_ok(), "a parked page must not prevent other pages' timers from running");+ assert!(started.elapsed() < std::time::Duration::from_millis(500),+ "ready timers must not need the outer timeout's wake: {:?}", started.elapsed());+ ctx.get_page_mut(&pages[2]).unwrap().evaluate(+ "setTimeout(() => { globalThis.finalTurn = true; }, 0)",+ );+ idle_pages.clear(); // A new command rearms previously idle pages.+ // This turn can do work and become idle in the same poll. Its result+ // must still be published while the first page remains parked.+ tokio::time::sleep(std::time::Duration::from_millis(50)).await;+ let started = std::time::Instant::now();+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ loop {+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();+ if ctx.get_page_mut(&pages[2]).unwrap().evaluate("globalThis.finalTurn") == json!(true) {+ break;+ }+ }+ }).await.expect("the last completed task must not remain buffered");+ assert!(started.elapsed() < std::time::Duration::from_millis(500),+ "a task becoming idle must still publish progress: {:?}", started.elapsed());+ let parked = tokio::time::timeout(std::time::Duration::from_millis(50), async {+ for _ in 0..8 {+ pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap();+ }+ }).await;+ assert!(parked.is_err(), "idle peers must not repeatedly wake a parked connection");+ ctx.get_page_mut(&pages[0]).unwrap().evaluate("clearTimeout(globalThis.slow)");+ idle_pages.clear();+ tokio::time::timeout(std::time::Duration::from_secs(1), async {+ while !pump_live_page_event_loop(&mut ctx, &mut idle_pages).await.unwrap() {}+ }).await.expect("all idle pages must disarm the pump instead of spinning");}+ // #569 / #365 — postData is declared `binary` in the CDP domain, so a+ // client sends it base64-encoded. Passing the encoded text straight through+ // made the request body the base64 itself, which a server reads as neither+ // the JSON nor the form the page meant to send.+ #[test]+ fn continue_request_decodes_a_base64_post_data() {+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();++ // base64 of {"a":1}+ assert!(handle_fetch_resolution(+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"eyJhIjoxfQ=="}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));++ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {+ panic!("continueRequest must resolve as Continue");+ };+ assert_eq!(+ body.as_deref(),+ Some(br#"{"a":1}"#.as_slice()),+ "the request body must be the bytes the client encoded, not the base64 text"+ );+ }++ // The body a client rewrites is not always text -- a protobuf, a gzip+ // stream, a multipart file part. Carrying it through `String` replaced+ // every non-UTF-8 sequence with U+FFFD before it reached the wire.+ #[test]+ fn continue_request_keeps_a_non_utf8_post_data_intact() {+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();++ // base64 of the three bytes 00 FF FE, which are not valid UTF-8+ assert!(handle_fetch_resolution(+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"AP/+"}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));++ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {+ panic!("continueRequest must resolve as Continue");+ };+ assert_eq!(+ body.as_deref(),+ Some([0x00u8, 0xFF, 0xFE].as_slice()),+ "a binary body must reach the wire byte for byte"+ );+ }++ // Base64 that is not base64 is a client mistake. Answering it silently is+ // how raw text reached the wire as mangled bytes; answering it with an+ // error while keeping the pause lets the client correct itself.+ #[test]+ fn continue_request_rejects_post_data_that_is_not_base64_without_wedging_it() {+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();++ assert!(handle_fetch_resolution(+ r#"{"id":9,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"!!!not-base64"}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));++ let rejected: serde_json::Value =+ serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();+ assert_eq!(rejected["id"], 9);+ assert_eq!(rejected["error"]["code"], -32602);+ assert!(+ rejected["error"]["message"]+ .as_str()+ .unwrap_or_default()+ .starts_with("continueRequest postData is not valid base64"),+ "the error must name the field, got {:?}",+ rejected["error"]["message"]+ );+ assert!(+ resolution_rx.try_recv().is_err(),+ "a rejected command must not resolve the pause"+ );+ assert!(+ paused.contains_key("request-1"),+ "a rejected command must leave the request paused so it can be answered again"+ );++ // the same requestId, answered correctly, still drives the fetch+ assert!(handle_fetch_resolution(+ r#"{"id":10,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":"eyJhIjoxfQ=="}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));+ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {+ panic!("the corrected continueRequest must resolve as Continue");+ };+ assert_eq!(body.as_deref(), Some(br#"{"a":1}"#.as_slice()));+ }++ // The same strictness on the other arm carrying a CDP `binary` field.+ #[test]+ fn fulfill_request_rejects_a_body_that_is_not_base64() {+ let (resolution_tx, _resolution_rx) = tokio::sync::oneshot::channel();+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);+ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();++ assert!(handle_fetch_resolution(+ r#"{"id":11,"method":"Fetch.fulfillRequest","params":{"requestId":"request-1","responseCode":200,"body":"<html>"}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));++ let rejected: serde_json::Value =+ serde_json::from_str(&reply_rx.try_recv().expect("one command response")).unwrap();+ assert_eq!(rejected["error"]["code"], -32602);+ assert!(+ rejected["error"]["message"]+ .as_str()+ .unwrap_or_default()+ .starts_with("fulfillRequest body is not valid base64")+ );+ assert!(paused.contains_key("request-1"), "the request must stay paused");+ }+++ #[test]++ fn continue_request_rejects_invalid_binary_values_and_allows_retry() {++ for value in [json!(null), json!(42), json!("AA="), json!("AB=="), json!("====")] {++ let (tx, mut rx) = tokio::sync::oneshot::channel();++ let mut paused = HashMap::from([("request-1".to_string(), tx)]);++ let (reply_tx, mut reply_rx) = tokio::sync::mpsc::unbounded_channel();++ let command = json!({"id": 1, "method": "Fetch.continueRequest",++ "params": {"requestId": "request-1", "postData": value}}).to_string();++ assert!(handle_fetch_resolution(&command, &reply_tx, &mut paused));++ let reply: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();++ assert_eq!(reply["error"]["code"], -32602, "accepted invalid postData: {value}");++ assert!(matches!(rx.try_recv(), Err(tokio::sync::oneshot::error::TryRecvError::Empty)));++ assert!(paused.contains_key("request-1"));++ assert!(handle_fetch_resolution(++ r#"{"id":2,"method":"Fetch.continueRequest","params":{"requestId":"request-1","postData":""}}"#,++ &reply_tx, &mut paused,++ ));++ assert!(matches!(rx.try_recv(), Ok(obscura_js::ops::InterceptResolution::Continue { body: Some(body), .. }) if body.is_empty()));++ let reply: serde_json::Value = serde_json::from_str(&reply_rx.try_recv().unwrap()).unwrap();++ assert!(reply.get("error").is_none());++ }++ }+++ // An absent postData still means "leave the body alone", not "send empty".+ #[test]+ fn continue_request_without_post_data_leaves_the_body_alone() {+ let (resolution_tx, mut resolution_rx) = tokio::sync::oneshot::channel();+ let mut paused = HashMap::from([("request-1".to_string(), resolution_tx)]);+ let (reply_tx, _reply_rx) = tokio::sync::mpsc::unbounded_channel::<String>();- let mut ctx = crate::dispatch::CdpContext::new();++ assert!(handle_fetch_resolution(+ r#"{"id":1,"method":"Fetch.continueRequest","params":{"requestId":"request-1"}}"#,- &mut ctx,+ &reply_tx,+ &mut paused,+ ));++ let Ok(obscura_js::ops::InterceptResolution::Continue { body, .. }) = resolution_rx.try_recv() else {+ panic!("continueRequest must resolve as Continue");+ };+ assert!(body.is_none(), "an absent postData must not become an empty body");+ }+#[cfg(feature = "render")]#[tokio::test(flavor = "current_thread")]async fn autonomous_screencast_pumps_timers_and_retains_backpressured_damage() {diff --cc docs/Use-as-a-Rust-library.mdindex 97df3f33,97df3f33..acb49400--- a/docs/Use-as-a-Rust-library.md+++ b/docs/Use-as-a-Rust-library.md@@@ -122,6 -122,6 +122,8 @@@ page.settle(2000).awaitA `Continue` with `url: Some(...)` rewrites the target. The new URL is re-checked against the SSRF / private-network gate, so a rewrite cannot reach an internal address that would otherwise need `--allow-private-network`.++`Continue.body` takes raw bytes as `Option<Vec<u8>>`, not a base64 string. Migrate text overrides from `Some(text)` to `Some(text.into_bytes())`. `None` preserves the original request body; `Some(Vec::new())` replaces it with an empty body. CDP clients still send base64 `postData` as specified by the protocol; Obscura decodes it before sending the request.++### Preload scripts`add_preload_script` runs a script before any of the page's own `<script>` tags (the CDP `Page.addScriptToEvaluateOnNewDocument` contract), so it can install hooks before the page bootstraps. Call it before `goto`.