The headless browser for AI agents and web scraping
fix(fetch): preserve final redirect response identity
Jernej Barbaric3f85ecdb4f88
diff --git a/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rsindex 3d1941ad..384d690d 100644--- a/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs+++ b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs@@ -11,7 +11,8 @@ use serde_json::{json, Value};use tokio::io::{AsyncReadExt, AsyncWriteExt};use tokio::net::TcpListener;-// Serves an HTML page that, on load, fetches /api/data.json, plus that JSON.+// Serves an HTML page that fetches /api/start.json, which redirects to the+// JSON response at /api/data.json.async fn serve() -> String {let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();let addr = listener.local_addr().unwrap();@@ -22,6 +23,11 @@ async fn serve() -> String {let mut buf = [0u8; 2048];let _ = socket.read(&mut buf).await.unwrap();let req = String::from_utf8_lossy(&buf[..]);+ if req.starts_with("GET /api/start.json") {+ let resp = "HTTP/1.1 302 Found\r\nLocation: /api/data.json\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";+ let _ = socket.write_all(resp.as_bytes()).await;+ return;+ }let (ct, body) = if req.starts_with("GET /api/data.json") {("application/json", "{\"value\":42}")} else {@@ -31,7 +37,7 @@ async fn serve() -> String {<div id="r">stage1</div><script>window.__done = new Promise(function (resolve) {- fetch("/api/data.json")+ fetch("/api/start.json").then(function (r) { return r.json(); }).then(function (d) { document.getElementById("r").textContent = "got:" + d.value; resolve("ok"); }).catch(function (e) { resolve("err:" + e); });@@ -117,7 +123,7 @@ async fn js_fetch_emits_network_request_and_response() {).await;- // The fetched JSON URL must appear as a requestWillBeSent event.+ // The final fetched JSON URL must appear as a requestWillBeSent event.let request_urls = ctx.pending_events.iter()@@ -128,7 +134,6 @@ async fn js_fetch_emits_network_request_and_response() {request_urls.iter().any(|u| u.contains("/api/data.json")),"script-initiated fetch must emit Network.requestWillBeSent; saw {request_urls:?}");-// And its response body must be resolvable via the same requestId, so a// client can read the captured JSON.let request_id = response_request_id(&ctx, "/api/data.json")diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.jsindex de67bd2d..537dd075 100644--- a/crates/obscura-js/js/bootstrap.js+++ b/crates/obscura-js/js/bootstrap.js@@ -7130,6 +7130,7 @@ globalThis.fetch = async (input, init = {}) => {const body = _serializeBody(initBody, _h, !(inheritsRequestBody && init.headers !== undefined));const hdrs = JSON.stringify(_h);const fetchMode = init.mode || (request ? request.mode : "cors");+ const fetchRedirect = init.redirect || (request ? request.redirect : "follow");const fetchCredentials = init.credentials !== undefined? String(init.credentials): (request ? request.credentials : "same-origin");@@ -7148,15 +7149,16 @@ globalThis.fetch = async (input, init = {}) => {if (parsed.corsBlocked) {throw new TypeError('Failed to fetch: ' + (parsed.corsError || 'CORS error'));}- const respType = parsed.status === 0 ? "opaque" : (fetchMode === "no-cors" ? "opaque" : "basic");+ const respType = parsed.status === 0 || parsed.opaque ? "opaque" : "basic";+ const exposeRedirectMetadata = respType !== "opaque" && fetchRedirect === "follow";const responseBody = parsed.bodyBase64 ? _base64ToUint8Array(parsed.bodyBase64) : (parsed.body || "");const response = new Response(responseBody, {status: parsed.status,statusText: "",headers: parsed.headers || {},type: respType,- url: parsed.url || url,- redirected: false,+ url: exposeRedirectMetadata ? (parsed.url || url) : (respType === "opaque" ? "" : url),+ redirected: exposeRedirectMetadata && !!parsed.redirected,});if (parsed.requestId) {Object.defineProperty(response, "__obscuraRequestId", {diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rsindex 0a3fcfbc..518709a3 100644--- a/crates/obscura-js/src/ops.rs+++ b/crates/obscura-js/src/ops.rs@@ -2605,6 +2605,8 @@ async fn op_fetch_url(let mut current_method = req_method;let mut current_body = body;let mut redirects_followed: usize = 0;+ let mut redirected_from = Vec::new();+ let mut crossed_origin = is_cross_origin;let response = loop {let mut req = client.request(current_method.clone(), ¤t_url)@@ -2613,6 +2615,7 @@ async fn op_fetch_url(let current_is_cross_origin = request_origin(¤t_url).map(|request_origin| request_origin != page_origin).unwrap_or(false);+ crossed_origin |= current_is_cross_origin;if current_is_cross_origin {req = req.header("Origin", &page_origin);}@@ -2734,9 +2737,11 @@ async fn op_fetch_url(current_body.clear();}+ redirected_from.push(base);current_url = next_url.to_string();};+ let redirected = redirects_followed > 0;let status = response.status().as_u16();let resp_headers: std::collections::HashMap<String, String> = response@@ -2783,10 +2788,16 @@ async fn op_fetch_url(let resp_body_base64 = BASE64.encode(&resp_bytes);if let Some(ref cbs) = callbacks {if cbs.has_response_callbacks().await {- let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.to_vec());+ let resp = fetch_response(+ current_url.as_str(),+ status,+ resp_headers.clone(),+ resp_bytes.to_vec(),+ redirected_from,+ );let info = RequestInfo {url: resp.url.clone(),- method: method.clone(),+ method: current_method.as_str().to_string(),headers: resp_headers.clone(),resource_type: ResourceType::Fetch,};@@ -2826,8 +2837,8 @@ async fn op_fetch_url(.as_secs_f64();gs.js_network_events.push(JsNetworkEvent {request_id: request_id.clone(),- url: url.clone(),- method: method.clone(),+ url: current_url.clone(),+ method: current_method.as_str().to_string(),status,response_headers: resp_headers.clone(),body_size: resp_bytes.len(),@@ -2853,7 +2864,9 @@ async fn op_fetch_url("body": resp_body,"bodyBase64": resp_body_base64,"requestId": response_request_id,- "url": url,+ "url": current_url,+ "redirected": redirected,+ "opaque": mode == "no-cors" && crossed_origin,"headers": resp_headers,}).to_string())@@ -2867,13 +2880,14 @@ fn fetch_response(status: u16,headers: HashMap<String, String>,body: Vec<u8>,+ redirected_from: Vec<url::Url>,) -> Response {Response {url: url::Url::parse(url).unwrap_or_else(|_| url::Url::parse("http://0.0.0.0/").unwrap()),status,headers,body,- redirected_from: Vec::new(),+ redirected_from,}}@@ -2900,6 +2914,10 @@ async fn stealth_fetch_all(let mut current_method = method;let mut current_body = body;let mut redirects_followed: usize = 0;+ let mut redirected_from = Vec::new();+ let mut crossed_origin = request_origin(¤t_url)+ .map(|request_origin| request_origin != page_origin)+ .unwrap_or(false);let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {let parsed_current = match url::Url::parse(¤t_url) {@@ -2914,6 +2932,7 @@ async fn stealth_fetch_all(let mut req_headers: HashMap<String, String> = HashMap::new();let current_is_cross_origin = parsed_current.origin().ascii_serialization() != page_origin;+ crossed_origin |= current_is_cross_origin;if current_is_cross_origin {req_headers.insert("origin".to_string(), page_origin.clone());}@@ -2968,6 +2987,7 @@ async fn stealth_fetch_all(current_method = "GET".to_string();current_body.clear();}+ redirected_from.push(parsed_current);current_url = next_url.to_string();};@@ -3007,7 +3027,13 @@ async fn stealth_fetch_all(let resp_body_base64 = BASE64.encode(&resp_bytes);if let Some(ref cbs) = callbacks {if cbs.has_response_callbacks().await {- let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.clone());+ let resp = fetch_response(+ current_url.as_str(),+ status,+ resp_headers.clone(),+ resp_bytes.clone(),+ redirected_from,+ );let info = RequestInfo {url: resp.url.clone(),method: current_method.clone(),@@ -3022,7 +3048,9 @@ async fn stealth_fetch_all("status": status,"body": resp_body,"bodyBase64": resp_body_base64,- "url": url,+ "url": current_url,+ "redirected": redirects_followed > 0,+ "opaque": mode == "no-cors" && crossed_origin,"headers": resp_headers,}).to_string())diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rsindex ddef284f..6376aa71 100644--- a/crates/obscura-js/src/runtime.rs+++ b/crates/obscura-js/src/runtime.rs@@ -15496,7 +15496,10 @@ mod tests {}});- let origin = format!("http://{address}");+ redirect_runtime_for_origin(&format!("http://{address}"))+ }++ fn redirect_runtime_for_origin(origin: &str) -> ObscuraJsRuntime {let mut rt = ObscuraJsRuntime::new();rt.set_dom(parse_html("<html><body></body></html>"));rt.set_url(&format!("{origin}/page"));@@ -15511,6 +15514,74 @@ mod tests {rt}+ fn redirect_method_runtime(+ status: u16,+ ) -> (+ ObscuraJsRuntime,+ std::sync::Arc<std::sync::Mutex<Vec<String>>>,+ ) {+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let address = listener.local_addr().unwrap();+ let requests = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));+ let requests_capture = requests.clone();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};++ for _ in 0..2 {+ let (mut stream, _) = listener.accept().unwrap();+ let mut buffer = [0u8; 2048];+ let read = stream.read(&mut buffer).unwrap_or(0);+ let request = String::from_utf8_lossy(&buffer[..read]);+ requests_capture+ .lock()+ .unwrap()+ .push(request.lines().next().unwrap_or_default().to_string());+ let response = if request.contains(" /start ") {+ format!(+ "HTTP/1.1 {status} Redirect\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"+ )+ } else {+ "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string()+ };+ let _ = stream.write_all(response.as_bytes());+ }+ });++ (+ redirect_runtime_for_origin(&format!("http://{address}")),+ requests,+ )+ }++ fn cross_origin_redirect_runtime() -> ObscuraJsRuntime {+ let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let target_address = target.local_addr().unwrap();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};+ let (mut stream, _) = target.accept().unwrap();+ let mut buffer = [0u8; 1024];+ let _ = stream.read(&mut buffer);+ let _ = stream.write_all(+ b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",+ );+ });++ let source = std::net::TcpListener::bind("127.0.0.1:0").unwrap();+ let source_address = source.local_addr().unwrap();+ std::thread::spawn(move || {+ use std::io::{Read as _, Write as _};+ let (mut stream, _) = source.accept().unwrap();+ let mut buffer = [0u8; 1024];+ let _ = stream.read(&mut buffer);+ let response = format!(+ "HTTP/1.1 302 Found\r\nLocation: http://{target_address}/final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"+ );+ let _ = stream.write_all(response.as_bytes());+ });++ redirect_runtime_for_origin(&format!("http://{source_address}"))+ }+/// HTTP-redirect fetch returns a network error as soon as the/// redirect count *reaches* 20, and only increments it afterwards. So/// the twentieth hop must still succeed:@@ -15533,6 +15604,248 @@ mod tests {assert_eq!(result.value.unwrap(), serde_json::json!("arrived"));}+ #[tokio::test(flavor = "current_thread")]+ async fn fetch_response_reports_the_final_redirect_url() {+ let mut rt = redirect_chain_runtime(3);+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/hop/1");+ const direct = await fetch("/hop/0");+ return {+ url: response.url,+ redirected: response.redirected,+ cloneUrl: response.clone().url,+ directRedirected: direct.redirected,+ };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();++ let value = result.value.unwrap();+ assert!(+ value["url"].as_str().unwrap_or_default().ends_with("/hop/0"),+ "Response.url did not report the final URL: {value}"+ );+ assert_eq!(value["redirected"], true);+ assert_eq!(value["cloneUrl"], value["url"]);+ assert_eq!(value["directRedirected"], false);++ let events = rt.take_js_network_events();+ assert_eq!(events.len(), 2);+ assert!(+ events[0].url.ends_with("/hop/0"),+ "network response event did not report the final URL: {:?}",+ events[0].url+ );+ }++ #[cfg(feature = "stealth")]+ #[tokio::test(flavor = "current_thread")]+ async fn stealth_fetch_response_reports_the_final_redirect_url() {+ let mut rt = redirect_chain_runtime(2);+ rt.set_stealth_client(std::sync::Arc::new(+ obscura_net::StealthHttpClient::new(std::sync::Arc::new(+ obscura_net::CookieJar::new(),+ )),+ ));+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/hop/1");+ return { url: response.url, redirected: response.redirected };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();++ let value = result.value.unwrap();+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/hop/0"));+ assert_eq!(value["redirected"], true);+ }++ #[tokio::test(flavor = "current_thread")]+ async fn xhr_response_url_reports_the_final_redirect_url() {+ let mut rt = redirect_chain_runtime(2);+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => await new Promise((resolve, reject) => {+ const xhr = new XMLHttpRequest();+ xhr.open("GET", "/hop/1");+ xhr.onload = () => resolve(xhr.responseURL);+ xhr.onerror = reject;+ xhr.send();+ })"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();++ assert!(+ result+ .value+ .unwrap()+ .as_str()+ .unwrap_or_default()+ .ends_with("/hop/0")+ );+ }++ async fn assert_post_redirect_method(status: u16, expected_method: &str) {+ let (mut rt, wire_requests) = redirect_method_runtime(status);+ let request_callbacks = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));+ let response_callbacks = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));+ let callbacks = std::sync::Arc::new(obscura_net::CallbackRegistry::new());+ let request_capture = request_callbacks.clone();+ callbacks.add_request(std::sync::Arc::new(move |request| {+ request_capture+ .lock()+ .unwrap()+ .push((request.method.clone(), request.url.path().to_string()));+ }));+ let response_capture = response_callbacks.clone();+ callbacks.add_response(std::sync::Arc::new(move |request, response| {+ response_capture.lock().unwrap().push((+ request.method.clone(),+ request.url.path().to_string(),+ response.url.path().to_string(),+ response.redirected_from.len(),+ ));+ }));+ rt.set_callbacks(callbacks);++ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/start", { method: "POST", body: "payload" });+ return { url: response.url, redirected: response.redirected };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();+ let value = result.value.unwrap();+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/final"));+ assert_eq!(value["redirected"], true);+ assert_eq!(+ *request_callbacks.lock().unwrap(),+ vec![("POST".to_string(), "/start".to_string())]+ );+ assert_eq!(+ *response_callbacks.lock().unwrap(),+ vec![(+ expected_method.to_string(),+ "/final".to_string(),+ "/final".to_string(),+ 1,+ )]+ );+ let events = rt.take_js_network_events();+ assert_eq!(events.len(), 1);+ assert_eq!(events[0].method, expected_method);+ assert!(events[0].url.ends_with("/final"));+ let wire_requests = wire_requests.lock().unwrap();+ assert!(wire_requests[0].starts_with("POST /start "));+ assert!(wire_requests[1].starts_with(&format!("{expected_method} /final ")));+ }++ #[tokio::test(flavor = "current_thread")]+ async fn fetch_302_response_uses_the_final_get_method() {+ assert_post_redirect_method(302, "GET").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn fetch_307_response_preserves_the_post_method() {+ assert_post_redirect_method(307, "POST").await;+ }++ #[tokio::test(flavor = "current_thread")]+ async fn same_origin_no_cors_redirect_keeps_response_identity() {+ let mut rt = redirect_chain_runtime(2);+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/hop/1", { mode: "no-cors" });+ return { type: response.type, url: response.url, redirected: response.redirected };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();+ let value = result.value.unwrap();+ assert_eq!(value["type"], "basic");+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/hop/0"));+ assert_eq!(value["redirected"], true);+ }++ #[tokio::test(flavor = "current_thread")]+ async fn cross_origin_no_cors_redirect_filters_response_identity() {+ let mut rt = cross_origin_redirect_runtime();+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/start", { mode: "no-cors" });+ return { type: response.type, url: response.url, redirected: response.redirected };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();+ assert_eq!(+ result.value.unwrap(),+ serde_json::json!({ "type": "opaque", "url": "", "redirected": false })+ );+ }++ #[cfg(feature = "stealth")]+ #[tokio::test(flavor = "current_thread")]+ async fn stealth_cross_origin_no_cors_filters_response_identity() {+ let mut rt = cross_origin_redirect_runtime();+ rt.set_stealth_client(std::sync::Arc::new(+ obscura_net::StealthHttpClient::new(std::sync::Arc::new(+ obscura_net::CookieJar::new(),+ )),+ ));+ let result = rt+ .call_function_on_for_cdp(+ r#"async () => {+ const response = await fetch("/start", { mode: "no-cors" });+ return { type: response.type, url: response.url, redirected: response.redirected };+ }"#,+ None,+ &[],+ true,+ true,+ )+ .await+ .unwrap();+ assert_eq!(+ result.value.unwrap(),+ serde_json::json!({ "type": "opaque", "url": "", "redirected": false })+ );+ }+/// The other end of the same pair: the twenty-first redirect must/// fail. `fetch` reports a rejected result as a `TypeError`.#[tokio::test(flavor = "current_thread")]