Skip to content

The headless browser for AI agents and web scraping

fix(fetch): preserve final redirect response identity

Jernej Barbaric3f85ecdb4f88
diff --git a/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs
index 3d1941ad..384d690d 100644
--- a/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs
+++ b/crates/obscura-cdp/tests/js_fetch_emits_network_events.rs
@@ -11,7 +11,8 @@ use serde_json::{json, Value};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
-// Serves an HTML page that, on load, fetches /api/data.json, plus that JSON.
+// Serves an HTML page that fetches /api/start.json, which redirects to the
+// JSON response at /api/data.json.
async fn serve() -> String {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
@@ -22,6 +23,11 @@ async fn serve() -> String {
let mut buf = [0u8; 2048];
let _ = socket.read(&mut buf).await.unwrap();
let req = String::from_utf8_lossy(&buf[..]);
+ if req.starts_with("GET /api/start.json") {
+ let resp = "HTTP/1.1 302 Found\r\nLocation: /api/data.json\r\nContent-Length: 0\r\nConnection: close\r\n\r\n";
+ let _ = socket.write_all(resp.as_bytes()).await;
+ return;
+ }
let (ct, body) = if req.starts_with("GET /api/data.json") {
("application/json", "{\"value\":42}")
} else {
@@ -31,7 +37,7 @@ async fn serve() -> String {
<div id="r">stage1</div>
<script>
window.__done = new Promise(function (resolve) {
- fetch("/api/data.json")
+ fetch("/api/start.json")
.then(function (r) { return r.json(); })
.then(function (d) { document.getElementById("r").textContent = "got:" + d.value; resolve("ok"); })
.catch(function (e) { resolve("err:" + e); });
@@ -117,7 +123,7 @@ async fn js_fetch_emits_network_request_and_response() {
)
.await;
- // The fetched JSON URL must appear as a requestWillBeSent event.
+ // The final fetched JSON URL must appear as a requestWillBeSent event.
let request_urls = ctx
.pending_events
.iter()
@@ -128,7 +134,6 @@ async fn js_fetch_emits_network_request_and_response() {
request_urls.iter().any(|u| u.contains("/api/data.json")),
"script-initiated fetch must emit Network.requestWillBeSent; saw {request_urls:?}"
);
-
// And its response body must be resolvable via the same requestId, so a
// client can read the captured JSON.
let request_id = response_request_id(&ctx, "/api/data.json")
diff --git a/crates/obscura-js/js/bootstrap.js b/crates/obscura-js/js/bootstrap.js
index de67bd2d..537dd075 100644
--- a/crates/obscura-js/js/bootstrap.js
+++ b/crates/obscura-js/js/bootstrap.js
@@ -7130,6 +7130,7 @@ globalThis.fetch = async (input, init = {}) => {
const body = _serializeBody(initBody, _h, !(inheritsRequestBody && init.headers !== undefined));
const hdrs = JSON.stringify(_h);
const fetchMode = init.mode || (request ? request.mode : "cors");
+ const fetchRedirect = init.redirect || (request ? request.redirect : "follow");
const fetchCredentials = init.credentials !== undefined
? String(init.credentials)
: (request ? request.credentials : "same-origin");
@@ -7148,15 +7149,16 @@ globalThis.fetch = async (input, init = {}) => {
if (parsed.corsBlocked) {
throw new TypeError('Failed to fetch: ' + (parsed.corsError || 'CORS error'));
}
- const respType = parsed.status === 0 ? "opaque" : (fetchMode === "no-cors" ? "opaque" : "basic");
+ const respType = parsed.status === 0 || parsed.opaque ? "opaque" : "basic";
+ const exposeRedirectMetadata = respType !== "opaque" && fetchRedirect === "follow";
const responseBody = parsed.bodyBase64 ? _base64ToUint8Array(parsed.bodyBase64) : (parsed.body || "");
const response = new Response(responseBody, {
status: parsed.status,
statusText: "",
headers: parsed.headers || {},
type: respType,
- url: parsed.url || url,
- redirected: false,
+ url: exposeRedirectMetadata ? (parsed.url || url) : (respType === "opaque" ? "" : url),
+ redirected: exposeRedirectMetadata && !!parsed.redirected,
});
if (parsed.requestId) {
Object.defineProperty(response, "__obscuraRequestId", {
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index 0a3fcfbc..518709a3 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -2605,6 +2605,8 @@ async fn op_fetch_url(
let mut current_method = req_method;
let mut current_body = body;
let mut redirects_followed: usize = 0;
+ let mut redirected_from = Vec::new();
+ let mut crossed_origin = is_cross_origin;
let response = loop {
let mut req = client
.request(current_method.clone(), &current_url)
@@ -2613,6 +2615,7 @@ async fn op_fetch_url(
let current_is_cross_origin = request_origin(&current_url)
.map(|request_origin| request_origin != page_origin)
.unwrap_or(false);
+ crossed_origin |= current_is_cross_origin;
if current_is_cross_origin {
req = req.header("Origin", &page_origin);
}
@@ -2734,9 +2737,11 @@ async fn op_fetch_url(
current_body.clear();
}
+ redirected_from.push(base);
current_url = next_url.to_string();
};
+ let redirected = redirects_followed > 0;
let status = response.status().as_u16();
let resp_headers: std::collections::HashMap<String, String> = response
@@ -2783,10 +2788,16 @@ async fn op_fetch_url(
let resp_body_base64 = BASE64.encode(&resp_bytes);
if let Some(ref cbs) = callbacks {
if cbs.has_response_callbacks().await {
- let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.to_vec());
+ let resp = fetch_response(
+ current_url.as_str(),
+ status,
+ resp_headers.clone(),
+ resp_bytes.to_vec(),
+ redirected_from,
+ );
let info = RequestInfo {
url: resp.url.clone(),
- method: method.clone(),
+ method: current_method.as_str().to_string(),
headers: resp_headers.clone(),
resource_type: ResourceType::Fetch,
};
@@ -2826,8 +2837,8 @@ async fn op_fetch_url(
.as_secs_f64();
gs.js_network_events.push(JsNetworkEvent {
request_id: request_id.clone(),
- url: url.clone(),
- method: method.clone(),
+ url: current_url.clone(),
+ method: current_method.as_str().to_string(),
status,
response_headers: resp_headers.clone(),
body_size: resp_bytes.len(),
@@ -2853,7 +2864,9 @@ async fn op_fetch_url(
"body": resp_body,
"bodyBase64": resp_body_base64,
"requestId": response_request_id,
- "url": url,
+ "url": current_url,
+ "redirected": redirected,
+ "opaque": mode == "no-cors" && crossed_origin,
"headers": resp_headers,
})
.to_string())
@@ -2867,13 +2880,14 @@ fn fetch_response(
status: u16,
headers: HashMap<String, String>,
body: Vec<u8>,
+ redirected_from: Vec<url::Url>,
) -> Response {
Response {
url: url::Url::parse(url).unwrap_or_else(|_| url::Url::parse("http://0.0.0.0/").unwrap()),
status,
headers,
body,
- redirected_from: Vec::new(),
+ redirected_from,
}
}
@@ -2900,6 +2914,10 @@ async fn stealth_fetch_all(
let mut current_method = method;
let mut current_body = body;
let mut redirects_followed: usize = 0;
+ let mut redirected_from = Vec::new();
+ let mut crossed_origin = request_origin(&current_url)
+ .map(|request_origin| request_origin != page_origin)
+ .unwrap_or(false);
let (status, resp_headers, resp_bytes): (u16, HashMap<String, String>, Vec<u8>) = loop {
let parsed_current = match url::Url::parse(&current_url) {
@@ -2914,6 +2932,7 @@ async fn stealth_fetch_all(
let mut req_headers: HashMap<String, String> = HashMap::new();
let current_is_cross_origin = parsed_current.origin().ascii_serialization() != page_origin;
+ crossed_origin |= current_is_cross_origin;
if current_is_cross_origin {
req_headers.insert("origin".to_string(), page_origin.clone());
}
@@ -2968,6 +2987,7 @@ async fn stealth_fetch_all(
current_method = "GET".to_string();
current_body.clear();
}
+ redirected_from.push(parsed_current);
current_url = next_url.to_string();
};
@@ -3007,7 +3027,13 @@ async fn stealth_fetch_all(
let resp_body_base64 = BASE64.encode(&resp_bytes);
if let Some(ref cbs) = callbacks {
if cbs.has_response_callbacks().await {
- let resp = fetch_response(&url, status, resp_headers.clone(), resp_bytes.clone());
+ let resp = fetch_response(
+ current_url.as_str(),
+ status,
+ resp_headers.clone(),
+ resp_bytes.clone(),
+ redirected_from,
+ );
let info = RequestInfo {
url: resp.url.clone(),
method: current_method.clone(),
@@ -3022,7 +3048,9 @@ async fn stealth_fetch_all(
"status": status,
"body": resp_body,
"bodyBase64": resp_body_base64,
- "url": url,
+ "url": current_url,
+ "redirected": redirects_followed > 0,
+ "opaque": mode == "no-cors" && crossed_origin,
"headers": resp_headers,
})
.to_string())
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index ddef284f..6376aa71 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -15496,7 +15496,10 @@ mod tests {
}
});
- let origin = format!("http://{address}");
+ redirect_runtime_for_origin(&format!("http://{address}"))
+ }
+
+ fn redirect_runtime_for_origin(origin: &str) -> ObscuraJsRuntime {
let mut rt = ObscuraJsRuntime::new();
rt.set_dom(parse_html("<html><body></body></html>"));
rt.set_url(&format!("{origin}/page"));
@@ -15511,6 +15514,74 @@ mod tests {
rt
}
+ fn redirect_method_runtime(
+ status: u16,
+ ) -> (
+ ObscuraJsRuntime,
+ std::sync::Arc<std::sync::Mutex<Vec<String>>>,
+ ) {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let address = listener.local_addr().unwrap();
+ let requests = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
+ let requests_capture = requests.clone();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+
+ for _ in 0..2 {
+ let (mut stream, _) = listener.accept().unwrap();
+ let mut buffer = [0u8; 2048];
+ let read = stream.read(&mut buffer).unwrap_or(0);
+ let request = String::from_utf8_lossy(&buffer[..read]);
+ requests_capture
+ .lock()
+ .unwrap()
+ .push(request.lines().next().unwrap_or_default().to_string());
+ let response = if request.contains(" /start ") {
+ format!(
+ "HTTP/1.1 {status} Redirect\r\nLocation: /final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ )
+ } else {
+ "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok".to_string()
+ };
+ let _ = stream.write_all(response.as_bytes());
+ }
+ });
+
+ (
+ redirect_runtime_for_origin(&format!("http://{address}")),
+ requests,
+ )
+ }
+
+ fn cross_origin_redirect_runtime() -> ObscuraJsRuntime {
+ let target = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let target_address = target.local_addr().unwrap();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+ let (mut stream, _) = target.accept().unwrap();
+ let mut buffer = [0u8; 1024];
+ let _ = stream.read(&mut buffer);
+ let _ = stream.write_all(
+ b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok",
+ );
+ });
+
+ let source = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let source_address = source.local_addr().unwrap();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+ let (mut stream, _) = source.accept().unwrap();
+ let mut buffer = [0u8; 1024];
+ let _ = stream.read(&mut buffer);
+ let response = format!(
+ "HTTP/1.1 302 Found\r\nLocation: http://{target_address}/final\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ );
+ let _ = stream.write_all(response.as_bytes());
+ });
+
+ redirect_runtime_for_origin(&format!("http://{source_address}"))
+ }
+
/// HTTP-redirect fetch returns a network error as soon as the
/// redirect count *reaches* 20, and only increments it afterwards. So
/// the twentieth hop must still succeed:
@@ -15533,6 +15604,248 @@ mod tests {
assert_eq!(result.value.unwrap(), serde_json::json!("arrived"));
}
+ #[tokio::test(flavor = "current_thread")]
+ async fn fetch_response_reports_the_final_redirect_url() {
+ let mut rt = redirect_chain_runtime(3);
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/hop/1");
+ const direct = await fetch("/hop/0");
+ return {
+ url: response.url,
+ redirected: response.redirected,
+ cloneUrl: response.clone().url,
+ directRedirected: direct.redirected,
+ };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+
+ let value = result.value.unwrap();
+ assert!(
+ value["url"].as_str().unwrap_or_default().ends_with("/hop/0"),
+ "Response.url did not report the final URL: {value}"
+ );
+ assert_eq!(value["redirected"], true);
+ assert_eq!(value["cloneUrl"], value["url"]);
+ assert_eq!(value["directRedirected"], false);
+
+ let events = rt.take_js_network_events();
+ assert_eq!(events.len(), 2);
+ assert!(
+ events[0].url.ends_with("/hop/0"),
+ "network response event did not report the final URL: {:?}",
+ events[0].url
+ );
+ }
+
+ #[cfg(feature = "stealth")]
+ #[tokio::test(flavor = "current_thread")]
+ async fn stealth_fetch_response_reports_the_final_redirect_url() {
+ let mut rt = redirect_chain_runtime(2);
+ rt.set_stealth_client(std::sync::Arc::new(
+ obscura_net::StealthHttpClient::new(std::sync::Arc::new(
+ obscura_net::CookieJar::new(),
+ )),
+ ));
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/hop/1");
+ return { url: response.url, redirected: response.redirected };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+
+ let value = result.value.unwrap();
+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/hop/0"));
+ assert_eq!(value["redirected"], true);
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn xhr_response_url_reports_the_final_redirect_url() {
+ let mut rt = redirect_chain_runtime(2);
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => await new Promise((resolve, reject) => {
+ const xhr = new XMLHttpRequest();
+ xhr.open("GET", "/hop/1");
+ xhr.onload = () => resolve(xhr.responseURL);
+ xhr.onerror = reject;
+ xhr.send();
+ })"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+
+ assert!(
+ result
+ .value
+ .unwrap()
+ .as_str()
+ .unwrap_or_default()
+ .ends_with("/hop/0")
+ );
+ }
+
+ async fn assert_post_redirect_method(status: u16, expected_method: &str) {
+ let (mut rt, wire_requests) = redirect_method_runtime(status);
+ let request_callbacks = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
+ let response_callbacks = std::sync::Arc::new(std::sync::Mutex::new(Vec::new()));
+ let callbacks = std::sync::Arc::new(obscura_net::CallbackRegistry::new());
+ let request_capture = request_callbacks.clone();
+ callbacks.add_request(std::sync::Arc::new(move |request| {
+ request_capture
+ .lock()
+ .unwrap()
+ .push((request.method.clone(), request.url.path().to_string()));
+ }));
+ let response_capture = response_callbacks.clone();
+ callbacks.add_response(std::sync::Arc::new(move |request, response| {
+ response_capture.lock().unwrap().push((
+ request.method.clone(),
+ request.url.path().to_string(),
+ response.url.path().to_string(),
+ response.redirected_from.len(),
+ ));
+ }));
+ rt.set_callbacks(callbacks);
+
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/start", { method: "POST", body: "payload" });
+ return { url: response.url, redirected: response.redirected };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+ let value = result.value.unwrap();
+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/final"));
+ assert_eq!(value["redirected"], true);
+ assert_eq!(
+ *request_callbacks.lock().unwrap(),
+ vec![("POST".to_string(), "/start".to_string())]
+ );
+ assert_eq!(
+ *response_callbacks.lock().unwrap(),
+ vec![(
+ expected_method.to_string(),
+ "/final".to_string(),
+ "/final".to_string(),
+ 1,
+ )]
+ );
+ let events = rt.take_js_network_events();
+ assert_eq!(events.len(), 1);
+ assert_eq!(events[0].method, expected_method);
+ assert!(events[0].url.ends_with("/final"));
+ let wire_requests = wire_requests.lock().unwrap();
+ assert!(wire_requests[0].starts_with("POST /start "));
+ assert!(wire_requests[1].starts_with(&format!("{expected_method} /final ")));
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn fetch_302_response_uses_the_final_get_method() {
+ assert_post_redirect_method(302, "GET").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn fetch_307_response_preserves_the_post_method() {
+ assert_post_redirect_method(307, "POST").await;
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn same_origin_no_cors_redirect_keeps_response_identity() {
+ let mut rt = redirect_chain_runtime(2);
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/hop/1", { mode: "no-cors" });
+ return { type: response.type, url: response.url, redirected: response.redirected };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+ let value = result.value.unwrap();
+ assert_eq!(value["type"], "basic");
+ assert!(value["url"].as_str().unwrap_or_default().ends_with("/hop/0"));
+ assert_eq!(value["redirected"], true);
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn cross_origin_no_cors_redirect_filters_response_identity() {
+ let mut rt = cross_origin_redirect_runtime();
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/start", { mode: "no-cors" });
+ return { type: response.type, url: response.url, redirected: response.redirected };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ result.value.unwrap(),
+ serde_json::json!({ "type": "opaque", "url": "", "redirected": false })
+ );
+ }
+
+ #[cfg(feature = "stealth")]
+ #[tokio::test(flavor = "current_thread")]
+ async fn stealth_cross_origin_no_cors_filters_response_identity() {
+ let mut rt = cross_origin_redirect_runtime();
+ rt.set_stealth_client(std::sync::Arc::new(
+ obscura_net::StealthHttpClient::new(std::sync::Arc::new(
+ obscura_net::CookieJar::new(),
+ )),
+ ));
+ let result = rt
+ .call_function_on_for_cdp(
+ r#"async () => {
+ const response = await fetch("/start", { mode: "no-cors" });
+ return { type: response.type, url: response.url, redirected: response.redirected };
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+ assert_eq!(
+ result.value.unwrap(),
+ serde_json::json!({ "type": "opaque", "url": "", "redirected": false })
+ );
+ }
+
/// The other end of the same pair: the twenty-first redirect must
/// fail. `fetch` reports a rejected result as a `TypeError`.
#[tokio::test(flavor = "current_thread")]