Skip to content

The headless browser for AI agents and web scraping

Fix Origin on same-origin non-GET requests

SGavrlda2bce8430cd
diff --git a/crates/obscura-js/src/ops.rs b/crates/obscura-js/src/ops.rs
index cfba8ff9..969c5957 100644
--- a/crates/obscura-js/src/ops.rs
+++ b/crates/obscura-js/src/ops.rs
@@ -2704,6 +2704,12 @@ fn request_origin(request_url: &str) -> Option<String> {
.map(|url| url.origin().ascii_serialization())
}
+/// Fetch "append a request `Origin` header": cross-origin requests and any
+/// request whose method is neither GET nor HEAD carry the requester's origin.
+fn sends_origin_header(method: &str, cross_origin: bool) -> bool {
+ cross_origin || !matches!(method, "GET" | "HEAD")
+}
+
/// Strip headers that must not survive a redirect. On a redirect that changes
/// origin, credential headers (`Authorization`, `Proxy-Authorization`, an
/// explicit `Cookie`) are removed so they are not forwarded to a different
@@ -3410,7 +3416,7 @@ async fn op_fetch_url(
.map(|request_origin| request_origin != page_origin)
.unwrap_or(false);
crossed_origin |= current_is_cross_origin;
- if current_is_cross_origin {
+ if sends_origin_header(current_method.as_str(), current_is_cross_origin) {
req = req.header("Origin", &page_origin);
}
@@ -3779,7 +3785,7 @@ async fn stealth_fetch_all(
let mut req_headers: HashMap<String, String> = HashMap::new();
let current_is_cross_origin = parsed_current.origin().ascii_serialization() != page_origin;
crossed_origin |= current_is_cross_origin;
- if current_is_cross_origin {
+ if sends_origin_header(&current_method, current_is_cross_origin) {
req_headers.insert("origin".to_string(), page_origin.clone());
}
for (k, v) in &current_headers {
diff --git a/crates/obscura-js/src/runtime.rs b/crates/obscura-js/src/runtime.rs
index 3fcc159c..5b42b479 100644
--- a/crates/obscura-js/src/runtime.rs
+++ b/crates/obscura-js/src/runtime.rs
@@ -18849,6 +18849,104 @@ mod tests {
assert_post_redirect_method(307, "POST").await;
}
+ /// Serves `count` requests on a fresh loopback origin and forwards each raw
+ /// request. `POST /redirect` answers 302 to `/after`, everything else 200.
+ fn raw_request_runtime(
+ count: usize,
+ ) -> (ObscuraJsRuntime, String, std::sync::mpsc::Receiver<String>) {
+ let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
+ let origin = format!("http://{}", listener.local_addr().unwrap());
+ let (requests_tx, requests_rx) = std::sync::mpsc::channel();
+ std::thread::spawn(move || {
+ use std::io::{Read as _, Write as _};
+
+ for _ in 0..count {
+ let Ok((mut stream, _)) = listener.accept() else {
+ break;
+ };
+ let mut request = [0u8; 4096];
+ let length = stream.read(&mut request).unwrap_or(0);
+ let request = String::from_utf8_lossy(&request[..length]).to_string();
+ let response = if request.starts_with("POST /redirect ") {
+ "HTTP/1.1 302 Found\r\nLocation: /after\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"
+ } else {
+ "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nConnection: close\r\n\r\nok"
+ };
+ let _ = requests_tx.send(request);
+ let _ = stream.write_all(response.as_bytes());
+ }
+ });
+ (redirect_runtime_for_origin(&origin), origin, requests_rx)
+ }
+
+ // Fetch "append a request Origin header": a same-origin request carries
+ // Origin only when its method is neither GET nor HEAD, decided per hop.
+ async fn assert_origin_sent_for_non_get_methods(
+ mut rt: ObscuraJsRuntime,
+ origin: &str,
+ requests: std::sync::mpsc::Receiver<String>,
+ ) {
+ rt.call_function_on_for_cdp(
+ r#"async () => {
+ await fetch("/get");
+ await fetch("/head", { method: "HEAD" });
+ await fetch("/post", { method: "POST", body: "x" });
+ await fetch("/put", { method: "PUT", body: "x" });
+ await fetch("/redirect", { method: "POST", body: "x" });
+ }"#,
+ None,
+ &[],
+ true,
+ true,
+ )
+ .await
+ .unwrap();
+
+ let observed: Vec<(String, Option<String>)> = (0..6)
+ .map(|_| {
+ let request = requests
+ .recv_timeout(std::time::Duration::from_secs(5))
+ .unwrap();
+ let head = request.split("\r\n\r\n").next().unwrap_or_default();
+ let target = head.split(" HTTP/").next().unwrap_or_default().to_string();
+ let origin_header = head.lines().find_map(|line| {
+ let (name, value) = line.split_once(':')?;
+ name.eq_ignore_ascii_case("origin")
+ .then(|| value.trim().to_string())
+ });
+ (target, origin_header)
+ })
+ .collect();
+ let sent = Some(origin.to_string());
+ assert_eq!(
+ observed,
+ vec![
+ ("GET /get".to_string(), None),
+ ("HEAD /head".to_string(), None),
+ ("POST /post".to_string(), sent.clone()),
+ ("PUT /put".to_string(), sent.clone()),
+ ("POST /redirect".to_string(), sent),
+ ("GET /after".to_string(), None),
+ ]
+ );
+ }
+
+ #[tokio::test(flavor = "current_thread")]
+ async fn same_origin_fetch_sends_origin_for_non_get_methods() {
+ let (rt, origin, requests) = raw_request_runtime(6);
+ assert_origin_sent_for_non_get_methods(rt, &origin, requests).await;
+ }
+
+ #[cfg(feature = "stealth")]
+ #[tokio::test(flavor = "current_thread")]
+ async fn stealth_same_origin_fetch_sends_origin_for_non_get_methods() {
+ let (mut rt, origin, requests) = raw_request_runtime(6);
+ rt.set_stealth_client(std::sync::Arc::new(obscura_net::StealthHttpClient::new(
+ std::sync::Arc::new(obscura_net::CookieJar::new()),
+ )));
+ assert_origin_sent_for_non_get_methods(rt, &origin, requests).await;
+ }
+
#[tokio::test(flavor = "current_thread")]
async fn same_origin_no_cors_redirect_keeps_response_identity() {
let mut rt = redirect_chain_runtime(2);