Best headless browser for AI agents. Lite, Fast, High-Compatibility. Built in Rust
Rust ai-agentsai-toolsbrowserbrowser-automationcloud-browserheadlessheadless-browserkitesurfplaywrightpuppeteerrustservoweb-crawlerweb-crawlingweb-scraperweb-scraping
fix(fetch): retain referrer selection across redirect hops
ldm099c24fd8debf
diff --git a/moli-fetch/src/request.rs b/moli-fetch/src/request.rsindex 277e14c98..22a6b30e4 100644--- a/moli-fetch/src/request.rs+++ b/moli-fetch/src/request.rs@@ -762,6 +762,27 @@ impl Request {)}+ /// Retains the referrer already selected for this hop, then applies the+ /// redirect response's policy. A later, more permissive policy cannot+ /// restore an omitted referrer or its stripped path.+ pub fn update_referrer_for_redirect(+ &mut self,+ response_url: &Url,+ response_headers: &[(String, Vec<u8>)],+ ) {+ self.referrer_url = self+ .referrer_header_value(response_url)+ .and_then(|value| Url::parse(&value).ok());+ if self.referrer_url.is_none() {+ self.infer_referrer_from_initiator = false;+ }+ if let Some(policy) = crate::response_referrer_policy_from_headers(response_headers) {+ self.subresource_request_metadata+ .get_or_insert_default()+ .referrer_policy = Some(policy);+ }+ }+pub fn with_page_network_policy(mut self) -> Self {self.use_page_network_policy = true;selfdiff --git a/moli-fetch/src/runtime.rs b/moli-fetch/src/runtime.rsindex 36641c8c9..c18648eb6 100644--- a/moli-fetch/src/runtime.rs+++ b/moli-fetch/src/runtime.rs@@ -1576,6 +1576,8 @@ impl RuntimeOwner {if let Some(next_url) = next_url&& job.request.follow_redirects{+ job.request+ .update_referrer_for_redirect(&response.final_url, &response.headers);let redirect_has_extra_info = request_extra_info.is_some() && !response.from_cache;job.request.redirect_chain.push(RedirectInfo {source: crate::RedirectSource::Network,@@ -1875,6 +1877,8 @@ impl RuntimeOwner {{tracing::debug!(url = %job.current_url, "failed to store streaming redirect response in disk cache: {error}");}+ job.request+ .update_referrer_for_redirect(&final_url, &headers);let redirect_has_extra_info = request_extra_info.is_some();job.request.redirect_chain.push(RedirectInfo {source: crate::RedirectSource::Network,@@ -2268,6 +2272,8 @@ impl RuntimeOwner {{tracing::debug!(url = %job.current_url, "failed to store raw streaming redirect response in disk cache: {error}");}+ job.request+ .update_referrer_for_redirect(&final_url, &headers);let redirect_has_extra_info = request_extra_info.is_some();job.request.redirect_chain.push(RedirectInfo {source: crate::RedirectSource::Network,@@ -2409,6 +2415,8 @@ impl RuntimeOwner {if let Some(next_url) = next_url&& job.request.follow_redirects{+ job.request+ .update_referrer_for_redirect(&final_url, &cached.headers);job.request.redirect_chain.push(RedirectInfo {source: crate::RedirectSource::Network,from_url: final_url,@@ -2471,6 +2479,8 @@ impl RuntimeOwner {if let Some(next_url) = next_url&& job.request.follow_redirects{+ job.request+ .update_referrer_for_redirect(&final_url, &cached.headers);job.request.redirect_chain.push(RedirectInfo {source: crate::RedirectSource::Network,from_url: final_url,diff --git a/moli-fetch/src/tests/mod.rs b/moli-fetch/src/tests/mod.rsindex 5224f4c89..040180821 100644--- a/moli-fetch/src/tests/mod.rs+++ b/moli-fetch/src/tests/mod.rs@@ -6417,3 +6417,5 @@ fn panicking_fetch_runtime_owner_drop_does_not_replace_an_outer_unwind() {}mod range_encoding;++mod referrer_redirects;diff --git a/moli-fetch/src/tests/referrer_redirects.rs b/moli-fetch/src/tests/referrer_redirects.rsnew file mode 100644index 000000000..b2d83aaf5--- /dev/null+++ b/moli-fetch/src/tests/referrer_redirects.rs@@ -0,0 +1,338 @@+use super::*;++#[derive(Clone, Copy, Debug)]+enum Transport {+ Buffered,+ Html,+ Raw,+}++impl Transport {+ async fn fetch(self, client: &FetchClientHandle, request: Request) -> Result<Response> {+ match self {+ Self::Buffered => {+ let (tx, rx) = oneshot::channel();+ client.fetch_with_cancel_callback(+ request,+ FetchCancelHandle::new(),+ move |result| {+ let _ = tx.send(result);+ },+ )?;+ rx.await?+ }+ Self::Html => {+ client+ .fetch_html_stream(request)+ .await?+ .into_materialized_text_response()+ .await+ }+ Self::Raw => {+ client+ .fetch_raw_stream_with_cancel(request, FetchCancelHandle::new())+ .await?+ .into_lossy_materialized_text_response()+ .await+ }+ }+ }+}++#[derive(Clone, Copy, Debug)]+enum Referrer {+ Omitted,+ Origin,+ Full,+}++fn header<'a>(request: &'a str, name: &str) -> Option<&'a str> {+ request.lines().find_map(|line| {+ let (key, value) = line.split_once(':')?;+ key.eq_ignore_ascii_case(name).then(|| value.trim())+ })+}++async fn check_redirect_referrers(+ transport: Transport,+ policy: Option<&str>,+ document_policy: bool,+ response_policies: &[&str],+ expected: [Referrer; 3],+ cached: bool,+) -> Result<()> {+ let runs = if cached { 2 } else { 1 };+ // The first hop is cross-origin; both subsequent hops return to the+ // document origin. Relaxing the policy on the middle hop must not recover+ // anything removed by either of the earlier requests.+ let target = ScriptedHttpServer::spawn(+ (0..runs)+ .flat_map(|_| {+ [+ ScriptedResponse::status(302, "Found")+ .with_header("Location", "/final")+ .with_header("Referrer-Policy", "unsafe-url")+ .with_header("Cache-Control", "no-store"),+ ScriptedResponse::ok("final").with_header("Cache-Control", "no-store"),+ ]+ })+ .collect(),+ );+ let mut redirect = ScriptedResponse::status(302, "Found")+ .with_header("Location", &target.url_path("/middle"))+ .with_header(+ "Cache-Control",+ if cached { "max-age=600" } else { "no-store" },+ );+ for value in response_policies {+ redirect = redirect.with_header("rEfErReR-pOlIcY", value);+ }+ let source = ScriptedHttpServer::spawn(vec![redirect]);+ let document = Url::parse(&target.url_path("/private/document?source=1#fragment"))?;+ let full_referrer = target.url_path("/private/document?source=1");+ let origin_referrer = target.url_path("/");+ let metadata = if document_policy {+ SubresourceRequestMetadata {+ document_referrer_policy: policy.map(str::to_owned),+ ..Default::default()+ }+ } else {+ SubresourceRequestMetadata {+ referrer_policy: policy.map(str::to_owned),+ ..Default::default()+ }+ };+ let request = Request::get(&source.url())?+ .with_initiator_url(&document)+ .with_subresource_request_metadata(metadata);+ let mut config = FetchConfig::default();+ let cache_dir = cached.then(unique_test_cache_dir);+ if let Some(cache_dir) = &cache_dir {+ config.set_http_cache_dir(Some(cache_dir.display().to_string()));+ }+ let client = FetchClient::new(&config, new_shared_browser_cookie_store());+ let mut responses = Vec::new();+ for _ in 0..runs {+ responses.push(transport.fetch(&client, request.clone()).await?);+ }+ let source_requests = source.requests();+ let target_requests = target.requests();+ source.shutdown();+ target.shutdown();+ drop(client);+ if let Some(cache_dir) = cache_dir {+ fs::remove_dir_all(cache_dir)?;+ }++ let case = format!(+ "{transport:?}, {policy:?}, document={document_policy}, headers={response_policies:?}, cached={cached}"+ );+ assert_eq!(source_requests.len(), 1, "{case}: cacheable first redirect");+ assert_eq!(+ target_requests.len(),+ runs * 2,+ "{case}: middle and final hops"+ );+ let expected = expected.map(|value| match value {+ Referrer::Omitted => None,+ Referrer::Origin => Some(origin_referrer.as_str()),+ Referrer::Full => Some(full_referrer.as_str()),+ });+ assert_eq!(+ header(&source_requests[0], "Referer"),+ expected[0],+ "{case}: first hop"+ );+ for (run, response) in responses.iter().enumerate() {+ assert_eq!(response.status, 200, "{case}");+ assert_eq!(response.body_text(), "final", "{case}");+ assert_eq!(response.final_url.path(), "/final", "{case}");+ assert!(response.redirected, "{case}");+ assert!(!response.from_cache, "{case}");+ assert_eq!(response.redirect_chain.len(), 2, "{case}");+ assert_eq!(response.redirect_chain[0].from_cache, run == 1, "{case}");+ assert!(!response.redirect_chain[1].from_cache, "{case}");+ for hop in 0..2 {+ let received = &target_requests[run * 2 + hop];+ assert_eq!(+ header(received, "Referer"),+ expected[hop + 1],+ "{case}: run {run}, hop {hop}"+ );+ }+ }+ Ok(())+}++async fn check_transport(transport: Transport) -> Result<()> {+ use Referrer::{Full, Omitted, Origin};+ let cache_modes: &[bool] = match transport {+ Transport::Buffered => &[false],+ Transport::Html | Transport::Raw => &[false, true],+ };+ for &cached in cache_modes {+ for document_policy in [false, true] {+ for (policy, expected) in [+ (None, Origin),+ (Some("no-referrer"), Omitted),+ (Some("no-referrer-when-downgrade"), Full),+ (Some("origin"), Origin),+ (Some("origin-when-cross-origin"), Origin),+ (Some("same-origin"), Omitted),+ (Some("strict-origin"), Origin),+ (Some("strict-origin-when-cross-origin"), Origin),+ (Some("unsafe-url"), Full),+ ] {+ check_redirect_referrers(+ transport,+ policy,+ document_policy,+ &[],+ [expected; 3],+ cached,+ )+ .await?;+ }+ for (policy, headers, expected) in [+ ("unsafe-url", &["no-referrer"][..], [Full, Omitted, Omitted]),+ ("unsafe-url", &["origin"][..], [Full, Origin, Origin]),+ ("origin", &["unsafe-url"][..], [Origin; 3]),+ ("same-origin", &["unsafe-url"][..], [Omitted; 3]),+ ("no-referrer", &["unsafe-url"][..], [Omitted; 3]),+ ("unsafe-url", &["", "future-policy"][..], [Full; 3]),+ (+ "unsafe-url",+ &["no-referrer, origin", "future-policy"][..],+ [Full, Origin, Origin],+ ),+ (+ "unsafe-url",+ &["unsafe-url", "origin, no-referrer, future-policy"][..],+ [Full, Omitted, Omitted],+ ),+ ] {+ check_redirect_referrers(+ transport,+ Some(policy),+ document_policy,+ headers,+ expected,+ cached,+ )+ .await?;+ }+ }+ }+ Ok(())+}++#[tokio::test]+async fn buffered_redirects_preserve_referrer_state() -> Result<()> {+ check_transport(Transport::Buffered).await+}++#[tokio::test]+async fn html_redirects_preserve_referrer_state() -> Result<()> {+ check_transport(Transport::Html).await+}++#[tokio::test]+async fn raw_redirects_preserve_referrer_state() -> Result<()> {+ check_transport(Transport::Raw).await+}++#[tokio::test]+async fn manual_redirects_keep_the_current_response_and_referrer() -> Result<()> {+ let target = ScriptedHttpServer::spawn(vec![ScriptedResponse::ok("unreachable")]);+ let source = ScriptedHttpServer::spawn(vec![+ ScriptedResponse::status(302, "Found")+ .with_header("Location", &target.url())+ .with_header("Referrer-Policy", "no-referrer")+ .with_body("redirect");+ 2+ ]);+ let initiator = Url::parse("http://document.test/private?query=1#fragment")?;+ let request = Request::get(&source.url())?+ .with_initiator_url(&initiator)+ .with_subresource_request_metadata(SubresourceRequestMetadata {+ referrer_policy: Some("unsafe-url".to_owned()),+ ..Default::default()+ })+ .with_follow_redirects(false);+ let client = FetchClient::new(&FetchConfig::default(), new_shared_browser_cookie_store());+ // Manual responses use the buffered or raw API. Raw streaming intentionally+ // returns only the redirect head so its owner need not wait for the body.+ for (transport, body) in [(Transport::Buffered, "redirect"), (Transport::Raw, "")] {+ let response = transport.fetch(&client, request.clone()).await?;+ assert_eq!(response.status, 302, "{transport:?}");+ assert_eq!(response.body_text(), body, "{transport:?}");+ assert_eq!(response.final_url, request.url, "{transport:?}");+ assert!(!response.redirected, "{transport:?}");+ assert!(response.redirect_chain.is_empty(), "{transport:?}");+ }+ let received = source.requests();+ let target_hits = target.hits();+ source.shutdown();+ target.shutdown();+ assert_eq!(target_hits, 0);+ assert_eq!(received.len(), 2);+ for request in received {+ assert_eq!(+ header(&request, "Referer"),+ Some("http://document.test/private?query=1")+ );+ }+ Ok(())+}++#[test]+fn redirect_referrer_updates_preserve_request_identity_and_metadata() -> Result<()> {+ let initiator = Url::parse("https://document.test/private?query=1#fragment")?;+ let first_hop = Url::parse("https://other.test/redirect")?;+ for (policy, expected) in [+ ("same-origin", None),+ ("origin", Some("https://document.test/")),+ ] {+ let mut request = Request::new_browser(+ "POST",+ first_hop.clone(),+ None,+ Vec::new(),+ (&initiator).into(),+ )+ .with_initiator_url(&initiator)+ .with_browser_request_metadata(BrowserRequestMetadata::Fetch)+ .with_network_partition_key(Some("original-partition".to_owned()))+ .with_subresource_request_metadata(SubresourceRequestMetadata {+ referrer_policy: Some(policy.to_owned()),+ document_referrer_policy: Some("strict-origin".to_owned()),+ integrity: Some("original-integrity".to_owned()),+ });+ request.update_referrer_for_redirect(+ &first_hop,+ &[("Referrer-Policy".to_owned(), b"unsafe-url".to_vec())],+ );+ request.url = initiator.join("/returned")?;+ let headers = crate::outgoing_request_headers(&FetchConfig::default(), &request, None);+ let value = |name: &str| {+ headers+ .iter()+ .find(|(key, _)| key.eq_ignore_ascii_case(name))+ .map(|(_, value)| value.as_str())+ };+ assert_eq!(value("Referer"), expected);+ assert_eq!(value("Origin"), Some("https://document.test"));+ assert_eq!(+ request.cookie_context.initiator_url.as_ref(),+ Some(&initiator)+ );+ assert_eq!(request.network_partition_key(), Some("original-partition"));+ let metadata = request.subresource_request_metadata().unwrap();+ assert_eq!(metadata.integrity.as_deref(), Some("original-integrity"));+ assert_eq!(+ metadata.document_referrer_policy.as_deref(),+ Some("strict-origin")+ );+ }+ Ok(())+}